framework code has been completed basicly. Through the use of aios_shell, we are now able to get agents run able (at openai compute node)

src/component/ndn_client/ndn_client.py

@@ -1,13 +1,117 @@
+import asyncio,aiofiles,aiohttp
+import logging
+from typing import Optional
 
 from .cid import ContentId 
 
+logger = logging.getLogger(__name__)
+
+NDN_GET_TASK_STATE_INIT = 0
+NDN_GET_TAKS_CONNECTING = 1
+NDN_GET_TASK_STATE_DOWNLOADING = 2
+NDN_GET_TASK_STATE_VERIFYING = 3
+NDN_GET_TASK_STATE_DONE = 4
+NDN_GET_TASK_STATE_ERROR = 5
+
 class NDN_GetTask:
     def __init__(self) -> None:
+        self.cid:str = None
+        self.target_path:str = None
+        self.urls:[str] = None
+        self.options:Optional[dict] = None
+
+        self.working_task = None
+        self.state = NDN_GET_TASK_STATE_INIT
+        self.total_size = 0
+        self.recv_bytes = 0
+        self.write_bytes = 0
+        self.error_str = None
+        self.chunk_queue = None
+       
+        self.retry_count = 0
+        self.used_urls = []
+        self.hash_update = None
+        
+    
+    def select_url(self,index:int)->str:
+        return self.urls[0]
+    
+    def get_chunk_for_download(self)->bytes:
         pass
 
 class NDN_Client:
     def __init__(self):
-        pass
+        self.cache_dir = ""
+        self.default_ndn_http_gateway = ""
+        self.all_task = {}
+        self.memory_chunk_size = 1024*1024*2
+        self.chunk_queue_size = 16
 
-    def get_file(self,cid:ContentId,target_path:str,urls:{}=None,options:{}=None):
-        pass
+    def load_config(self,config:dict):
+        if config.get("cache_dir"):
+            self.cache_dir = config.get("cache_dir")
+        if config.get("dndn_gateway"):
+            self.default_ndn_http_gateway = config.get("ndn_gateway")
+
+    def get_file(self,cid:ContentId,target_path:str,urls:{}=None,options:{}=None)->NDN_GetTask:
+        get_task = self.all_task.get(cid.as_str())
+        if get_task:
+            return get_task
+        else:
+            get_task = NDN_GetTask()
+            self.all_task[cid.as_str()] = get_task
+            
+            get_task.cid = cid
+            get_task.target_path = target_path
+            get_task.urls = urls
+            get_task.options = options
+            if get_task.urls is None:
+                get_task.urls = [f"{self.default_ndn_http_gateway}/{cid.as_str()}"]
+                logger.info(f"get_file {cid.as_str()} urls is None, use {get_task.urls[0]} as default")
+
+
+            async def get_file_async():
+                target_file = aiofiles.open(target, 'wb')
+                # if file exist, check hash first
+
+                http_session = aiohttp.ClientSession()
+                resp = http_session.get(get_task.select_url(0))
+                if resp.status != 200:
+                    get_task.error_str = f"get_file {cid.as_str()} failed,http status:{resp.status}"
+                    return
+                get_task.total_size = resp.content_length
+               
+                async def write_file_async():
+                    while True:
+                        chunk = await get_task.chunk_queue.pop()
+                        chunk_size = len(chunk)
+                        if not chunk or chunk_size == 0:
+                            break
+                        get_task.hash_update.update(chunk)
+                        await target_file.write(chunk)
+                        get_task.write_bytes += chunk_size
+                    
+                    #verify
+                    get_task.state = NDN_GET_TASK_STATE_VERIFYING
+                    await target_file.close()
+                    return
+
+                write_task = asyncio.create_task(write_file_async())
+                while True:
+                    await get_task.chunk_queue.pop()
+                    chunk = resp.content.read(self.memory_chunk_size) 
+                    chunk_size = len(chunk)
+                    if not chunk or chunk_size == 0:
+                        break
+
+                    get_task.recv_bytes += len(chunk)
+                    get_task.chunk_queue.push(chunk)
+
+                
+                get_task.state = NDN_GET_TASK_STATE_DONE
+                await write_task
+
+            get_task.working_task = asyncio.create_task(get_file_async())
+            return get_task
+
+