framework code has been completed basicly. Through the use of aios_shell, we are now able to get agents run able (at openai compute node)

2023-08-27 18:07:33 -07:00
parent 1a6cf1ad7a
commit ccbef2104b
25 changed files with 1011 additions and 198 deletions
@@ -1,3 +1 @@
-from .agent_manager import AgentManager,AgentManagerClient
-from .agent import ai_agent
-from .templete import ai_agent_templete
+from .agent_manager import AgentManager
@@ -22,13 +22,13 @@ class AgentManager:
    

    def initial(self,root_dir:str) -> None:
-        self.agent_templete_env : PackageEnv = PackageEnvManager().get_env(f"{root_dir}templetes/agent_templetes.cfg")
+        self.agent_templete_env : PackageEnv = PackageEnvManager().get_env(f"{root_dir}templetes/templetes.cfg")
        self.agent_env : PackageEnv = PackageEnvManager().get_env(f"{root_dir}agents/agents.cfg")
        if self.agent_templete_env is None:
            raise Exception("agent_manager initial failed")
        
    
-    def get(self,agent_id:str) -> AIAgent:
+    async def get(self,agent_id:str) -> AIAgent:
        the_agent = self.loaded_agent_instance.get(agent_id)
        if the_agent:
            return the_agent
@@ -38,17 +38,17 @@ class AgentManager:
        if agent_media_info is None:
            return None
        
-        the_agent : AIAgent = self._load_agent_from_media(agent_media_info)
+        the_agent : AIAgent = await self._load_agent_from_media(agent_media_info)
        if the_agent is None:
            logger.warn(f"load agent {agent_id} from media failed!")
-            
+        else:
+            the_agent.start()    
        return the_agent

-
    def remove(self,agent_id:str)->int:
        pass

-    def get_templete(self,templete_id) -> AIAgentTemplete:
+    async def get_templete(self,templete_id) -> AIAgentTemplete:
        template_media_info = self.agent_templete_env.get(templete_id)
        if template_media_info is None:
            return None
@@ -61,11 +61,33 @@ class AgentManager:
    def uninstall(self,templete_id) -> int:
        pass 
    
-    def _load_templete_from_media(self,templete_media:PackageMediaInfo) -> AIAgentTemplete:
+    async def _load_templete_from_media(self,templete_media:PackageMediaInfo) -> AIAgentTemplete:
        pass

-    def _load_agent_from_media(self,agent_media:PackageMediaInfo) -> AIAgent:
-        pass
+    async def _load_agent_from_media(self,agent_media:PackageMediaInfo) -> AIAgent:
+        reader = self.agent_env._create_media_loader(agent_media)
+        if reader is None:
+            logger.error(f"create media loader for {agent_media} failed!")
+            return None
+        
+        try:
+            config_file = await reader.read("agent.toml","r")
+            if config_file is None:
+                logger.error(f"read agent config from {agent_media} failed!")
+                return None
+
+            config_data = await config_file.read()
+            config = toml.loads(config_data)
+            result_agent = AIAgent()
+            if result_agent.load_from_config(config) is False:
+                logger.error(f"load agent from {agent_media} failed!")
+                return None
+            return result_agent
+        except Exception as e:
+            logger.error(f"read agent.toml cfg from {agent_media} failed! unexpected error occurred: {str(e)}")
+            return None
+ 
+
    
    def create(self,template,agent_name,agent_last_name,agent_introduce) -> AIAgent:
        pass
@@ -4,6 +4,9 @@ class ContentId:
    def __init__(self) -> None:
        pass

+    def as_str(self) -> str:
+        pass
+
    @staticmethod
    def create_from_str(cid_str:str):
        pass
@@ -1,13 +1,117 @@
+import asyncio,aiofiles,aiohttp
+import logging
+from typing import Optional

 from .cid import ContentId 

+logger = logging.getLogger(__name__)
+
+NDN_GET_TASK_STATE_INIT = 0
+NDN_GET_TAKS_CONNECTING = 1
+NDN_GET_TASK_STATE_DOWNLOADING = 2
+NDN_GET_TASK_STATE_VERIFYING = 3
+NDN_GET_TASK_STATE_DONE = 4
+NDN_GET_TASK_STATE_ERROR = 5
+
 class NDN_GetTask:
    def __init__(self) -> None:
+        self.cid:str = None
+        self.target_path:str = None
+        self.urls:[str] = None
+        self.options:Optional[dict] = None
+
+        self.working_task = None
+        self.state = NDN_GET_TASK_STATE_INIT
+        self.total_size = 0
+        self.recv_bytes = 0
+        self.write_bytes = 0
+        self.error_str = None
+        self.chunk_queue = None
+       
+        self.retry_count = 0
+        self.used_urls = []
+        self.hash_update = None
+        
+    
+    def select_url(self,index:int)->str:
+        return self.urls[0]
+    
+    def get_chunk_for_download(self)->bytes:
        pass

 class NDN_Client:
    def __init__(self):
-        pass
+        self.cache_dir = ""
+        self.default_ndn_http_gateway = ""
+        self.all_task = {}
+        self.memory_chunk_size = 1024*1024*2
+        self.chunk_queue_size = 16

-    def get_file(self,cid:ContentId,target_path:str,urls:{}=None,options:{}=None):
-        pass
+    def load_config(self,config:dict):
+        if config.get("cache_dir"):
+            self.cache_dir = config.get("cache_dir")
+        if config.get("dndn_gateway"):
+            self.default_ndn_http_gateway = config.get("ndn_gateway")
+
+    def get_file(self,cid:ContentId,target_path:str,urls:{}=None,options:{}=None)->NDN_GetTask:
+        get_task = self.all_task.get(cid.as_str())
+        if get_task:
+            return get_task
+        else:
+            get_task = NDN_GetTask()
+            self.all_task[cid.as_str()] = get_task
+            
+            get_task.cid = cid
+            get_task.target_path = target_path
+            get_task.urls = urls
+            get_task.options = options
+            if get_task.urls is None:
+                get_task.urls = [f"{self.default_ndn_http_gateway}/{cid.as_str()}"]
+                logger.info(f"get_file {cid.as_str()} urls is None, use {get_task.urls[0]} as default")
+
+
+            async def get_file_async():
+                target_file = aiofiles.open(target, 'wb')
+                # if file exist, check hash first
+
+                http_session = aiohttp.ClientSession()
+                resp = http_session.get(get_task.select_url(0))
+                if resp.status != 200:
+                    get_task.error_str = f"get_file {cid.as_str()} failed,http status:{resp.status}"
+                    return
+                get_task.total_size = resp.content_length
+               
+                async def write_file_async():
+                    while True:
+                        chunk = await get_task.chunk_queue.pop()
+                        chunk_size = len(chunk)
+                        if not chunk or chunk_size == 0:
+                            break
+                        get_task.hash_update.update(chunk)
+                        await target_file.write(chunk)
+                        get_task.write_bytes += chunk_size
+                    
+                    #verify
+                    get_task.state = NDN_GET_TASK_STATE_VERIFYING
+                    await target_file.close()
+                    return
+
+                write_task = asyncio.create_task(write_file_async())
+                while True:
+                    await get_task.chunk_queue.pop()
+                    chunk = resp.content.read(self.memory_chunk_size) 
+                    chunk_size = len(chunk)
+                    if not chunk or chunk_size == 0:
+                        break
+
+                    get_task.recv_bytes += len(chunk)
+                    get_task.chunk_queue.push(chunk)
+
+                
+                get_task.state = NDN_GET_TASK_STATE_DONE
+                await write_task
+
+            get_task.working_task = asyncio.create_task(get_file_async())
+            return get_task
+
+    
@@ -1,13 +1,137 @@

 import logging
 import toml
+import os

-from .pkg import PackageInfo,PackageMediaInfo,
-from .env import PackageEnv
-from .installer import PackageInstaller,PackageInstallTask
+from .pkg import PackageInfo,PackageMediaInfo
+from .media_reader import MediaReader

 logger = logging.getLogger(__name__)

+
+class PackageEnv:
+    def __init__(self,cfg_path:str) -> None:
+        self.pkg_dir : str = "./pkgs/"
+        self.pkg_obj_dir : str = "./.pkgs/"
+
+        self.locked_index : str = "./pkg.lock"
+        self.is_strict : bool = True
+        self.parent_envs : list[PackageEnv] = None
+        self.index_dbs = None
+
+        self.env_dir = None
+        self.cfg_path = cfg_path
+        self._load_pkg_cfg(cfg_path)
+        pass
+
+    def load_from_config(self,config:dict) -> bool:
+        if config.get("main") is not None:
+            self.pkg_dir = os.path.abspath(self.env_dir + "/" + config["main"])
+
+        if config.get("cache") is not None:
+            self.pkg_obj_dir = os.path.abspath(self.env_dir + "/ " + config["cache"])
+
+    def load(self,pkg_name:str,search_parent=True) -> PackageMediaInfo:
+        pkg_path = None
+        pkg_id,verion_str,cid = PackageInfo.parse_pkg_name(pkg_name)
+        
+        if cid is None:
+            if verion_str is None:
+                pkg_path = f"{self.pkg_dir}/{pkg_id}"
+            else:
+                #TODO fix bug about channel here
+                channel:str = self.get_pkg_channel_from_version(verion_str)
+                the_version:str = self.get_exact_version_from_installed(verion_str)
+                if the_version is None:
+                    logger.warn(f"load {pkg_name} failed: no match version from {verion_str}")
+                    return None
+                if channel is None:
+                    pkg_path = f"{self.pkg_dir}/{pkg_id}#{the_version}"
+                else:
+                    pkg_path = f"{self.pkg_dir}/{pkg_id}#{channel}#{the_version}"
+        else:
+            pkg_path = f"{self.pkg_obj_dir}/.{pkg_id}/{cid}"
+
+        media_info:PackageMediaInfo = self.try_load_pkg_media_info(pkg_path)
+        if media_info is None:
+            if search_parent is True and self.parent_envs is not None:
+                for parent_env in self.parent_envs:
+                    media_info = parent_env.load(pkg_id,cid,False)
+                    if media_info is not None:
+                        return media_info
+            
+        if media_info is None:
+            logger.warn(f"pkg_load {pkg_id}, cid:{cid} error,not found ,search_parent={search_parent}")
+
+        return media_info
+    
+    def get_exact_version_from_installed(self,verion_str:str) -> str:
+        pass
+
+    def get_pkg_channel_from_version(self,pkg_version:str) -> str:
+        args = pkg_version.split("~")
+        if len(args) == 1:
+            return None
+        else: 
+            return args[0]
+        
+
+    def get_pkg_media_info(self,pkg_name:str)->PackageMediaInfo:
+        pass
+
+    def try_load_pkg_media_info(self,pkg_full_path:str) -> PackageMediaInfo:
+        the_result : PackageMediaInfo = None
+        logger.debug(f"try load pkng from:{pkg_full_path}")
+        if os.path.isdir(pkg_full_path):
+            the_result = PackageMediaInfo(pkg_full_path,"dir")
+        
+        return the_result
+    
+    def _create_media_loader(self,media_info:PackageMediaInfo) -> MediaReader:
+        match media_info.media_type:
+            case "dir":
+                from .media_reader import FolderMediaReader 
+                return FolderMediaReader(media_info.full_path)
+                
+        logger.error(f"create media loader for {media_info} failed!")
+        return None  
+
+    def get_installed_pkg_info(self,pkg_name:str) -> PackageInfo:
+        pass
+
+    def lookup(self,pkg_id:str,version_str:str) -> PackageInfo:
+        # to make sure pkg.cid is correct, we MUST verfiy eveything here 
+        pass
+
+    @classmethod    
+    def is_valied_media(pkg_full_path:str) -> bool:
+        pass
+    
+    def do_pkg_media_trans(self,pkg_info:PackageInfo,source_path:str,target_path:str) -> bool:
+        pass
+
+    def _load_pkg_cfg(self,cfg_path:str):
+        if cfg_path is None:
+            return
+        
+        cfg = None
+        if len(cfg_path) < 1:
+            return
+        try:
+            cfg = toml.load(cfg_path)
+            self.env_dir = os.path.abspath(os.path.dirname(cfg_path))
+            self.cfg_path = os.path.abspath(cfg_path)
+        except Exception as e:
+            logger.error(f"read pkg cfg from {cfg_path} failed! unexpected error occurred: {str(e)}")
+            return
+
+        return self.load_from_config(cfg)
+
+    
+   
+    def _preprocess_prefixs(self,prefixs):
+        pass
+
 class PackageEnvManager:
    _instance = None
    def __new__(cls):
@@ -33,105 +157,3 @@ class PackageEnvManager:

    def get_system_env(self) -> PackageEnv:
        pass
-
-class PackageEnv:
-    def __init__(self,cfg_path:str) -> None:
-        self.pkg_dir : str = ""
-        self.pkg_obj_dir : str = ""
-        self.is_strict : bool = True
-        self.parent_envs : list[PackageEnv] = None
-        self.index_dbs = None
-        
-        self.cfg_path = cfg_path
-        self._load_pkg_cfg(cfg_path)
-        pass
-
-    def load(self,pkg_name:str,search_parent=True) -> PackageMediaInfo:
-        pkg_path = None
-        pkg_id,verion_str,cid = PackageInfo.parse_pkg_name(pkg_name)
-        
-        if cid is None:
-            if verion_str is None:
-                channel:str = self.get_pkg_channel_from_version(verion_str)
-                if channel is None:
-                    pkg_path = f"{self.pkg_dir}{pkg_id}"
-                else: 
-                    pkg_path = f"{self.pkg_dir}{pkg_id}#{channel}" 
-            else:
-                channel:str = self.get_pkg_channel_from_version(verion_str)
-                the_version:str = self.get_exact_version_from_installed(verion_str)
-                if the_version is None:
-                    logger.warn(f"load {pkg_name} failed: no match version from {verion_str}")
-                    return None
-                if channel is None:
-                    pkg_path = f"{self.pkg_dir}{pkg_id}#{the_version}"
-                else:
-                    pkg_path = f"{self.pkg_dir}{pkg_id}#{channel}#{the_version}"
-        else:
-            pkg_path = f"{self.pkg_obj_dir}.{pkg_id}/{cid}"
-
-        media_info:PackageMediaInfo = self.try_load_pkg_media_info(pkg_path)
-        if media_info is None:
-            if search_parent:
-                for parent_env in self.parent_envs:
-                    media_info = parent_env.load(pkg_id,cid,False)
-                    if media_info is not None:
-                        return media_info
-                    
-        logger.warn(f"load {pkg_id}#{cid} error,not found ,search_parent={search_parent}")
-        return None
-    
-    def get_exact_version_from_installed(self,verion_str:str) -> str:
-        pass
-
-    def get_pkg_channel_from_version(self,pkg_version:str) -> str:
-        pass
-
-    def get_pkg_media_info(self,pkg_name:str)->PackageMediaInfo:
-        pass
-
-    def try_load_pkg_media_info(self,pkg_full_path:str) -> PackageMediaInfo:
-        pass
-    
-
-    def get_installed_pkg_info(self,pkg_name:str) -> PackageInfo:
-        pass
-
-    def lookup(self,pkg_id:str,version_str:str) -> PackageInfo:
-        # to make sure pkg.cid is correct, we MUST verfiy eveything here 
-        pass
-
-    def get_installer(self) -> PackageInstaller:
-        pass
-
-    @classmethod    
-    def is_valied_media(pkg_full_path:str) -> bool:
-        pass
-    
-    def do_pkg_media_trans(self,pkg_info:PackageInfo,source_path:str,target_path:str) -> bool:
-        pass
-
-    def _load_pkg_cfg(self,cfg_path:str):
-        if cfg_path is None:
-            return
-        
-        cfg = None
-        if len(cfg_path) < 1:
-            return
-        try:
-            cfg = toml.load(cfg_path)
-        except Exception as e:
-            logger.error(f"read pkg cfg from {cfg_path} failed! unexpected error occurred: {str(e)}")
-            return
-        
-        if cfg:
-            if cfg.env:
-                if cfg.env.is_strict is not None:
-                    self.is_strict = cfg.env.is_strict
-                
-                if cfg.env.prefixs is not None:
-                    self.prefixs = self._preprocess_prefixs(cfg.env.prefixs)
-    
-   
-    def _preprocess_prefixs(self,prefixs):
-        pass
@@ -1,9 +1,18 @@
-class MediaReader:
-    def __init__(self):
+from abc import ABC, abstractmethod
+import aiofiles
+
+class MediaReader(ABC):
+    @abstractmethod
+    async def read(self, inner_path:str,mode:str):
        pass

-    def load_from_media(self,media_info:str) -> int:
+
+class FolderMediaReader(MediaReader):
+    def __init__(self, root_dir:str) -> None:
+        self.root_dir = root_dir
        pass

-    def read(self, inner_path:str):
-        raise NotImplementedError
+    async def read(self, inner_path:str,mode:str):
+        full_path = self.root_dir + "/" + inner_path
+        result_file = await aiofiles.open(full_path, mode,encoding='utf-8')
+        return result_file
@@ -11,10 +11,20 @@ class PackageInfo:
        self.target_media_type = "dir"
        self.source_media_type = "7z"

-    @classmethod
-    def parse_pkg_name(cls,pkg_name:str) -> Tuple[str, str, str]:
-        """parse pkg name like test-pkg#nightly#>0.2.31#sha1:323423423 to test-pkg,nightly#>0.2.31,sha1:323423423"""
-        pass
+    @staticmethod
+    def parse_pkg_name(pkg_name:str) -> Tuple[str, str, str]:
+        """parse pkg name like test-pkg#nightly~>0.2.31#sha1:323423423 to test-pkg,nightly#>0.2.31,sha1:323423423"""
+        args = pkg_name.split("#")
+        if len(args) == 1:
+            return args[0],None,None
+        elif len(args) == 2:  
+            return args[0],None,arg[2]
+        elif len(args) == 3:
+            return args[0],args[1],args[2]
+        else:
+            logger.error(f"parse pkg name {pkg_name} failed!")
+            return None,None,None
+
    


@@ -23,8 +33,9 @@ class PackageInfo:
        return self.cid

 class PackageMediaInfo:
-    def __init__(self) -> None:
-        pass
+    def __init__(self,full_path,media_type) -> None:
+        self.media_type = media_type
+        self.full_path = full_path



@@ -0,0 +1 @@
+from .workflow_manager import WorkflowManager
@@ -0,0 +1,11 @@
+from aios_kernel import Workflow
+
+class WorkflowManager:
+    def __init__(self) -> None:
+        pass
+    
+    def initial(self,root_dir:str) -> None:
+        pass
+
+    def get_workflow(self,workflow_id:str) -> Workflow:
+        pass
				`@@ -0,0 +1 @@`
				`from .workflow_manager import WorkflowManager`