From 3d8f6fd5c8f5cddb2b607fee54f6cf4a839ddb77 Mon Sep 17 00:00:00 2001 From: Liu Zhicong Date: Mon, 14 Aug 2023 12:44:19 -0700 Subject: [PATCH 1/2] Temporary commit --- buildin/agents/english-teacher/agent.cfg | 0 .../projectA/.pkg/pkg_nameB.zip | Bin 0 -> 22 bytes doc/mvp/pkg_system_demo/projectA/pkg.cfg.toml | 0 doc/mvp/pkg_system_demo/projectA/pkg.lock | 0 .../pkg_system_demo/projectA/pkg_index.toml | 0 src/component/agent_manager/__init__.py | 3 + src/component/agent_manager/agent.py | 3 + src/component/agent_manager/agent_manager.py | 40 ++++++ src/component/agent_manager/templete.py | 3 + src/component/ndn_client/__init__.py | 1 + src/component/ndn_client/cid.py | 8 ++ src/component/package_manager/__init__.py | 0 src/component/package_manager/env.py | 124 ++++++++++++++++++ src/component/package_manager/index_db.py | 0 src/component/package_manager/index_syncer.py | 1 + src/component/package_manager/installer.py | 102 ++++++++++++++ src/component/package_manager/media_reader.py | 0 src/component/package_manager/pkg.py | 22 ++++ test/agent_test.py | 35 +++++ test/pkg.cfg.toml | 15 +++ test/spider_test.py | 23 ++++ 21 files changed, 380 insertions(+) create mode 100644 buildin/agents/english-teacher/agent.cfg create mode 100644 doc/mvp/pkg_system_demo/projectA/.pkg/pkg_nameB.zip create mode 100644 doc/mvp/pkg_system_demo/projectA/pkg.cfg.toml create mode 100644 doc/mvp/pkg_system_demo/projectA/pkg.lock create mode 100644 doc/mvp/pkg_system_demo/projectA/pkg_index.toml create mode 100644 src/component/agent_manager/__init__.py create mode 100644 src/component/agent_manager/agent.py create mode 100644 src/component/agent_manager/agent_manager.py create mode 100644 src/component/agent_manager/templete.py create mode 100644 src/component/ndn_client/__init__.py create mode 100644 src/component/ndn_client/cid.py create mode 100644 src/component/package_manager/__init__.py create mode 100644 src/component/package_manager/env.py create mode 100644 src/component/package_manager/index_db.py create mode 100644 src/component/package_manager/index_syncer.py create mode 100644 src/component/package_manager/installer.py create mode 100644 src/component/package_manager/media_reader.py create mode 100644 src/component/package_manager/pkg.py create mode 100644 test/agent_test.py create mode 100644 test/pkg.cfg.toml create mode 100644 test/spider_test.py diff --git a/buildin/agents/english-teacher/agent.cfg b/buildin/agents/english-teacher/agent.cfg new file mode 100644 index 0000000..e69de29 diff --git a/doc/mvp/pkg_system_demo/projectA/.pkg/pkg_nameB.zip b/doc/mvp/pkg_system_demo/projectA/.pkg/pkg_nameB.zip new file mode 100644 index 0000000000000000000000000000000000000000..15cb0ecb3e219d1701294bfdf0fe3f5cb5d208e7 GIT binary patch literal 22 NcmWIWW@Tf*000g10H*)| literal 0 HcmV?d00001 diff --git a/doc/mvp/pkg_system_demo/projectA/pkg.cfg.toml b/doc/mvp/pkg_system_demo/projectA/pkg.cfg.toml new file mode 100644 index 0000000..e69de29 diff --git a/doc/mvp/pkg_system_demo/projectA/pkg.lock b/doc/mvp/pkg_system_demo/projectA/pkg.lock new file mode 100644 index 0000000..e69de29 diff --git a/doc/mvp/pkg_system_demo/projectA/pkg_index.toml b/doc/mvp/pkg_system_demo/projectA/pkg_index.toml new file mode 100644 index 0000000..e69de29 diff --git a/src/component/agent_manager/__init__.py b/src/component/agent_manager/__init__.py new file mode 100644 index 0000000..863c4d1 --- /dev/null +++ b/src/component/agent_manager/__init__.py @@ -0,0 +1,3 @@ +from .agent_manager import agent_manager,agent_manager_client +from .agent import ai_agent +from .templete import ai_agent_templete \ No newline at end of file diff --git a/src/component/agent_manager/agent.py b/src/component/agent_manager/agent.py new file mode 100644 index 0000000..26f8b71 --- /dev/null +++ b/src/component/agent_manager/agent.py @@ -0,0 +1,3 @@ +class ai_agent: + def __init__(self) -> None: + pass \ No newline at end of file diff --git a/src/component/agent_manager/agent_manager.py b/src/component/agent_manager/agent_manager.py new file mode 100644 index 0000000..d837175 --- /dev/null +++ b/src/component/agent_manager/agent_manager.py @@ -0,0 +1,40 @@ + +from .agent import ai_agent +from .templete import ai_agent_templete + +class agent_manager: + _instance = None + + def __init__(self) -> None: + pass + + def __new__(cls): + if cls._instance is None: + cls._instance = super(agent_manager, cls).__new__(cls) + return cls._instance + + + def initial(self,root_dir:str) -> None: + pass + + + def get(self,agent_id) -> ai_agent: + pass + + + def get_templete(self,templete_id) -> ai_agent_templete: + pass + + def install(self,templete_id) -> int: + + installer = None + #installer.install(templete_id) + pass + + def create(self,template,agent_name,agent_last_name,agent_introduce) -> ai_agent: + pass + + +class agent_manager_client: + def __init__(self) -> None: + pass \ No newline at end of file diff --git a/src/component/agent_manager/templete.py b/src/component/agent_manager/templete.py new file mode 100644 index 0000000..87540bf --- /dev/null +++ b/src/component/agent_manager/templete.py @@ -0,0 +1,3 @@ +class ai_agent_templete: + def __init__(self) -> None: + pass \ No newline at end of file diff --git a/src/component/ndn_client/__init__.py b/src/component/ndn_client/__init__.py new file mode 100644 index 0000000..6b4585c --- /dev/null +++ b/src/component/ndn_client/__init__.py @@ -0,0 +1 @@ +from .cid import content_id \ No newline at end of file diff --git a/src/component/ndn_client/cid.py b/src/component/ndn_client/cid.py new file mode 100644 index 0000000..981df3b --- /dev/null +++ b/src/component/ndn_client/cid.py @@ -0,0 +1,8 @@ + + +class content_id: + def __init__(self) -> None: + pass + + def create_from_str(cid_str:str): + pass \ No newline at end of file diff --git a/src/component/package_manager/__init__.py b/src/component/package_manager/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/component/package_manager/env.py b/src/component/package_manager/env.py new file mode 100644 index 0000000..a5a463b --- /dev/null +++ b/src/component/package_manager/env.py @@ -0,0 +1,124 @@ + +import logging +import toml + +from .pkg import pkg_info,pkg_media_info +from .installer import pkg_installer + +logger = logging.getLogger(__name__) + +#pkg_env_mgr,以cfg_path为key管理多个pkg_env,是个单件 +class pkg_env_mgr: + _instance = None + def __new__(cls): + if cls._instance is None: + cls._instance = super(pkg_env_mgr, cls).__new__(cls) + return cls._instance + + def __init__(self) -> None: + self._pkg_envs = {} + pass + + def get_env(self,cfg_path:str) -> pkg_env: + if cfg_path in self._pkg_envs: + return self._pkg_envs[cfg_path] + else: + pkg_env = pkg_env(cfg_path) + self._pkg_envs[cfg_path] = pkg_env + return pkg_env + + def get_user_env(self) -> pkg_env: + pass + + def get_system_env(self) -> pkg_env: + pass + +class pkg_env: + def __init__(self,cfg_path:str,inheritance = True) -> None: + self.prefixs : list[str]= [] + self.is_strict : bool = True + self.parent_envs : list[pkg_env] = [pkg_env_mgr().get_user_env()] + self.index_dbs = None + + self.cfg_path = cfg_path + self._load_pkg_cfg(cfg_path) + pass + + def load(self,pkg_name:str,cid:None) -> pkg_media_info: + if cid is None: + #lookup pkg's cid from indexDb + pkg_id,verion_str,cid = pkg_info.parse_pkg_name(pkg_name) + if cid is None: + #lookup include pkg-index-db search + pkg_info = self.lookup(pkg_id,verion_str) + if pkg_info is not None: + return self.load(pkg_id,pkg_info.cid) + + if self.is_strict is False: + #load pkg not in index-db, easy for debug + for prefix in self.prefixs: + fullpath = prefix + pkg_name + logger.debug("try load {pkg_name} at {fullpath} ...") + if pkg_env.is_valied_media(fullpath): + return self.load_pkg_media_info(fullpath) + + logger.warn(f"load {pkg_name} failed: pkg not found in index-db and pkg_dirs") + return None + else: + #got cid here~ + return self.load(pkg_id,cid) + else: + #search pkg_id#cid and load pkg_media_info + for prefix in self.prefixs: + fullpath =f"{prefix}{pkg_id}#{cid}" + logger.debug(f"try load {pkg_name} from {fullpath}") + if pkg_env.is_valied_media(fullpath): + media_info = self.load_pkg_media_info(fullpath) + return media_info + + logger.warn(f"load {pkg_id}#{cid} error,not found") + + + def get_pkg_media_info(self,pkg_name:str)->pkg_media_info: + pass + + def load_pkg_media_info(self,pkg_full_path:str) -> pkg_media_info: + pass + + + def lookup(self,pkg_id:str,version_str:str) -> pkg_info: + + pass + + def get_installer(self) -> pkg_installer: + pass + + @classmethod + def is_valied_media(pkg_full_path:str) -> bool: + pass + + + def _load_pkg_cfg(self,cfg_path:str): + if cfg_path is None: + return + + cfg = None + if len(cfg_path) < 1: + return + try: + cfg = toml.load(cfg_path) + except Exception as e: + logger.error(f"read pkg cfg from {cfg_path} failed! unexpected error occurred: {str(e)}") + return + + if cfg: + if cfg.env: + if cfg.env.is_strict is not None: + self.is_strict = cfg.env.is_strict + + if cfg.env.prefixs is not None: + self.prefixs = self._preprocess_prefixs(cfg.env.prefixs) + + + def _preprocess_prefixs(self,prefixs): + pass \ No newline at end of file diff --git a/src/component/package_manager/index_db.py b/src/component/package_manager/index_db.py new file mode 100644 index 0000000..e69de29 diff --git a/src/component/package_manager/index_syncer.py b/src/component/package_manager/index_syncer.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/component/package_manager/index_syncer.py @@ -0,0 +1 @@ + diff --git a/src/component/package_manager/installer.py b/src/component/package_manager/installer.py new file mode 100644 index 0000000..83ed220 --- /dev/null +++ b/src/component/package_manager/installer.py @@ -0,0 +1,102 @@ +# installer download pkg by cid, than install it to target dir +import logging +import asyncio +import aiohttp +import aiofiles + +from threading import RLock + +from ndn_client import content_id +from .pkg import pkg_info +from .env import pkg_env + +logger = logging.getLogger(__name__) + +INSTALL_TASK_STATE_DONE = 0 +INSTALL_TASK_STATE_CHECK_DEPENDENCY = 1 +INSTALL_TASK_STATE_INSTALL_DEPENDENCY = 2 +INSTALL_TASK_STATE_DOWNLOADING = 3 +INSTALL_TASK_STATE_INSTALLING = 4 +INSTALL_TAKS_STATE_ERROR = 5 + +class install_task: + def __init__(self,owner:pkg_env) -> None: + self.owner = owner + self.state = INSTALL_TASK_STATE_CHECK_DEPENDENCY + self.working_task = None + self.error_str = None + + + +class pkg_installer: + def __init__(self,owner_env:pkg_env) -> None: + self.all_tasks = {} + self.owner_env = owner_env + + def install(self,pkg_name:str, + install_from_dependency = False, can_upgrade = True,options = None)->Tuple[install_task,str]: + media_info = self.owner_env.get_media_info(pkg_name) + is_upgrade = False + if media_info is not None: + is_upgrade = True + if can_upgrade is False: + error_str = f"pkg:{pkg_name} already installed and can't upgrade" + logger.error(error_str) + return None,error_str + + pkg_id,version_str,content_id = pkg_info.parse_pkg_name(pkg_name) + + the_pkg_info = self.owner_env.lookup(pkg_id,version_str,content_id) + if the_pkg_info is None: + error_str = f"pkg:{pkg_name} ,content_id:{content_id} not found in index db" + logger.error(error_str) + return None,error_str + + result_task = self.all_tasks.get(the_pkg_info.cid) + if result_task is not None: + return result_task,None + + result_task = install_task() + self.all_tasks[the_pkg_info.cid] = result_task + async def install_pkg(): + # check dependency + if await self.check_dependency(the_pkg_info) is False: + error_str = f"pkg:{pkg_name} check dependency failed" + logger.error(error_str) + result_task.error_str = error_str + result_task.state = INSTALL_TAKS_STATE_ERROR + return + + # install dependency + result_task.state = INSTALL_TASK_STATE_INSTALL_DEPENDENCY + for depend_pkg_name in the_pkg_info.depends: + # TODO：这里可能需要用安装队列，直接等待安装完成可能会太慢，也可能会导致循环等待 + sub_task,err_str = self.install(depend_pkg_name,True,can_upgrade,options) + + + result_task.working_task = asyncio.create_task(install_pkg()) + return result_task,None + + + def uninstall(self): + pass + + async def check_dependency(self,pkg:pkg_info) -> bool: + #TODO:这里可能会有循环依赖导致的递归无法退出的问题，需要记录已经检查过的pkg + for depend_pkg_name in pkg.depends: + depend_pkg_info = self.owner_env.lookup(depend_pkg_name) + if depend_pkg_info is None: + logger.warn(f"{pkg.name}'s depend pkg {depend_pkg_name} not found in index db") + return False + + if await self.check_dependency(depend_pkg_info) is False: + return False + + return True + + + + + + + diff --git a/src/component/package_manager/media_reader.py b/src/component/package_manager/media_reader.py new file mode 100644 index 0000000..e69de29 diff --git a/src/component/package_manager/pkg.py b/src/component/package_manager/pkg.py new file mode 100644 index 0000000..abaeb25 --- /dev/null +++ b/src/component/package_manager/pkg.py @@ -0,0 +1,22 @@ + +class pkg_info: + def __init__(self) -> None: + self.name = "" + self.cid = None + self.depends : list[str] = None + pass + + def parse_pkg_name(pkg_name:str) -> Tuple[str, str, str]: + #return pkg_id,version_str,cid + pass + + @property + def cid(self) -> str: + return self.cid + +class pkg_media_info: + def __init__(self) -> None: + pass + + + diff --git a/test/agent_test.py b/test/agent_test.py new file mode 100644 index 0000000..e0fd3e3 --- /dev/null +++ b/test/agent_test.py @@ -0,0 +1,35 @@ +import sys +sys.path.append('../src/component/') + +from agent_manager import agent_manager + +def clean_agent(): + print("clean_agent") + +def clean_agent_templete(): + print("clean_agent_templte") + +def test_agent(): + am = agent_manager() + am.initial("root_dir") + agent = am.get("english_teacher") + if agent is None: + agent_templete = am.get_templete("english_teacher") + + if agent_templete is None : + op = am.install("english_teacher") + #wait install done + + agent = am.create(agent_templete,"Tracy","Wang","Tracy Wang is my english teacher") + + print("Agent Tracy Wang load success!"); + + #print(agent.get_introduce()); + + #chat_session = agent.get_default_chat_session("master"); + #chat_session.chat("给我讲一个英文笑话!"); + #chat_session.wait_response(); + #print(chat_session.last_msg()); + +if __name__ == "__main__": + test_agent() \ No newline at end of file diff --git a/test/pkg.cfg.toml b/test/pkg.cfg.toml new file mode 100644 index 0000000..46acf0a --- /dev/null +++ b/test/pkg.cfg.toml @@ -0,0 +1,15 @@ + +[authors] +"buckyos.org" = {public="aaaa",name="BuckyOS Core Dev Team"} +[authors."cyfs.com"] +public="bbbb" +name="CYFS Core Dev Team" + +[env] +is_strict = false +prefixs = ["default","../m/s","abc"] +obj2 = {k1="bb",k2=300} +[env.obj] +k1="aa" +k2=200 + diff --git a/test/spider_test.py b/test/spider_test.py new file mode 100644 index 0000000..296f382 --- /dev/null +++ b/test/spider_test.py @@ -0,0 +1,23 @@ + + +def test_main(): + sm = service_manager() + + knownlege_base_service = sm.get('knowlege_base') + knownlege_base_service.start() + + email_spider = sm.get('email_spider') + email_spider.start() + + doc_embeding_service = sm.get('doc_embeding_service') + doc_embeding_service.start() + + ia = agents_manager().get("ai_info_assistor") + chat_session = ia.get_default_chat_session("master"); + chat_session.chat("Who responded to my issue last week"); + chat_session.wait_response(); + #print(chat_session.last_msg()); + +if __name__ == '__main__': + test_main() + \ No newline at end of file From 6b39379d7cb7829278865a2ef7216ecc9d705261 Mon Sep 17 00:00:00 2001 From: Liu Zhicong Date: Wed, 16 Aug 2023 20:06:09 -0700 Subject: [PATCH 2/2] complete pkg_system basic design and impl --- doc/mvp/pkg_install_procedure.drawio | 377 ++++++++++++++++++ .../{projectA => demo1}/pkg.cfg.toml | 0 .../{projectA => demo1}/pkg.lock | 0 doc/mvp/pkg_system_demo/demo1/pkg_index.toml | 3 + .../.pkg => demo1/pkgs}/pkg_nameB.zip | Bin .../pkg_system_demo/projectA/pkg_index.toml | 0 src/component/ndn_client/ndn_client.py | 13 + src/component/package_manager/env.py | 88 ++-- src/component/package_manager/installer.py | 142 +++++-- src/component/package_manager/pkg.py | 7 +- 10 files changed, 554 insertions(+), 76 deletions(-) create mode 100644 doc/mvp/pkg_install_procedure.drawio rename doc/mvp/pkg_system_demo/{projectA => demo1}/pkg.cfg.toml (100%) rename doc/mvp/pkg_system_demo/{projectA => demo1}/pkg.lock (100%) create mode 100644 doc/mvp/pkg_system_demo/demo1/pkg_index.toml rename doc/mvp/pkg_system_demo/{projectA/.pkg => demo1/pkgs}/pkg_nameB.zip (100%) delete mode 100644 doc/mvp/pkg_system_demo/projectA/pkg_index.toml create mode 100644 src/component/ndn_client/ndn_client.py diff --git a/doc/mvp/pkg_install_procedure.drawio b/doc/mvp/pkg_install_procedure.drawio new file mode 100644 index 0000000..b77bb77 --- /dev/null +++ b/doc/mvp/pkg_install_procedure.drawio @@ -0,0 +1,377 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/mvp/pkg_system_demo/projectA/pkg.cfg.toml b/doc/mvp/pkg_system_demo/demo1/pkg.cfg.toml similarity index 100% rename from doc/mvp/pkg_system_demo/projectA/pkg.cfg.toml rename to doc/mvp/pkg_system_demo/demo1/pkg.cfg.toml diff --git a/doc/mvp/pkg_system_demo/projectA/pkg.lock b/doc/mvp/pkg_system_demo/demo1/pkg.lock similarity index 100% rename from doc/mvp/pkg_system_demo/projectA/pkg.lock rename to doc/mvp/pkg_system_demo/demo1/pkg.lock diff --git a/doc/mvp/pkg_system_demo/demo1/pkg_index.toml b/doc/mvp/pkg_system_demo/demo1/pkg_index.toml new file mode 100644 index 0000000..f72d8a4 --- /dev/null +++ b/doc/mvp/pkg_system_demo/demo1/pkg_index.toml @@ -0,0 +1,3 @@ +[authors] + + diff --git a/doc/mvp/pkg_system_demo/projectA/.pkg/pkg_nameB.zip b/doc/mvp/pkg_system_demo/demo1/pkgs/pkg_nameB.zip similarity index 100% rename from doc/mvp/pkg_system_demo/projectA/.pkg/pkg_nameB.zip rename to doc/mvp/pkg_system_demo/demo1/pkgs/pkg_nameB.zip diff --git a/doc/mvp/pkg_system_demo/projectA/pkg_index.toml b/doc/mvp/pkg_system_demo/projectA/pkg_index.toml deleted file mode 100644 index e69de29..0000000 diff --git a/src/component/ndn_client/ndn_client.py b/src/component/ndn_client/ndn_client.py new file mode 100644 index 0000000..c507fb1 --- /dev/null +++ b/src/component/ndn_client/ndn_client.py @@ -0,0 +1,13 @@ + +from .cid import content_id + +class ndn_get_task: + def __init__(self) -> None: + pass + +class ndn_client: + def __init__(self): + pass + + def get_file(cid:content_id,target_path:str,urls:{}=None,options:{}=None): + pass \ No newline at end of file diff --git a/src/component/package_manager/env.py b/src/component/package_manager/env.py index a5a463b..85c6b2a 100644 --- a/src/component/package_manager/env.py +++ b/src/component/package_manager/env.py @@ -7,7 +7,6 @@ from .installer import pkg_installer logger = logging.getLogger(__name__) -#pkg_env_mgr,以cfg_path为key管理多个pkg_env,是个单件 class pkg_env_mgr: _instance = None def __new__(cls): @@ -17,6 +16,7 @@ class pkg_env_mgr: def __init__(self) -> None: self._pkg_envs = {} + pass def get_env(self,cfg_path:str) -> pkg_env: @@ -34,60 +34,70 @@ class pkg_env_mgr: pass class pkg_env: - def __init__(self,cfg_path:str,inheritance = True) -> None: - self.prefixs : list[str]= [] + def __init__(self,cfg_path:str) -> None: + self.pkg_dir : str = "" + self.pkg_obj_dir : str = "" self.is_strict : bool = True - self.parent_envs : list[pkg_env] = [pkg_env_mgr().get_user_env()] + self.parent_envs : list[pkg_env] = None self.index_dbs = None - + self.cfg_path = cfg_path self._load_pkg_cfg(cfg_path) pass - def load(self,pkg_name:str,cid:None) -> pkg_media_info: + def load(self,pkg_name:str,search_parent=True) -> pkg_media_info: + pkg_path = None + pkg_id,verion_str,cid = pkg_info.parse_pkg_name(pkg_name) + if cid is None: - #lookup pkg's cid from indexDb - pkg_id,verion_str,cid = pkg_info.parse_pkg_name(pkg_name) - if cid is None: - #lookup include pkg-index-db search - pkg_info = self.lookup(pkg_id,verion_str) - if pkg_info is not None: - return self.load(pkg_id,pkg_info.cid) - - if self.is_strict is False: - #load pkg not in index-db, easy for debug - for prefix in self.prefixs: - fullpath = prefix + pkg_name - logger.debug("try load {pkg_name} at {fullpath} ...") - if pkg_env.is_valied_media(fullpath): - return self.load_pkg_media_info(fullpath) - - logger.warn(f"load {pkg_name} failed: pkg not found in index-db and pkg_dirs") - return None + if verion_str is None: + channel = self.get_pkg_channel_from_version(verion_str) + if channel is None: + pkg_path = f"{self.pkg_dir}{pkg_id}" + else: + pkg_path = f"{self.pkg_dir}{pkg_id}#{channel}" else: - #got cid here~ - return self.load(pkg_id,cid) + channel = self.get_pkg_channel_from_version(verion_str) + the_version = self.get_exact_version_from_installed(verion_str) + if the_version is None: + logger.warn(f"load {pkg_name} failed: no match version from {verion_str}") + return None + if channel is None: + pkg_path = f"{self.pkg_dir}{pkg_id}#{the_version}" + else: + pkg_path = f"{self.pkg_dir}{pkg_id}#{channel}#{the_version}" else: - #search pkg_id#cid and load pkg_media_info - for prefix in self.prefixs: - fullpath =f"{prefix}{pkg_id}#{cid}" - logger.debug(f"try load {pkg_name} from {fullpath}") - if pkg_env.is_valied_media(fullpath): - media_info = self.load_pkg_media_info(fullpath) - return media_info - - logger.warn(f"load {pkg_id}#{cid} error,not found") + pkg_path = f"{self.pkg_obj_dir}.{pkg_id}/{cid}" + + media_info = self.try_load_pkg_media_info(pkg_id,pkg_path) + if media_info is None: + if search_parent: + for parent_env in self.parent_envs: + media_info = parent_env.load(pkg_id,cid,False) + if media_info is not None: + return media_info + + logger.warn(f"load {pkg_id}#{cid} error,not found ,search_parent={search_parent}") + return None + def get_exact_version_from_installed(self,verion_str:str) -> str: + pass + + def get_pkg_channel_from_version(self,pkg_version:str) -> str: + pass def get_pkg_media_info(self,pkg_name:str)->pkg_media_info: pass - def load_pkg_media_info(self,pkg_full_path:str) -> pkg_media_info: + def try_load_pkg_media_info(self,pkg_full_path:str) -> pkg_media_info: pass + def get_installed_pkg_info(self,pkg_name:str) -> pkg_info: + pass + def lookup(self,pkg_id:str,version_str:str) -> pkg_info: - + # to make sure pkg.cid is correct, we MUST verfiy eveything here pass def get_installer(self) -> pkg_installer: @@ -96,7 +106,9 @@ class pkg_env: @classmethod def is_valied_media(pkg_full_path:str) -> bool: pass - + + def do_pkg_media_trans(self,pkg_info:pkg_info,source_path:str,target_path:str) -> bool: + pass def _load_pkg_cfg(self,cfg_path:str): if cfg_path is None: diff --git a/src/component/package_manager/installer.py b/src/component/package_manager/installer.py index 83ed220..894de8f 100644 --- a/src/component/package_manager/installer.py +++ b/src/component/package_manager/installer.py @@ -3,13 +3,13 @@ import logging import asyncio import aiohttp import aiofiles +import os -from threading import RLock - -from ndn_client import content_id -from .pkg import pkg_info +from ndn_client import content_id,ndn_client +from .pkg import pkg_info,pkg_media_info from .env import pkg_env + logger = logging.getLogger(__name__) INSTALL_TASK_STATE_DONE = 0 @@ -23,73 +23,141 @@ class install_task: def __init__(self,owner:pkg_env) -> None: self.owner = owner self.state = INSTALL_TASK_STATE_CHECK_DEPENDENCY + + self.pkg_media_info = None self.working_task = None + self.dependency_tasks = None self.error_str = None - - class pkg_installer: def __init__(self,owner_env:pkg_env) -> None: self.all_tasks = {} self.owner_env = owner_env def install(self,pkg_name:str, - install_from_dependency = False, can_upgrade = True,options = None)->Tuple[install_task,str]: - media_info = self.owner_env.get_media_info(pkg_name) - is_upgrade = False - if media_info is not None: - is_upgrade = True - if can_upgrade is False: - error_str = f"pkg:{pkg_name} already installed and can't upgrade" - logger.error(error_str) - return None,error_str - - pkg_id,version_str,content_id = pkg_info.parse_pkg_name(pkg_name) + install_from_dependency = False, can_upgrade = True,skip_depends = False,options = None)->Tuple[install_task,str]: + + the_pkg_info : pkg_info = None + is_upgrade : bool = False + need_backup : bool = False + + pkg_id,version_str,cid = pkg_info.parse_pkg_name(pkg_name) + media_info : pkg_media_info = self.owner_env.get_media_info(pkg_name) # must use index-db? + if media_info is not None: + if cid is not None: + if can_upgrade: + is_upgrade = True + else: + error_str = f"{pkg_name},{cid} already installed!" + logger.error(error_str) + return None,error_str + else: + the_pkg_info = self.owner_env.lookup(pkg_id,version_str,None) + if the_pkg_info is None: + error_str = f"{pkg_name} old version exist in local but not found in index db!" + logger.error(error_str) + return None,error_str + else: + is_upgrade = True + need_backup = True - the_pkg_info = self.owner_env.lookup(pkg_id,version_str,content_id) if the_pkg_info is None: - error_str = f"pkg:{pkg_name} ,content_id:{content_id} not found in index db" + the_pkg_info = self.owner_env.lookup(pkg_id,version_str,cid) + + if the_pkg_info is None: + error_str = f"{pkg_name} ,cid:{cid} not found in index db" logger.error(error_str) return None,error_str result_task = self.all_tasks.get(the_pkg_info.cid) if result_task is not None: - return result_task,None - + return result_task,"already installing" + + logger.info(f"start download&install {pkg_name},install_from_dependency={install_from_dependency},upgrade={is_upgrade},backup={need_backup},target_pkg_info={the_pkg_info}") result_task = install_task() self.all_tasks[the_pkg_info.cid] = result_task - async def install_pkg(): + async def download_and_install_pkg()->int: # check dependency - if await self.check_dependency(the_pkg_info) is False: - error_str = f"pkg:{pkg_name} check dependency failed" - logger.error(error_str) - result_task.error_str = error_str - result_task.state = INSTALL_TAKS_STATE_ERROR - return + if skip_depends is False: + result_task.dependency_tasks = {} + self.get_dependency_tasks(the_pkg_info,result_task.dependency_tasks) + result_task.state = INSTALL_TASK_STATE_INSTALL_DEPENDENCY + for depend_pkg_name in result_task.dependency_tasks: + # check pkg in local? + # install miss pkg + pass - # install dependency - result_task.state = INSTALL_TASK_STATE_INSTALL_DEPENDENCY - for depend_pkg_name in the_pkg_info.depends: - # TODO：这里可能需要用安装队列，直接等待安装完成可能会太慢，也可能会导致循环等待 - sub_task,err_str = self.install(depend_pkg_name,True,can_upgrade,options) + result_task.state = INSTALL_TASK_STATE_DOWNLOADING + install_full_path = "" + target_full_path = "" + old_package_full_path = "" + is_download_directy = False + + if the_pkg_info.target_media_type == the_pkg_info.source_media_type: + is_download_directy = True + if is_upgrade: + target_full_path = "" + else: + target_full_path = "" + else: + pass + + urls = self.owner_env.get_pkg_urls(the_pkg_info) + #download + client = ndn_client() # set watch + download_result = await client.get_file(the_pkg_info.cid,urls,target_full_path,options) + if download_result !=0: + result_task.state = INSTALL_TAKS_STATE_ERROR + return result_task.state + + result_task.state = INSTALL_TASK_STATE_INSTALLING + if is_download_directy is False: + install_media_result = False + install_media_result = await self.owner_env.do_pkg_media_trans(the_pkg_info,target_full_path,install_full_path) + if install_media_result is False: + result_task.state = INSTALL_TAKS_STATE_ERROR + result_task.error_str = "install media error,from {target_full_path} to {install_full_path}" + return result_task.state + + # last step,save install flag : install by manual or install by dependency + ## save cid dir + if is_upgrade: + os.rename(old_package_full_path, old_package_full_path + ".old" ) + os.rename(target_full_path,install_full_path) + ## update/create version link + + ## update pkg state + ## remove old version + + result_task.state = INSTALL_TASK_STATE_DONE + return result_task.state - result_task.working_task = asyncio.create_task(install_pkg()) + result_task.working_task = asyncio.create_task(download_and_install_pkg()) return result_task,None def uninstall(self): pass + + def get_dependency_tasks(self,pkg:pkg_info,dependency_tasks): + pass - async def check_dependency(self,pkg:pkg_info) -> bool: - #TODO:这里可能会有循环依赖导致的递归无法退出的问题，需要记录已经检查过的pkg + async def check_dependency(self,pkg:pkg_info,task_list:{}) -> bool: for depend_pkg_name in pkg.depends: + depend_task = task_list.get(depend_pkg_name) + if depend_task is not None: + logger.debug(f"{pkg.name}'s depend pkg {depend_pkg_name} already in task list") + continue + depend_task = install_task() + task_list[depend_pkg_name] = depend_task + depend_pkg_info = self.owner_env.lookup(depend_pkg_name) if depend_pkg_info is None: logger.warn(f"{pkg.name}'s depend pkg {depend_pkg_name} not found in index db") return False - if await self.check_dependency(depend_pkg_info) is False: + if await self.check_dependency(depend_pkg_info,task_list) is False: return False return True diff --git a/src/component/package_manager/pkg.py b/src/component/package_manager/pkg.py index abaeb25..ebc9194 100644 --- a/src/component/package_manager/pkg.py +++ b/src/component/package_manager/pkg.py @@ -4,7 +4,12 @@ class pkg_info: self.name = "" self.cid = None self.depends : list[str] = None - pass + self.author = None + self.remote_urls = None + self.target_media_type = "dir" + self.source_media_type = "7z" + + def parse_pkg_name(pkg_name:str) -> Tuple[str, str, str]: #return pkg_id,version_str,cid