diff --git a/doc/mvp/pkg_install_procedure.drawio b/doc/mvp/pkg_install_procedure.drawio new file mode 100644 index 0000000..b77bb77 --- /dev/null +++ b/doc/mvp/pkg_install_procedure.drawio @@ -0,0 +1,377 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/mvp/pkg_system_demo/projectA/pkg.cfg.toml b/doc/mvp/pkg_system_demo/demo1/pkg.cfg.toml similarity index 100% rename from doc/mvp/pkg_system_demo/projectA/pkg.cfg.toml rename to doc/mvp/pkg_system_demo/demo1/pkg.cfg.toml diff --git a/doc/mvp/pkg_system_demo/projectA/pkg.lock b/doc/mvp/pkg_system_demo/demo1/pkg.lock similarity index 100% rename from doc/mvp/pkg_system_demo/projectA/pkg.lock rename to doc/mvp/pkg_system_demo/demo1/pkg.lock diff --git a/doc/mvp/pkg_system_demo/demo1/pkg_index.toml b/doc/mvp/pkg_system_demo/demo1/pkg_index.toml new file mode 100644 index 0000000..f72d8a4 --- /dev/null +++ b/doc/mvp/pkg_system_demo/demo1/pkg_index.toml @@ -0,0 +1,3 @@ +[authors] + + diff --git a/doc/mvp/pkg_system_demo/projectA/.pkg/pkg_nameB.zip b/doc/mvp/pkg_system_demo/demo1/pkgs/pkg_nameB.zip similarity index 100% rename from doc/mvp/pkg_system_demo/projectA/.pkg/pkg_nameB.zip rename to doc/mvp/pkg_system_demo/demo1/pkgs/pkg_nameB.zip diff --git a/doc/mvp/pkg_system_demo/projectA/pkg_index.toml b/doc/mvp/pkg_system_demo/projectA/pkg_index.toml deleted file mode 100644 index e69de29..0000000 diff --git a/src/component/ndn_client/ndn_client.py b/src/component/ndn_client/ndn_client.py new file mode 100644 index 0000000..c507fb1 --- /dev/null +++ b/src/component/ndn_client/ndn_client.py @@ -0,0 +1,13 @@ + +from .cid import content_id + +class ndn_get_task: + def __init__(self) -> None: + pass + +class ndn_client: + def __init__(self): + pass + + def get_file(cid:content_id,target_path:str,urls:{}=None,options:{}=None): + pass \ No newline at end of file diff --git a/src/component/package_manager/env.py b/src/component/package_manager/env.py index a5a463b..85c6b2a 100644 --- a/src/component/package_manager/env.py +++ b/src/component/package_manager/env.py @@ -7,7 +7,6 @@ from .installer import pkg_installer logger = logging.getLogger(__name__) -#pkg_env_mgr,以cfg_path为key管理多个pkg_env,是个单件 class pkg_env_mgr: _instance = None def __new__(cls): @@ -17,6 +16,7 @@ class pkg_env_mgr: def __init__(self) -> None: self._pkg_envs = {} + pass def get_env(self,cfg_path:str) -> pkg_env: @@ -34,60 +34,70 @@ class pkg_env_mgr: pass class pkg_env: - def __init__(self,cfg_path:str,inheritance = True) -> None: - self.prefixs : list[str]= [] + def __init__(self,cfg_path:str) -> None: + self.pkg_dir : str = "" + self.pkg_obj_dir : str = "" self.is_strict : bool = True - self.parent_envs : list[pkg_env] = [pkg_env_mgr().get_user_env()] + self.parent_envs : list[pkg_env] = None self.index_dbs = None - + self.cfg_path = cfg_path self._load_pkg_cfg(cfg_path) pass - def load(self,pkg_name:str,cid:None) -> pkg_media_info: + def load(self,pkg_name:str,search_parent=True) -> pkg_media_info: + pkg_path = None + pkg_id,verion_str,cid = pkg_info.parse_pkg_name(pkg_name) + if cid is None: - #lookup pkg's cid from indexDb - pkg_id,verion_str,cid = pkg_info.parse_pkg_name(pkg_name) - if cid is None: - #lookup include pkg-index-db search - pkg_info = self.lookup(pkg_id,verion_str) - if pkg_info is not None: - return self.load(pkg_id,pkg_info.cid) - - if self.is_strict is False: - #load pkg not in index-db, easy for debug - for prefix in self.prefixs: - fullpath = prefix + pkg_name - logger.debug("try load {pkg_name} at {fullpath} ...") - if pkg_env.is_valied_media(fullpath): - return self.load_pkg_media_info(fullpath) - - logger.warn(f"load {pkg_name} failed: pkg not found in index-db and pkg_dirs") - return None + if verion_str is None: + channel = self.get_pkg_channel_from_version(verion_str) + if channel is None: + pkg_path = f"{self.pkg_dir}{pkg_id}" + else: + pkg_path = f"{self.pkg_dir}{pkg_id}#{channel}" else: - #got cid here~ - return self.load(pkg_id,cid) + channel = self.get_pkg_channel_from_version(verion_str) + the_version = self.get_exact_version_from_installed(verion_str) + if the_version is None: + logger.warn(f"load {pkg_name} failed: no match version from {verion_str}") + return None + if channel is None: + pkg_path = f"{self.pkg_dir}{pkg_id}#{the_version}" + else: + pkg_path = f"{self.pkg_dir}{pkg_id}#{channel}#{the_version}" else: - #search pkg_id#cid and load pkg_media_info - for prefix in self.prefixs: - fullpath =f"{prefix}{pkg_id}#{cid}" - logger.debug(f"try load {pkg_name} from {fullpath}") - if pkg_env.is_valied_media(fullpath): - media_info = self.load_pkg_media_info(fullpath) - return media_info - - logger.warn(f"load {pkg_id}#{cid} error,not found") + pkg_path = f"{self.pkg_obj_dir}.{pkg_id}/{cid}" + + media_info = self.try_load_pkg_media_info(pkg_id,pkg_path) + if media_info is None: + if search_parent: + for parent_env in self.parent_envs: + media_info = parent_env.load(pkg_id,cid,False) + if media_info is not None: + return media_info + + logger.warn(f"load {pkg_id}#{cid} error,not found ,search_parent={search_parent}") + return None + def get_exact_version_from_installed(self,verion_str:str) -> str: + pass + + def get_pkg_channel_from_version(self,pkg_version:str) -> str: + pass def get_pkg_media_info(self,pkg_name:str)->pkg_media_info: pass - def load_pkg_media_info(self,pkg_full_path:str) -> pkg_media_info: + def try_load_pkg_media_info(self,pkg_full_path:str) -> pkg_media_info: pass + def get_installed_pkg_info(self,pkg_name:str) -> pkg_info: + pass + def lookup(self,pkg_id:str,version_str:str) -> pkg_info: - + # to make sure pkg.cid is correct, we MUST verfiy eveything here pass def get_installer(self) -> pkg_installer: @@ -96,7 +106,9 @@ class pkg_env: @classmethod def is_valied_media(pkg_full_path:str) -> bool: pass - + + def do_pkg_media_trans(self,pkg_info:pkg_info,source_path:str,target_path:str) -> bool: + pass def _load_pkg_cfg(self,cfg_path:str): if cfg_path is None: diff --git a/src/component/package_manager/installer.py b/src/component/package_manager/installer.py index 83ed220..894de8f 100644 --- a/src/component/package_manager/installer.py +++ b/src/component/package_manager/installer.py @@ -3,13 +3,13 @@ import logging import asyncio import aiohttp import aiofiles +import os -from threading import RLock - -from ndn_client import content_id -from .pkg import pkg_info +from ndn_client import content_id,ndn_client +from .pkg import pkg_info,pkg_media_info from .env import pkg_env + logger = logging.getLogger(__name__) INSTALL_TASK_STATE_DONE = 0 @@ -23,73 +23,141 @@ class install_task: def __init__(self,owner:pkg_env) -> None: self.owner = owner self.state = INSTALL_TASK_STATE_CHECK_DEPENDENCY + + self.pkg_media_info = None self.working_task = None + self.dependency_tasks = None self.error_str = None - - class pkg_installer: def __init__(self,owner_env:pkg_env) -> None: self.all_tasks = {} self.owner_env = owner_env def install(self,pkg_name:str, - install_from_dependency = False, can_upgrade = True,options = None)->Tuple[install_task,str]: - media_info = self.owner_env.get_media_info(pkg_name) - is_upgrade = False - if media_info is not None: - is_upgrade = True - if can_upgrade is False: - error_str = f"pkg:{pkg_name} already installed and can't upgrade" - logger.error(error_str) - return None,error_str - - pkg_id,version_str,content_id = pkg_info.parse_pkg_name(pkg_name) + install_from_dependency = False, can_upgrade = True,skip_depends = False,options = None)->Tuple[install_task,str]: + + the_pkg_info : pkg_info = None + is_upgrade : bool = False + need_backup : bool = False + + pkg_id,version_str,cid = pkg_info.parse_pkg_name(pkg_name) + media_info : pkg_media_info = self.owner_env.get_media_info(pkg_name) # must use index-db? + if media_info is not None: + if cid is not None: + if can_upgrade: + is_upgrade = True + else: + error_str = f"{pkg_name},{cid} already installed!" + logger.error(error_str) + return None,error_str + else: + the_pkg_info = self.owner_env.lookup(pkg_id,version_str,None) + if the_pkg_info is None: + error_str = f"{pkg_name} old version exist in local but not found in index db!" + logger.error(error_str) + return None,error_str + else: + is_upgrade = True + need_backup = True - the_pkg_info = self.owner_env.lookup(pkg_id,version_str,content_id) if the_pkg_info is None: - error_str = f"pkg:{pkg_name} ,content_id:{content_id} not found in index db" + the_pkg_info = self.owner_env.lookup(pkg_id,version_str,cid) + + if the_pkg_info is None: + error_str = f"{pkg_name} ,cid:{cid} not found in index db" logger.error(error_str) return None,error_str result_task = self.all_tasks.get(the_pkg_info.cid) if result_task is not None: - return result_task,None - + return result_task,"already installing" + + logger.info(f"start download&install {pkg_name},install_from_dependency={install_from_dependency},upgrade={is_upgrade},backup={need_backup},target_pkg_info={the_pkg_info}") result_task = install_task() self.all_tasks[the_pkg_info.cid] = result_task - async def install_pkg(): + async def download_and_install_pkg()->int: # check dependency - if await self.check_dependency(the_pkg_info) is False: - error_str = f"pkg:{pkg_name} check dependency failed" - logger.error(error_str) - result_task.error_str = error_str - result_task.state = INSTALL_TAKS_STATE_ERROR - return + if skip_depends is False: + result_task.dependency_tasks = {} + self.get_dependency_tasks(the_pkg_info,result_task.dependency_tasks) + result_task.state = INSTALL_TASK_STATE_INSTALL_DEPENDENCY + for depend_pkg_name in result_task.dependency_tasks: + # check pkg in local? + # install miss pkg + pass - # install dependency - result_task.state = INSTALL_TASK_STATE_INSTALL_DEPENDENCY - for depend_pkg_name in the_pkg_info.depends: - # TODO：这里可能需要用安装队列，直接等待安装完成可能会太慢，也可能会导致循环等待 - sub_task,err_str = self.install(depend_pkg_name,True,can_upgrade,options) + result_task.state = INSTALL_TASK_STATE_DOWNLOADING + install_full_path = "" + target_full_path = "" + old_package_full_path = "" + is_download_directy = False + + if the_pkg_info.target_media_type == the_pkg_info.source_media_type: + is_download_directy = True + if is_upgrade: + target_full_path = "" + else: + target_full_path = "" + else: + pass + + urls = self.owner_env.get_pkg_urls(the_pkg_info) + #download + client = ndn_client() # set watch + download_result = await client.get_file(the_pkg_info.cid,urls,target_full_path,options) + if download_result !=0: + result_task.state = INSTALL_TAKS_STATE_ERROR + return result_task.state + + result_task.state = INSTALL_TASK_STATE_INSTALLING + if is_download_directy is False: + install_media_result = False + install_media_result = await self.owner_env.do_pkg_media_trans(the_pkg_info,target_full_path,install_full_path) + if install_media_result is False: + result_task.state = INSTALL_TAKS_STATE_ERROR + result_task.error_str = "install media error,from {target_full_path} to {install_full_path}" + return result_task.state + + # last step,save install flag : install by manual or install by dependency + ## save cid dir + if is_upgrade: + os.rename(old_package_full_path, old_package_full_path + ".old" ) + os.rename(target_full_path,install_full_path) + ## update/create version link + + ## update pkg state + ## remove old version + + result_task.state = INSTALL_TASK_STATE_DONE + return result_task.state - result_task.working_task = asyncio.create_task(install_pkg()) + result_task.working_task = asyncio.create_task(download_and_install_pkg()) return result_task,None def uninstall(self): pass + + def get_dependency_tasks(self,pkg:pkg_info,dependency_tasks): + pass - async def check_dependency(self,pkg:pkg_info) -> bool: - #TODO:这里可能会有循环依赖导致的递归无法退出的问题，需要记录已经检查过的pkg + async def check_dependency(self,pkg:pkg_info,task_list:{}) -> bool: for depend_pkg_name in pkg.depends: + depend_task = task_list.get(depend_pkg_name) + if depend_task is not None: + logger.debug(f"{pkg.name}'s depend pkg {depend_pkg_name} already in task list") + continue + depend_task = install_task() + task_list[depend_pkg_name] = depend_task + depend_pkg_info = self.owner_env.lookup(depend_pkg_name) if depend_pkg_info is None: logger.warn(f"{pkg.name}'s depend pkg {depend_pkg_name} not found in index db") return False - if await self.check_dependency(depend_pkg_info) is False: + if await self.check_dependency(depend_pkg_info,task_list) is False: return False return True diff --git a/src/component/package_manager/pkg.py b/src/component/package_manager/pkg.py index abaeb25..ebc9194 100644 --- a/src/component/package_manager/pkg.py +++ b/src/component/package_manager/pkg.py @@ -4,7 +4,12 @@ class pkg_info: self.name = "" self.cid = None self.depends : list[str] = None - pass + self.author = None + self.remote_urls = None + self.target_media_type = "dir" + self.source_media_type = "7z" + + def parse_pkg_name(pkg_name:str) -> Tuple[str, str, str]: #return pkg_id,version_str,cid