complete pkg_system basic design and impl

src/component/package_manager/env.py

@@ -7,7 +7,6 @@ from .installer import pkg_installer
 
 logger = logging.getLogger(__name__)
 
-#pkg_env_mgr,以cfg_path为key管理多个pkg_env,是个单件
 class pkg_env_mgr:
     _instance = None
     def __new__(cls):
@@ -17,6 +16,7 @@ class pkg_env_mgr:
     
     def __init__(self) -> None:
         self._pkg_envs = {}
+       
         pass
 
     def get_env(self,cfg_path:str) -> pkg_env:
@@ -34,60 +34,70 @@ class pkg_env_mgr:
         pass
 
 class pkg_env:
-    def __init__(self,cfg_path:str,inheritance = True) -> None:
-        self.prefixs : list[str]= []
+    def __init__(self,cfg_path:str) -> None:
+        self.pkg_dir : str = ""
+        self.pkg_obj_dir : str = ""
         self.is_strict : bool = True
-        self.parent_envs : list[pkg_env] = [pkg_env_mgr().get_user_env()]
+        self.parent_envs : list[pkg_env] = None
         self.index_dbs = None
-
+        
         self.cfg_path = cfg_path
         self._load_pkg_cfg(cfg_path)
         pass
 
-    def load(self,pkg_name:str,cid:None) -> pkg_media_info:
+    def load(self,pkg_name:str,search_parent=True) -> pkg_media_info:
+        pkg_path = None
+        pkg_id,verion_str,cid = pkg_info.parse_pkg_name(pkg_name)
+        
         if cid is None:
-            #lookup pkg's cid from indexDb
-            pkg_id,verion_str,cid = pkg_info.parse_pkg_name(pkg_name)
-            if cid is None:
-                #lookup include pkg-index-db search
-                pkg_info = self.lookup(pkg_id,verion_str)
-                if pkg_info is not None:
-                    return self.load(pkg_id,pkg_info.cid)
-                
-                if self.is_strict is False:
-                    #load pkg not in index-db, easy for debug
-                    for prefix in self.prefixs:
-                        fullpath = prefix + pkg_name
-                        logger.debug("try load {pkg_name} at {fullpath} ...")
-                        if pkg_env.is_valied_media(fullpath):
-                            return self.load_pkg_media_info(fullpath)
-                        
-                logger.warn(f"load {pkg_name} failed: pkg not found in index-db and pkg_dirs")
-                return None
+            if verion_str is None:
+                channel = self.get_pkg_channel_from_version(verion_str)
+                if channel is None:
+                    pkg_path = f"{self.pkg_dir}{pkg_id}"
+                else: 
+                    pkg_path = f"{self.pkg_dir}{pkg_id}#{channel}" 
             else:
-                #got cid here~
-                return self.load(pkg_id,cid)
+                channel = self.get_pkg_channel_from_version(verion_str)
+                the_version = self.get_exact_version_from_installed(verion_str)
+                if the_version is None:
+                    logger.warn(f"load {pkg_name} failed: no match version from {verion_str}")
+                    return None
+                if channel is None:
+                    pkg_path = f"{self.pkg_dir}{pkg_id}#{the_version}"
+                else:
+                    pkg_path = f"{self.pkg_dir}{pkg_id}#{channel}#{the_version}"
         else:
-            #search pkg_id#cid and load pkg_media_info
-            for prefix in self.prefixs:
-                fullpath =f"{prefix}{pkg_id}#{cid}"
-                logger.debug(f"try load {pkg_name} from {fullpath}")
-                if pkg_env.is_valied_media(fullpath):
-                    media_info = self.load_pkg_media_info(fullpath)
-                    return media_info
-            
-            logger.warn(f"load {pkg_id}#{cid} error,not found")
+            pkg_path = f"{self.pkg_obj_dir}.{pkg_id}/{cid}"
+
+        media_info = self.try_load_pkg_media_info(pkg_id,pkg_path)
+        if media_info is None:
+            if search_parent:
+                for parent_env in self.parent_envs:
+                    media_info = parent_env.load(pkg_id,cid,False)
+                    if media_info is not None:
+                        return media_info
+                    
+        logger.warn(f"load {pkg_id}#{cid} error,not found ,search_parent={search_parent}")
+        return None
     
+    def get_exact_version_from_installed(self,verion_str:str) -> str:
+        pass
+
+    def get_pkg_channel_from_version(self,pkg_version:str) -> str:
+        pass
 
     def get_pkg_media_info(self,pkg_name:str)->pkg_media_info:
         pass
 
-    def load_pkg_media_info(self,pkg_full_path:str) -> pkg_media_info:
+    def try_load_pkg_media_info(self,pkg_full_path:str) -> pkg_media_info:
         pass
     
 
+    def get_installed_pkg_info(self,pkg_name:str) -> pkg_info:
+        pass
+
     def lookup(self,pkg_id:str,version_str:str) -> pkg_info:
-        
+        # to make sure pkg.cid is correct, we MUST verfiy eveything here 
         pass
 
     def get_installer(self) -> pkg_installer:
@@ -96,7 +106,9 @@ class pkg_env:
     @classmethod    
     def is_valied_media(pkg_full_path:str) -> bool:
         pass
-
+    
+    def do_pkg_media_trans(self,pkg_info:pkg_info,source_path:str,target_path:str) -> bool:
+        pass
 
     def _load_pkg_cfg(self,cfg_path:str):
         if cfg_path is None:

src/component/package_manager/installer.py

@@ -3,13 +3,13 @@ import logging
 import asyncio
 import aiohttp
 import aiofiles
+import os
 
-from threading import RLock
-
-from ndn_client import content_id
-from .pkg import pkg_info
+from ndn_client import content_id,ndn_client
+from .pkg import pkg_info,pkg_media_info
 from .env import pkg_env
 
+
 logger = logging.getLogger(__name__)
 
 INSTALL_TASK_STATE_DONE = 0
@@ -23,73 +23,141 @@ class install_task:
     def __init__(self,owner:pkg_env) -> None:
         self.owner = owner
         self.state = INSTALL_TASK_STATE_CHECK_DEPENDENCY
+
+        self.pkg_media_info = None
         self.working_task = None
+        self.dependency_tasks = None
         self.error_str = None
 
-
-
 class pkg_installer:
     def __init__(self,owner_env:pkg_env) -> None:
         self.all_tasks = {}
         self.owner_env = owner_env
     
     def install(self,pkg_name:str,
-                install_from_dependency = False, can_upgrade = True,options = None)->Tuple[install_task,str]:
-        media_info = self.owner_env.get_media_info(pkg_name)
-        is_upgrade = False
-        if media_info is not None:
-            is_upgrade = True
-            if can_upgrade is False:
-                error_str = f"pkg:{pkg_name} already installed and can't upgrade"
-                logger.error(error_str)
-                return None,error_str
-        
-        pkg_id,version_str,content_id = pkg_info.parse_pkg_name(pkg_name)
+                install_from_dependency = False, can_upgrade = True,skip_depends = False,options = None)->Tuple[install_task,str]:
+
+        the_pkg_info : pkg_info = None
+        is_upgrade : bool = False
+        need_backup : bool = False
+        
+        pkg_id,version_str,cid = pkg_info.parse_pkg_name(pkg_name)
+        media_info : pkg_media_info = self.owner_env.get_media_info(pkg_name) # must use index-db?
+        if media_info is not None:
+            if cid is not None:
+                if can_upgrade:
+                    is_upgrade = True
+                else:
+                    error_str = f"{pkg_name},{cid} already installed!"
+                    logger.error(error_str)
+                    return None,error_str
+            else:
+                the_pkg_info = self.owner_env.lookup(pkg_id,version_str,None)
+                if the_pkg_info is None:
+                    error_str = f"{pkg_name} old version exist in local but not found in index db!"
+                    logger.error(error_str)
+                    return None,error_str
+                else:
+                    is_upgrade = True
+                    need_backup = True
 
-        the_pkg_info = self.owner_env.lookup(pkg_id,version_str,content_id)
         if the_pkg_info is None:
-            error_str = f"pkg:{pkg_name} ,content_id:{content_id} not found in index db"
+            the_pkg_info = self.owner_env.lookup(pkg_id,version_str,cid)
+        
+        if the_pkg_info is None:
+            error_str = f"{pkg_name} ,cid:{cid} not found in index db"
             logger.error(error_str)
             return None,error_str
         
         result_task = self.all_tasks.get(the_pkg_info.cid)
         if result_task is not None:
-            return result_task,None
-
+            return result_task,"already installing"
+        
+        logger.info(f"start download&install {pkg_name},install_from_dependency={install_from_dependency},upgrade={is_upgrade},backup={need_backup},target_pkg_info={the_pkg_info}")
         result_task = install_task()
         self.all_tasks[the_pkg_info.cid] = result_task
-        async def install_pkg():
+        async def download_and_install_pkg()->int:
             # check dependency
-            if await self.check_dependency(the_pkg_info) is False:
-                error_str = f"pkg:{pkg_name} check dependency failed"
-                logger.error(error_str)
-                result_task.error_str = error_str
-                result_task.state = INSTALL_TAKS_STATE_ERROR
-                return
+            if skip_depends is False:
+                result_task.dependency_tasks = {}
+                self.get_dependency_tasks(the_pkg_info,result_task.dependency_tasks)
+                result_task.state = INSTALL_TASK_STATE_INSTALL_DEPENDENCY
+                for depend_pkg_name in result_task.dependency_tasks:
+                    # check pkg in local?
+                    # install miss pkg
+                    pass
             
-            # install dependency
-            result_task.state = INSTALL_TASK_STATE_INSTALL_DEPENDENCY
-            for depend_pkg_name in the_pkg_info.depends:
-                # TODO：这里可能需要用安装队列，直接等待安装完成可能会太慢，也可能会导致循环等待
-                sub_task,err_str = self.install(depend_pkg_name,True,can_upgrade,options)
+            result_task.state = INSTALL_TASK_STATE_DOWNLOADING
+            install_full_path = ""
+            target_full_path = ""
+            old_package_full_path = ""
+            is_download_directy = False
+
+            if the_pkg_info.target_media_type == the_pkg_info.source_media_type:
+                is_download_directy = True
+                if is_upgrade:
+                    target_full_path = ""
+                else:
+                    target_full_path = ""
+            else:
+                pass
+
+            urls = self.owner_env.get_pkg_urls(the_pkg_info)
+            #download
+            client = ndn_client() # set watch
+            download_result = await client.get_file(the_pkg_info.cid,urls,target_full_path,options) 
+            if download_result !=0:
+                result_task.state = INSTALL_TAKS_STATE_ERROR
+                return result_task.state
+            
+            result_task.state = INSTALL_TASK_STATE_INSTALLING
+            if is_download_directy is False:
+                install_media_result = False
+                install_media_result = await self.owner_env.do_pkg_media_trans(the_pkg_info,target_full_path,install_full_path)
+                if install_media_result is False:
+                    result_task.state = INSTALL_TAKS_STATE_ERROR
+                    result_task.error_str = "install media error,from {target_full_path} to {install_full_path}"
+                    return result_task.state
+               
+            # last step,save install flag : install by manual or install by dependency
+            ## save cid dir
+            if is_upgrade:
+                os.rename(old_package_full_path, old_package_full_path + ".old" )
+                os.rename(target_full_path,install_full_path)
+            ## update/create version link
+            
+            ## update pkg state
+            ## remove old version
+    
+            result_task.state = INSTALL_TASK_STATE_DONE
+            return result_task.state
                 
 
-        result_task.working_task = asyncio.create_task(install_pkg())
+        result_task.working_task = asyncio.create_task(download_and_install_pkg())
         return result_task,None
 
         
     def uninstall(self):
         pass
+    
+    def get_dependency_tasks(self,pkg:pkg_info,dependency_tasks):
+        pass
 
-    async def check_dependency(self,pkg:pkg_info) -> bool:
-        #TODO:这里可能会有循环依赖导致的递归无法退出的问题，需要记录已经检查过的pkg
+    async def check_dependency(self,pkg:pkg_info,task_list:{}) -> bool:
         for depend_pkg_name in pkg.depends:
+            depend_task = task_list.get(depend_pkg_name)
+            if depend_task is not None:
+                logger.debug(f"{pkg.name}'s depend pkg {depend_pkg_name} already in task list")
+                continue
+            depend_task = install_task()
+            task_list[depend_pkg_name] = depend_task
+            
             depend_pkg_info = self.owner_env.lookup(depend_pkg_name)
             if depend_pkg_info is None:
                 logger.warn(f"{pkg.name}'s depend pkg {depend_pkg_name} not found in index db")
                 return False
             
-            if await self.check_dependency(depend_pkg_info) is False:
+            if await self.check_dependency(depend_pkg_info,task_list) is False:
                 return False
     
         return True

src/component/package_manager/pkg.py

@@ -4,7 +4,12 @@ class pkg_info:
         self.name = ""
         self.cid = None
         self.depends : list[str] = None
-        pass
+        self.author = None
+        self.remote_urls = None
+        self.target_media_type = "dir"
+        self.source_media_type = "7z"
+
+        
     
     def parse_pkg_name(pkg_name:str) -> Tuple[str, str, str]:
         #return pkg_id,version_str,cid