gyt/opendan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

framework code has been completed basicly. Through the use of aios_shell, we are now able to get agents run able (at openai compute node)

Browse Source
This commit is contained in:
Liu Zhicong
2023-08-27 18:07:33 -07:00
parent 1a6cf1ad7a
commit ccbef2104b
25 changed files with 1011 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/component/package_manager/env.py
+127 -105
View File
@@ -1,13 +1,137 @@
import logging
import toml
import os
from .pkg import PackageInfo,PackageMediaInfo,
from .env import PackageEnv
from .installer import PackageInstaller,PackageInstallTask
from .pkg import PackageInfo,PackageMediaInfo
from .media_reader import MediaReader
logger = logging.getLogger(__name__)
class PackageEnv:
def __init__(self,cfg_path:str) -> None:
self.pkg_dir : str = "./pkgs/"
self.pkg_obj_dir : str = "./.pkgs/"
self.locked_index : str = "./pkg.lock"
self.is_strict : bool = True
self.parent_envs : list[PackageEnv] = None
self.index_dbs = None
self.env_dir = None
self.cfg_path = cfg_path
self._load_pkg_cfg(cfg_path)
pass
def load_from_config(self,config:dict) -> bool:
if config.get("main") is not None:
self.pkg_dir = os.path.abspath(self.env_dir + "/" + config["main"])
if config.get("cache") is not None:
self.pkg_obj_dir = os.path.abspath(self.env_dir + "/ " + config["cache"])
def load(self,pkg_name:str,search_parent=True) -> PackageMediaInfo:
pkg_path = None
pkg_id,verion_str,cid = PackageInfo.parse_pkg_name(pkg_name)
if cid is None:
if verion_str is None:
pkg_path = f"{self.pkg_dir}/{pkg_id}"
else:
#TODO fix bug about channel here
channel:str = self.get_pkg_channel_from_version(verion_str)
the_version:str = self.get_exact_version_from_installed(verion_str)
if the_version is None:
logger.warn(f"load {pkg_name} failed: no match version from {verion_str}")
return None
if channel is None:
pkg_path = f"{self.pkg_dir}/{pkg_id}#{the_version}"
else:
pkg_path = f"{self.pkg_dir}/{pkg_id}#{channel}#{the_version}"
else:
pkg_path = f"{self.pkg_obj_dir}/.{pkg_id}/{cid}"
media_info:PackageMediaInfo = self.try_load_pkg_media_info(pkg_path)
if media_info is None:
if search_parent is True and self.parent_envs is not None:
for parent_env in self.parent_envs:
media_info = parent_env.load(pkg_id,cid,False)
if media_info is not None:
return media_info
if media_info is None:
logger.warn(f"pkg_load {pkg_id}, cid:{cid} error,not found ,search_parent={search_parent}")
return media_info
def get_exact_version_from_installed(self,verion_str:str) -> str:
pass
def get_pkg_channel_from_version(self,pkg_version:str) -> str:
args = pkg_version.split("~")
if len(args) == 1:
return None
else:
return args[0]
def get_pkg_media_info(self,pkg_name:str)->PackageMediaInfo:
pass
def try_load_pkg_media_info(self,pkg_full_path:str) -> PackageMediaInfo:
the_result : PackageMediaInfo = None
logger.debug(f"try load pkng from:{pkg_full_path}")
if os.path.isdir(pkg_full_path):
the_result = PackageMediaInfo(pkg_full_path,"dir")
return the_result
def _create_media_loader(self,media_info:PackageMediaInfo) -> MediaReader:
match media_info.media_type:
case "dir":
from .media_reader import FolderMediaReader
return FolderMediaReader(media_info.full_path)
logger.error(f"create media loader for {media_info} failed!")
return None
def get_installed_pkg_info(self,pkg_name:str) -> PackageInfo:
pass
def lookup(self,pkg_id:str,version_str:str) -> PackageInfo:
# to make sure pkg.cid is correct, we MUST verfiy eveything here
pass
@classmethod
def is_valied_media(pkg_full_path:str) -> bool:
pass
def do_pkg_media_trans(self,pkg_info:PackageInfo,source_path:str,target_path:str) -> bool:
pass
def _load_pkg_cfg(self,cfg_path:str):
if cfg_path is None:
return
cfg = None
if len(cfg_path) < 1:
return
try:
cfg = toml.load(cfg_path)
self.env_dir = os.path.abspath(os.path.dirname(cfg_path))
self.cfg_path = os.path.abspath(cfg_path)
except Exception as e:
logger.error(f"read pkg cfg from {cfg_path} failed! unexpected error occurred: {str(e)}")
return
return self.load_from_config(cfg)
def _preprocess_prefixs(self,prefixs):
pass
class PackageEnvManager:
_instance = None
def __new__(cls):
@@ -33,105 +157,3 @@ class PackageEnvManager:
def get_system_env(self) -> PackageEnv:
pass
class PackageEnv:
def __init__(self,cfg_path:str) -> None:
self.pkg_dir : str = ""
self.pkg_obj_dir : str = ""
self.is_strict : bool = True
self.parent_envs : list[PackageEnv] = None
self.index_dbs = None
self.cfg_path = cfg_path
self._load_pkg_cfg(cfg_path)
pass
def load(self,pkg_name:str,search_parent=True) -> PackageMediaInfo:
pkg_path = None
pkg_id,verion_str,cid = PackageInfo.parse_pkg_name(pkg_name)
if cid is None:
if verion_str is None:
channel:str = self.get_pkg_channel_from_version(verion_str)
if channel is None:
pkg_path = f"{self.pkg_dir}{pkg_id}"
else:
pkg_path = f"{self.pkg_dir}{pkg_id}#{channel}"
else:
channel:str = self.get_pkg_channel_from_version(verion_str)
the_version:str = self.get_exact_version_from_installed(verion_str)
if the_version is None:
logger.warn(f"load {pkg_name} failed: no match version from {verion_str}")
return None
if channel is None:
pkg_path = f"{self.pkg_dir}{pkg_id}#{the_version}"
else:
pkg_path = f"{self.pkg_dir}{pkg_id}#{channel}#{the_version}"
else:
pkg_path = f"{self.pkg_obj_dir}.{pkg_id}/{cid}"
media_info:PackageMediaInfo = self.try_load_pkg_media_info(pkg_path)
if media_info is None:
if search_parent:
for parent_env in self.parent_envs:
media_info = parent_env.load(pkg_id,cid,False)
if media_info is not None:
return media_info
logger.warn(f"load {pkg_id}#{cid} error,not found ,search_parent={search_parent}")
return None
def get_exact_version_from_installed(self,verion_str:str) -> str:
pass
def get_pkg_channel_from_version(self,pkg_version:str) -> str:
pass
def get_pkg_media_info(self,pkg_name:str)->PackageMediaInfo:
pass
def try_load_pkg_media_info(self,pkg_full_path:str) -> PackageMediaInfo:
pass
def get_installed_pkg_info(self,pkg_name:str) -> PackageInfo:
pass
def lookup(self,pkg_id:str,version_str:str) -> PackageInfo:
# to make sure pkg.cid is correct, we MUST verfiy eveything here
pass
def get_installer(self) -> PackageInstaller:
pass
@classmethod
def is_valied_media(pkg_full_path:str) -> bool:
pass
def do_pkg_media_trans(self,pkg_info:PackageInfo,source_path:str,target_path:str) -> bool:
pass
def _load_pkg_cfg(self,cfg_path:str):
if cfg_path is None:
return
cfg = None
if len(cfg_path) < 1:
return
try:
cfg = toml.load(cfg_path)
except Exception as e:
logger.error(f"read pkg cfg from {cfg_path} failed! unexpected error occurred: {str(e)}")
return
if cfg:
if cfg.env:
if cfg.env.is_strict is not None:
self.is_strict = cfg.env.is_strict
if cfg.env.prefixs is not None:
self.prefixs = self._preprocess_prefixs(cfg.env.prefixs)
def _preprocess_prefixs(self,prefixs):
pass
src/component/package_manager/media_reader.py
+14 -5
View File
@@ -1,9 +1,18 @@
class MediaReader:
def __init__(self):
from abc import ABC, abstractmethod
import aiofiles
class MediaReader(ABC):
@abstractmethod
async def read(self, inner_path:str,mode:str):
pass
def load_from_media(self,media_info:str) -> int:
class FolderMediaReader(MediaReader):
def __init__(self, root_dir:str) -> None:
self.root_dir = root_dir
pass
def read(self, inner_path:str):
raise NotImplementedError
async def read(self, inner_path:str,mode:str):
full_path = self.root_dir + "/" + inner_path
result_file = await aiofiles.open(full_path, mode,encoding='utf-8')
return result_file
src/component/package_manager/pkg.py
+17 -6
View File
@@ -11,10 +11,20 @@ class PackageInfo:
self.target_media_type = "dir"
self.source_media_type = "7z"
@classmethod
def parse_pkg_name(cls,pkg_name:str) -> Tuple[str, str, str]:
"""parse pkg name like test-pkg#nightly#>0.2.31#sha1:323423423 to test-pkg,nightly#>0.2.31,sha1:323423423"""
pass
@staticmethod
def parse_pkg_name(pkg_name:str) -> Tuple[str, str, str]:
"""parse pkg name like test-pkg#nightly~>0.2.31#sha1:323423423 to test-pkg,nightly#>0.2.31,sha1:323423423"""
args = pkg_name.split("#")
if len(args) == 1:
return args[0],None,None
elif len(args) == 2:
return args[0],None,arg[2]
elif len(args) == 3:
return args[0],args[1],args[2]
else:
logger.error(f"parse pkg name {pkg_name} failed!")
return None,None,None
@@ -23,8 +33,9 @@ class PackageInfo:
return self.cid
class PackageMediaInfo:
def __init__(self) -> None:
pass
def __init__(self,full_path,media_type) -> None:
self.media_type = media_type
self.full_path = full_path