diff --git a/buildin/agents/english-teacher/agent.cfg b/buildin/agents/english-teacher/agent.cfg new file mode 100644 index 0000000..e69de29 diff --git a/doc/mvp/pkg_install_procedure.drawio b/doc/mvp/pkg_install_procedure.drawio new file mode 100644 index 0000000..b77bb77 --- /dev/null +++ b/doc/mvp/pkg_install_procedure.drawio @@ -0,0 +1,377 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/mvp/pkg_system_demo/demo1/pkg.cfg.toml b/doc/mvp/pkg_system_demo/demo1/pkg.cfg.toml new file mode 100644 index 0000000..e69de29 diff --git a/doc/mvp/pkg_system_demo/demo1/pkg.lock b/doc/mvp/pkg_system_demo/demo1/pkg.lock new file mode 100644 index 0000000..e69de29 diff --git a/doc/mvp/pkg_system_demo/demo1/pkg_index.toml b/doc/mvp/pkg_system_demo/demo1/pkg_index.toml new file mode 100644 index 0000000..f72d8a4 --- /dev/null +++ b/doc/mvp/pkg_system_demo/demo1/pkg_index.toml @@ -0,0 +1,3 @@ +[authors] + + diff --git a/doc/mvp/pkg_system_demo/demo1/pkgs/pkg_nameB.zip b/doc/mvp/pkg_system_demo/demo1/pkgs/pkg_nameB.zip new file mode 100644 index 0000000..15cb0ec Binary files /dev/null and b/doc/mvp/pkg_system_demo/demo1/pkgs/pkg_nameB.zip differ diff --git a/src/component/agent_manager/__init__.py b/src/component/agent_manager/__init__.py new file mode 100644 index 0000000..863c4d1 --- /dev/null +++ b/src/component/agent_manager/__init__.py @@ -0,0 +1,3 @@ +from .agent_manager import agent_manager,agent_manager_client +from .agent import ai_agent +from .templete import ai_agent_templete \ No newline at end of file diff --git a/src/component/agent_manager/agent.py b/src/component/agent_manager/agent.py new file mode 100644 index 0000000..26f8b71 --- /dev/null +++ b/src/component/agent_manager/agent.py @@ -0,0 +1,3 @@ +class ai_agent: + def __init__(self) -> None: + pass \ No newline at end of file diff --git a/src/component/agent_manager/agent_manager.py b/src/component/agent_manager/agent_manager.py new file mode 100644 index 0000000..d837175 --- /dev/null +++ b/src/component/agent_manager/agent_manager.py @@ -0,0 +1,40 @@ + +from .agent import ai_agent +from .templete import ai_agent_templete + +class agent_manager: + _instance = None + + def __init__(self) -> None: + pass + + def __new__(cls): + if cls._instance is None: + cls._instance = super(agent_manager, cls).__new__(cls) + return cls._instance + + + def initial(self,root_dir:str) -> None: + pass + + + def get(self,agent_id) -> ai_agent: + pass + + + def get_templete(self,templete_id) -> ai_agent_templete: + pass + + def install(self,templete_id) -> int: + + installer = None + #installer.install(templete_id) + pass + + def create(self,template,agent_name,agent_last_name,agent_introduce) -> ai_agent: + pass + + +class agent_manager_client: + def __init__(self) -> None: + pass \ No newline at end of file diff --git a/src/component/agent_manager/templete.py b/src/component/agent_manager/templete.py new file mode 100644 index 0000000..87540bf --- /dev/null +++ b/src/component/agent_manager/templete.py @@ -0,0 +1,3 @@ +class ai_agent_templete: + def __init__(self) -> None: + pass \ No newline at end of file diff --git a/src/component/ndn_client/__init__.py b/src/component/ndn_client/__init__.py new file mode 100644 index 0000000..6b4585c --- /dev/null +++ b/src/component/ndn_client/__init__.py @@ -0,0 +1 @@ +from .cid import content_id \ No newline at end of file diff --git a/src/component/ndn_client/cid.py b/src/component/ndn_client/cid.py new file mode 100644 index 0000000..981df3b --- /dev/null +++ b/src/component/ndn_client/cid.py @@ -0,0 +1,8 @@ + + +class content_id: + def __init__(self) -> None: + pass + + def create_from_str(cid_str:str): + pass \ No newline at end of file diff --git a/src/component/ndn_client/ndn_client.py b/src/component/ndn_client/ndn_client.py new file mode 100644 index 0000000..c507fb1 --- /dev/null +++ b/src/component/ndn_client/ndn_client.py @@ -0,0 +1,13 @@ + +from .cid import content_id + +class ndn_get_task: + def __init__(self) -> None: + pass + +class ndn_client: + def __init__(self): + pass + + def get_file(cid:content_id,target_path:str,urls:{}=None,options:{}=None): + pass \ No newline at end of file diff --git a/src/component/package_manager/__init__.py b/src/component/package_manager/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/component/package_manager/env.py b/src/component/package_manager/env.py new file mode 100644 index 0000000..85c6b2a --- /dev/null +++ b/src/component/package_manager/env.py @@ -0,0 +1,136 @@ + +import logging +import toml + +from .pkg import pkg_info,pkg_media_info +from .installer import pkg_installer + +logger = logging.getLogger(__name__) + +class pkg_env_mgr: + _instance = None + def __new__(cls): + if cls._instance is None: + cls._instance = super(pkg_env_mgr, cls).__new__(cls) + return cls._instance + + def __init__(self) -> None: + self._pkg_envs = {} + + pass + + def get_env(self,cfg_path:str) -> pkg_env: + if cfg_path in self._pkg_envs: + return self._pkg_envs[cfg_path] + else: + pkg_env = pkg_env(cfg_path) + self._pkg_envs[cfg_path] = pkg_env + return pkg_env + + def get_user_env(self) -> pkg_env: + pass + + def get_system_env(self) -> pkg_env: + pass + +class pkg_env: + def __init__(self,cfg_path:str) -> None: + self.pkg_dir : str = "" + self.pkg_obj_dir : str = "" + self.is_strict : bool = True + self.parent_envs : list[pkg_env] = None + self.index_dbs = None + + self.cfg_path = cfg_path + self._load_pkg_cfg(cfg_path) + pass + + def load(self,pkg_name:str,search_parent=True) -> pkg_media_info: + pkg_path = None + pkg_id,verion_str,cid = pkg_info.parse_pkg_name(pkg_name) + + if cid is None: + if verion_str is None: + channel = self.get_pkg_channel_from_version(verion_str) + if channel is None: + pkg_path = f"{self.pkg_dir}{pkg_id}" + else: + pkg_path = f"{self.pkg_dir}{pkg_id}#{channel}" + else: + channel = self.get_pkg_channel_from_version(verion_str) + the_version = self.get_exact_version_from_installed(verion_str) + if the_version is None: + logger.warn(f"load {pkg_name} failed: no match version from {verion_str}") + return None + if channel is None: + pkg_path = f"{self.pkg_dir}{pkg_id}#{the_version}" + else: + pkg_path = f"{self.pkg_dir}{pkg_id}#{channel}#{the_version}" + else: + pkg_path = f"{self.pkg_obj_dir}.{pkg_id}/{cid}" + + media_info = self.try_load_pkg_media_info(pkg_id,pkg_path) + if media_info is None: + if search_parent: + for parent_env in self.parent_envs: + media_info = parent_env.load(pkg_id,cid,False) + if media_info is not None: + return media_info + + logger.warn(f"load {pkg_id}#{cid} error,not found ,search_parent={search_parent}") + return None + + def get_exact_version_from_installed(self,verion_str:str) -> str: + pass + + def get_pkg_channel_from_version(self,pkg_version:str) -> str: + pass + + def get_pkg_media_info(self,pkg_name:str)->pkg_media_info: + pass + + def try_load_pkg_media_info(self,pkg_full_path:str) -> pkg_media_info: + pass + + + def get_installed_pkg_info(self,pkg_name:str) -> pkg_info: + pass + + def lookup(self,pkg_id:str,version_str:str) -> pkg_info: + # to make sure pkg.cid is correct, we MUST verfiy eveything here + pass + + def get_installer(self) -> pkg_installer: + pass + + @classmethod + def is_valied_media(pkg_full_path:str) -> bool: + pass + + def do_pkg_media_trans(self,pkg_info:pkg_info,source_path:str,target_path:str) -> bool: + pass + + def _load_pkg_cfg(self,cfg_path:str): + if cfg_path is None: + return + + cfg = None + if len(cfg_path) < 1: + return + try: + cfg = toml.load(cfg_path) + except Exception as e: + logger.error(f"read pkg cfg from {cfg_path} failed! unexpected error occurred: {str(e)}") + return + + if cfg: + if cfg.env: + if cfg.env.is_strict is not None: + self.is_strict = cfg.env.is_strict + + if cfg.env.prefixs is not None: + self.prefixs = self._preprocess_prefixs(cfg.env.prefixs) + + + def _preprocess_prefixs(self,prefixs): + pass \ No newline at end of file diff --git a/src/component/package_manager/index_db.py b/src/component/package_manager/index_db.py new file mode 100644 index 0000000..e69de29 diff --git a/src/component/package_manager/index_syncer.py b/src/component/package_manager/index_syncer.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/component/package_manager/index_syncer.py @@ -0,0 +1 @@ + diff --git a/src/component/package_manager/installer.py b/src/component/package_manager/installer.py new file mode 100644 index 0000000..894de8f --- /dev/null +++ b/src/component/package_manager/installer.py @@ -0,0 +1,170 @@ +# installer download pkg by cid, than install it to target dir +import logging +import asyncio +import aiohttp +import aiofiles +import os + +from ndn_client import content_id,ndn_client +from .pkg import pkg_info,pkg_media_info +from .env import pkg_env + + +logger = logging.getLogger(__name__) + +INSTALL_TASK_STATE_DONE = 0 +INSTALL_TASK_STATE_CHECK_DEPENDENCY = 1 +INSTALL_TASK_STATE_INSTALL_DEPENDENCY = 2 +INSTALL_TASK_STATE_DOWNLOADING = 3 +INSTALL_TASK_STATE_INSTALLING = 4 +INSTALL_TAKS_STATE_ERROR = 5 + +class install_task: + def __init__(self,owner:pkg_env) -> None: + self.owner = owner + self.state = INSTALL_TASK_STATE_CHECK_DEPENDENCY + + self.pkg_media_info = None + self.working_task = None + self.dependency_tasks = None + self.error_str = None + +class pkg_installer: + def __init__(self,owner_env:pkg_env) -> None: + self.all_tasks = {} + self.owner_env = owner_env + + def install(self,pkg_name:str, + install_from_dependency = False, can_upgrade = True,skip_depends = False,options = None)->Tuple[install_task,str]: + + the_pkg_info : pkg_info = None + is_upgrade : bool = False + need_backup : bool = False + + pkg_id,version_str,cid = pkg_info.parse_pkg_name(pkg_name) + media_info : pkg_media_info = self.owner_env.get_media_info(pkg_name) # must use index-db? + if media_info is not None: + if cid is not None: + if can_upgrade: + is_upgrade = True + else: + error_str = f"{pkg_name},{cid} already installed!" + logger.error(error_str) + return None,error_str + else: + the_pkg_info = self.owner_env.lookup(pkg_id,version_str,None) + if the_pkg_info is None: + error_str = f"{pkg_name} old version exist in local but not found in index db!" + logger.error(error_str) + return None,error_str + else: + is_upgrade = True + need_backup = True + + if the_pkg_info is None: + the_pkg_info = self.owner_env.lookup(pkg_id,version_str,cid) + + if the_pkg_info is None: + error_str = f"{pkg_name} ,cid:{cid} not found in index db" + logger.error(error_str) + return None,error_str + + result_task = self.all_tasks.get(the_pkg_info.cid) + if result_task is not None: + return result_task,"already installing" + + logger.info(f"start download&install {pkg_name},install_from_dependency={install_from_dependency},upgrade={is_upgrade},backup={need_backup},target_pkg_info={the_pkg_info}") + result_task = install_task() + self.all_tasks[the_pkg_info.cid] = result_task + async def download_and_install_pkg()->int: + # check dependency + if skip_depends is False: + result_task.dependency_tasks = {} + self.get_dependency_tasks(the_pkg_info,result_task.dependency_tasks) + result_task.state = INSTALL_TASK_STATE_INSTALL_DEPENDENCY + for depend_pkg_name in result_task.dependency_tasks: + # check pkg in local? + # install miss pkg + pass + + result_task.state = INSTALL_TASK_STATE_DOWNLOADING + install_full_path = "" + target_full_path = "" + old_package_full_path = "" + is_download_directy = False + + if the_pkg_info.target_media_type == the_pkg_info.source_media_type: + is_download_directy = True + if is_upgrade: + target_full_path = "" + else: + target_full_path = "" + else: + pass + + urls = self.owner_env.get_pkg_urls(the_pkg_info) + #download + client = ndn_client() # set watch + download_result = await client.get_file(the_pkg_info.cid,urls,target_full_path,options) + if download_result !=0: + result_task.state = INSTALL_TAKS_STATE_ERROR + return result_task.state + + result_task.state = INSTALL_TASK_STATE_INSTALLING + if is_download_directy is False: + install_media_result = False + install_media_result = await self.owner_env.do_pkg_media_trans(the_pkg_info,target_full_path,install_full_path) + if install_media_result is False: + result_task.state = INSTALL_TAKS_STATE_ERROR + result_task.error_str = "install media error,from {target_full_path} to {install_full_path}" + return result_task.state + + # last step,save install flag : install by manual or install by dependency + ## save cid dir + if is_upgrade: + os.rename(old_package_full_path, old_package_full_path + ".old" ) + os.rename(target_full_path,install_full_path) + ## update/create version link + + ## update pkg state + ## remove old version + + result_task.state = INSTALL_TASK_STATE_DONE + return result_task.state + + + result_task.working_task = asyncio.create_task(download_and_install_pkg()) + return result_task,None + + + def uninstall(self): + pass + + def get_dependency_tasks(self,pkg:pkg_info,dependency_tasks): + pass + + async def check_dependency(self,pkg:pkg_info,task_list:{}) -> bool: + for depend_pkg_name in pkg.depends: + depend_task = task_list.get(depend_pkg_name) + if depend_task is not None: + logger.debug(f"{pkg.name}'s depend pkg {depend_pkg_name} already in task list") + continue + depend_task = install_task() + task_list[depend_pkg_name] = depend_task + + depend_pkg_info = self.owner_env.lookup(depend_pkg_name) + if depend_pkg_info is None: + logger.warn(f"{pkg.name}'s depend pkg {depend_pkg_name} not found in index db") + return False + + if await self.check_dependency(depend_pkg_info,task_list) is False: + return False + + return True + + + + + + + diff --git a/src/component/package_manager/media_reader.py b/src/component/package_manager/media_reader.py new file mode 100644 index 0000000..e69de29 diff --git a/src/component/package_manager/pkg.py b/src/component/package_manager/pkg.py new file mode 100644 index 0000000..ebc9194 --- /dev/null +++ b/src/component/package_manager/pkg.py @@ -0,0 +1,27 @@ + +class pkg_info: + def __init__(self) -> None: + self.name = "" + self.cid = None + self.depends : list[str] = None + self.author = None + self.remote_urls = None + self.target_media_type = "dir" + self.source_media_type = "7z" + + + + def parse_pkg_name(pkg_name:str) -> Tuple[str, str, str]: + #return pkg_id,version_str,cid + pass + + @property + def cid(self) -> str: + return self.cid + +class pkg_media_info: + def __init__(self) -> None: + pass + + + diff --git a/test/agent_test.py b/test/agent_test.py new file mode 100644 index 0000000..e0fd3e3 --- /dev/null +++ b/test/agent_test.py @@ -0,0 +1,35 @@ +import sys +sys.path.append('../src/component/') + +from agent_manager import agent_manager + +def clean_agent(): + print("clean_agent") + +def clean_agent_templete(): + print("clean_agent_templte") + +def test_agent(): + am = agent_manager() + am.initial("root_dir") + agent = am.get("english_teacher") + if agent is None: + agent_templete = am.get_templete("english_teacher") + + if agent_templete is None : + op = am.install("english_teacher") + #wait install done + + agent = am.create(agent_templete,"Tracy","Wang","Tracy Wang is my english teacher") + + print("Agent Tracy Wang load success!"); + + #print(agent.get_introduce()); + + #chat_session = agent.get_default_chat_session("master"); + #chat_session.chat("给我讲一个英文笑话!"); + #chat_session.wait_response(); + #print(chat_session.last_msg()); + +if __name__ == "__main__": + test_agent() \ No newline at end of file diff --git a/test/pkg.cfg.toml b/test/pkg.cfg.toml new file mode 100644 index 0000000..46acf0a --- /dev/null +++ b/test/pkg.cfg.toml @@ -0,0 +1,15 @@ + +[authors] +"buckyos.org" = {public="aaaa",name="BuckyOS Core Dev Team"} +[authors."cyfs.com"] +public="bbbb" +name="CYFS Core Dev Team" + +[env] +is_strict = false +prefixs = ["default","../m/s","abc"] +obj2 = {k1="bb",k2=300} +[env.obj] +k1="aa" +k2=200 + diff --git a/test/spider_test.py b/test/spider_test.py new file mode 100644 index 0000000..296f382 --- /dev/null +++ b/test/spider_test.py @@ -0,0 +1,23 @@ + + +def test_main(): + sm = service_manager() + + knownlege_base_service = sm.get('knowlege_base') + knownlege_base_service.start() + + email_spider = sm.get('email_spider') + email_spider.start() + + doc_embeding_service = sm.get('doc_embeding_service') + doc_embeding_service.start() + + ia = agents_manager().get("ai_info_assistor") + chat_session = ia.get_default_chat_session("master"); + chat_session.chat("Who responded to my issue last week"); + chat_session.wait_response(); + #print(chat_session.last_msg()); + +if __name__ == '__main__': + test_main() + \ No newline at end of file