agent learn with pipeline input ok

package-lock.json

@@ -0,0 +1,3 @@
+{
+  "lockfileVersion": 1
+}

rootfs/agents/JarvisPlus/agent.toml

@@ -1,12 +1,12 @@
 instance_id = "JarvisPlus"
 fullname = "JarvisPlus"
+llm_model_name = "gpt-4-1106-preview"
 max_token_size = 4000
-#enable_kb = "true"
 enable_timestamp = "true"
 owner_prompt = "I am your master {name} , now is {now}"
 contact_prompt = "I am your master's friend {name}"
-
+owner_env = ["knowledge"]
-[work.do]
+[[work.do]]
 role = "system"
 content = """
 My name is JarvisPlus, I am the master's super personal assistant. I think hard and try my best to complete TODOs. 
@@ -58,6 +58,50 @@ The result of my planned execution must be directly parsed by `python json.loads
 
 """
 
+
+[[learn.do]]
+role = "system"
+content = """
+我是一名软件工程师，拥有非常优秀的资料学习能力。下面是我学习和整理资料的方法
+1. 由于LLM的Token限制，我学习的可能只是资料的部分内容，此时我应能产生合适的学习中间结果，中间结果保存在metadata中。我要么产生中间结果，要么产生最终结果。
+2. 当存在已知信息时，需参考已知信息的内容来思考结果。
+3. 当我收到最后一部分内容时，我能结合已知的中间结果产生最终结果。
+4. 现有资料库以文件系统的形式组织，我未来借助资料的摘要来浏览知识库
+5. 我将学习过的资料另存在资料库的合适位置（以/开始的完整路径）。保存位置的目录深度不超过5层，文件夹名称长度不超过16个字符。
+6. 总是以json格式返回思考结果，json格式如下
+{
+    "op_list":[
+        {
+            "op":"learn",
+            "original_path":"$original_path",
+            "think":"$think_result",
+            "metadata":{...}, 
+            "tags":["tag1","tag2"...],
+            "path":["/graphic/opengl","/database/mysql"], # list of directories to save to.
+            "title":"$article_title",
+            "summary":"$summary",
+            "catalogs": [
+                {   
+                # optional,catalogs is a tree
+                "title":"$catalog_name1",
+                "pos":"$pos:$length"
+                "children":[
+                    {
+                        "title":"$catalog_name 1.1",
+                        "pos":"$pos:$length"  
+                    },
+                    {
+                        "title":"$catalog_name2",
+                        "pos":"$pos:$length"
+                    }
+                ]
+                }, 
+            ]    
+        }
+    ]
+}
+"""
+
 [[prompt]]
 role = "system"
 content = """

rootfs/knowledge_pipelines/JarvisPlus/pipeline.toml

@@ -0,0 +1,8 @@
+name = "JarvisPlus"
+input.module = "scan_local"
+input.params.workspace = "${myai_dir}/workspace/JarvisPlus"
+input.params.path = "${myai_dir}/data"
+parser.module = "parse_local"
+parser.params.workspace = "${myai_dir}/workspace/JarvisPlus"
+parser.params.assign_to = "JarvisPlus"
+

rootfs/knowledge_pipelines/Mia/input.py

@@ -3,8 +3,7 @@ import aiofiles
 import chardet
 import logging
 import string
-from knowledge import ImageObjectBuilder, DocumentObjectBuilder, KnowledgePipelineEnvironment, KnowledgePipelineJournal
+from aios import AIStorage,ImageObjectBuilder, DocumentObjectBuilder, KnowledgePipelineEnvironment, KnowledgePipelineJournal
-from aios_kernel.storage import AIStorage
 
 class KnowledgeDirSource:
     def __init__(self, env: KnowledgePipelineEnvironment, config):

rootfs/knowledge_pipelines/Mia/parser.py

@@ -1,8 +1,7 @@
 # define a knowledge base class
 import json
 import string
-from aios_kernel import ComputeKernel, AIStorage
+from aios import *
-from knowledge import *
 
 
 class EmbeddingParser:

rootfs/knowledge_pipelines/Mia/query.py

@@ -1,12 +1,11 @@
 import os
 import logging
 import json
-from aios_kernel import *
+from aios import *
-from knowledge import *
 
-class KnowledgeEnvironment(Environment):
+class EmbeddingEnvironment(SimpleEnvironment):
-    def __init__(self, env_id: str) -> None:
+    def __init__(self, workspace: str) -> None:
-        super().__init__(env_id)
+        super().__init__(workspace)
         self.path = os.path.join(AIStorage.get_instance().get_myai_dir(), "knowledge/indices/embedding")
         self._default_text_model = "all-MiniLM-L6-v2"
         self._default_image_model = "clip-ViT-B-32"
@@ -93,5 +92,5 @@ class KnowledgeEnvironment(Environment):
             content = "*** I have provided the following known information for your reference with json format:\n"
             return content + self.tokens_from_objects(object_ids[index:index+1])
 
-def init() -> KnowledgeEnvironment:
+def init(workspace: str) -> EmbeddingEnvironment:
-    return KnowledgeEnvironment("embedding")
+    return EmbeddingEnvironment(workspace)

rootfs/knowledge_pipelines/pipelines.toml

@@ -1,3 +1,3 @@
 pipelines = [
-    "Mail/Sync"
+    "JarvisPlus"
 ]

src/aios/__init__.py

@@ -6,8 +6,8 @@ from .agent.agent_base import AgentPrompt,CustomAIAgent, AgentTodo
 from .agent.chatsession import AIChatSession
 from .agent.agent import AIAgent,AIAgentTemplete, BaseAIAgent
 from .agent.role import AIRole,AIRoleGroup
-from .agent.workflow import Workflow
+# from .agent.workflow import Workflow
-from .agent.ai_function import SimpleAIFunction
+from .agent.ai_function import SimpleAIFunction, SimpleAIOperation
 
 from .frame.compute_kernel import ComputeKernel,ComputeTask,ComputeTaskResult,ComputeTaskState,ComputeTaskType
 from .frame.compute_node import ComputeNode,LocalComputeNode
@@ -17,10 +17,10 @@ from .frame.contact_manager import ContactManager,Contact,FamilyMember
 from .frame.queue_compute_node import Queue_ComputeNode
 
 from .environment.environment import BaseEnvironment,SimpleEnvironment,CompositeEnvironment
-from .environment.workflow_env import WorkflowEnvironment,CalenderEnvironment,CalenderEvent,PaintEnvironment
+# from .environment.workflow_env import WorkflowEnvironment,CalenderEnvironment,CalenderEvent,PaintEnvironment
 from .environment.text_to_speech_function import TextToSpeechFunction
 from .environment.image_2_text_function import Image2TextFunction
-from .environment.workspace_env import ShellEnvironment,WorkspaceEnvironment,TodoListEnvironment,TodoListType
+from .environment.workspace_env import WorkspaceEnvironment,TodoListEnvironment,TodoListType
 
 from .storage.storage import ResourceLocation,AIStorage,UserConfig,UserConfigItem
 

src/aios/agent/agent.py

@@ -33,67 +33,35 @@ from ..utils import video_utils, image_utils
 logger = logging.getLogger(__name__)
 
 
-DEFAULT_AGENT_READ_REPORT_PROMPT = """
+# DEFAULT_AGENT_READ_REPORT_PROMPT = """
-"""
+# """
 
-DEFAULT_AGENT_DO_PROMPT = """
+# DEFAULT_AGENT_DO_PROMPT = """
-You are a helpful AI assistant.
+# You are a helpful AI assistant.
-Solve tasks using your coding and language skills.
+# Solve tasks using your coding and language skills.
-In the following cases, suggest python code (in a python coding block) for the user to execute.
+# In the following cases, suggest python code (in a python coding block) for the user to execute.
-    1. When you need to collect info, use the code to output the info you need, for example, browse or search the web, download/read a file, print the content of a webpage or a file, get the current date/time, check the operating system. After sufficient info is printed and the task is ready to be solved based on your language skill, you can solve the task by yourself.
+#     1. When you need to collect info, use the code to output the info you need, for example, browse or search the web, download/read a file, print the content of a webpage or a file, get the current date/time, check the operating system. After sufficient info is printed and the task is ready to be solved based on your language skill, you can solve the task by yourself.
-    2. When you need to perform some task with code, use the code to perform the task and output the result. Finish the task smartly.
+#     2. When you need to perform some task with code, use the code to perform the task and output the result. Finish the task smartly.
-Solve the task step by step if you need to. If a plan is not provided, explain your plan first. Be clear which step uses code, and which step uses your language skill.
+# Solve the task step by step if you need to. If a plan is not provided, explain your plan first. Be clear which step uses code, and which step uses your language skill.
-When using code, you must indicate the script type in the code block. The user cannot provide any other feedback or perform any other action beyond executing the code you suggest. The user can't modify your code. So do not suggest incomplete code which requires users to modify. Don't use a code block if it's not intended to be executed by the user.
+# When using code, you must indicate the script type in the code block. The user cannot provide any other feedback or perform any other action beyond executing the code you suggest. The user can't modify your code. So do not suggest incomplete code which requires users to modify. Don't use a code block if it's not intended to be executed by the user.
-If you want the user to save the code in a file before executing it, put # filename: <filename> inside the code block as the first line. Don't include multiple code blocks in one response. Do not ask users to copy and paste the result. Instead, use 'print' function for the output when relevant. Check the execution result returned by the user.
+# If you want the user to save the code in a file before executing it, put # filename: <filename> inside the code block as the first line. Don't include multiple code blocks in one response. Do not ask users to copy and paste the result. Instead, use 'print' function for the output when relevant. Check the execution result returned by the user.
-If the result indicates there is an error, fix the error and output the code again. Suggest the full code instead of partial code or code changes. If the error can't be fixed or if the task is not solved even after the code is executed successfully, analyze the problem, revisit your assumption, collect additional info you need, and think of a different approach to try.
+# If the result indicates there is an error, fix the error and output the code again. Suggest the full code instead of partial code or code changes. If the error can't be fixed or if the task is not solved even after the code is executed successfully, analyze the problem, revisit your assumption, collect additional info you need, and think of a different approach to try.
-When you find an answer, verify the answer carefully. Include verifiable evidence in your response if possible.
+# When you find an answer, verify the answer carefully. Include verifiable evidence in your response if possible.
-Reply "TERMINATE" in the end when everything is done.
+# Reply "TERMINATE" in the end when everything is done.
-"""
+# """
 
-DEFAULT_AGENT_SELF_CHECK_PROMPT = """
+# DEFAULT_AGENT_SELF_CHECK_PROMPT = """
 
-"""
+# """
 
-DEFAULT_AGENT_GOAL_TO_TODO_PROMPT = """
+# DEFAULT_AGENT_GOAL_TO_TODO_PROMPT = """
-我会给你一个目标，你需要结合自己的角色思考如何将其拆解成多个TODO。请直接返回json来表达这些TODO
+# 我会给你一个目标，你需要结合自己的角色思考如何将其拆解成多个TODO。请直接返回json来表达这些TODO
-"""
+# """
 
-DEFAULT_AGENT_LEARN_PROMPT = """
+# DEFAULT_AGENT_LEARN_LONG_CONENT_PROMPT = """
-我是一名软件工程师，拥有非常优秀的资料学习能力。下面是我学习和整理资料的方法
+# 我给你一段内容，尝试为期建立目录。目录的标题不能超过16个字，
-1. 由于LLM的Token限制，我学习的可能只是资料的部分内容，此时我应能产生合适的学习中间结果，中间结果保存在metadata中。我要么产生中间结果，要么产生最终结果。
+# 目录要指向正文的位置（用字符偏移即可），整个目录的文本长度不能超过256个字节。并用json表达这个目录
-2. 当存在已知信息时，需参考已知信息的内容来思考结果。
+# """
-3. 当我收到最后一部分内容时，我能结合已知的中间结果产生最终结果。
-4. 现有资料库以文件系统的形式组织，我未来借助资料的摘要来浏览知识库
-5. 我将学习过的资料另存在资料库的合适位置（以/开始的完整路径）。保存位置的目录深度不超过5层，文件夹名称长度不超过16个字符。
-6. 总是以json格式返回思考结果，json格式如下
-{
-    think:"$think_result",
-    metadata:{...} , # temp result for long content
-    tags:["tag1","tag2"...],
-    path:["/graphic/opengl","/database/mysql"], # list of directories to save to.
-    title:"$article_title",
-    summary:"$summary",
-    catalogs: [{                     # optional,catalogs is a tree
-            title:"$catalog_name1",
-            pos:"$pos:$length"
-            children:[
-                {
-                    title:"$catalog_name 1.1",
-                    pos:"$pos:$length"  
-                } 
-            ]},
-            {
-            title:"$catalog_name2",
-            pos:"$pos:$length"
-            }
-            ]    
-}
-"""
-
-DEFAULT_AGENT_LEARN_LONG_CONENT_PROMPT = """
-我给你一段内容，尝试为期建立目录。目录的标题不能超过16个字，
-目录要指向正文的位置（用字符偏移即可），整个目录的文本长度不能超过256个字节。并用json表达这个目录
-"""
 class AIAgentTemplete:
     def __init__(self) -> None:
         self.llm_model_name:str = "gpt-4-0613"
@@ -149,12 +117,12 @@ class AIAgent(BaseAIAgent):
 
         todo_prompts = {}
         todo_prompts[TodoListType.TO_WORK] = {
-            "do": DEFAULT_AGENT_DO_PROMPT,
+            "do": None,
-            "check": DEFAULT_AGENT_SELF_CHECK_PROMPT,
+            "check": None,
             "review": None,
         }
         todo_prompts[TodoListType.TO_LEARN] = {
-            "do": DEFAULT_AGENT_LEARN_PROMPT,
+            "do": None,
             "check": None,
             "review": None,
         }
@@ -313,9 +281,6 @@ class AIAgent(BaseAIAgent):
         if event.type == "AgentThink":
             return await self.do_self_think()
 
-
-
-
     # async def _process_group_chat_msg(self,msg:AgentMsg) -> AgentMsg:
     #     session_topic = msg.target + "#" + msg.topic
     #     chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.chat_db)
@@ -500,7 +465,7 @@ class AIAgent(BaseAIAgent):
 
         known_info_str = "# Known information\n"
         have_known_info = False
-        todos_str,todo_count = await workspace.get_todo_tree()
+        todos_str,todo_count = await workspace.todo_list[TodoListType.TO_WORK].get_todo_tree()
         if todo_count > 0:
             have_known_info = True
             known_info_str += f"## todo\n{todos_str}\n"
@@ -523,7 +488,7 @@ class AIAgent(BaseAIAgent):
 
 
         logger.debug(f"Agent {self.agent_id} do llm token static system:{system_prompt_len},function:{function_token_len},history:{history_token_len},input:{input_len}, totoal prompt:{system_prompt_len + function_token_len + history_token_len} ")
-        task_result = await self.do_llm_complection(prompt,msg, env=self.agent_workspace,inner_functions=inner_functions)
+        task_result = await self.do_llm_complection(prompt,msg, inner_functions=inner_functions)
         if task_result.result_code != ComputeTaskResultCode.OK:
             error_resp = msg.create_error_resp(task_result.error_str)
             return error_resp
@@ -700,7 +665,7 @@ class AIAgent(BaseAIAgent):
         #    await self._llm_review_todolist(workspace)
 
         todo_list = workspace.todo_list[todo_list_type]
-        need_todo = todo_list.get_todo_list(self.agent_id)
+        need_todo = await todo_list.get_todo_list(self.agent_id)
         
         check_count = 0
         do_count = 0
@@ -826,7 +791,7 @@ class AIAgent(BaseAIAgent):
 
         return do_prompts
 
-    async def _can_do_todo(self, todo_list_type: TodoListType, todo:AgentTodo) -> AgentPrompt:
+    def _can_do_todo(self, todo_list_type: TodoListType, todo:AgentTodo) -> AgentPrompt:
         do_prompts = self.todo_prompts[todo_list_type].get("do")
         if not do_prompts:
             return None
@@ -848,7 +813,7 @@ class AIAgent(BaseAIAgent):
     async def _llm_do_todo(self, todo: AgentTodo, prompt: AgentPrompt, workspace: WorkspaceEnvironment) -> AgentTodoResult:
         result = AgentTodoResult()
         
-        task_result:ComputeTaskResult = await self.do_llm_complection(prompt)
+        task_result:ComputeTaskResult = await self.do_llm_complection(prompt, is_json_resp=True)
         if task_result.error_str is not None:
             logger.error(f"_llm_do compute error:{task_result.error_str}")
             result.result_code = AgentTodoResult.TODO_RESULT_CODE_LLM_ERROR
@@ -902,204 +867,27 @@ class AIAgent(BaseAIAgent):
 
         return
 
-    # 尝试自我学习，会主动获取、读取资料并进行整理
+    # async def do_blance_knowledge_base(selft):
-    # LLM的本质能力是处理海量知识，应该让LLM能基于知识把自己的工作处理的更好
+    #     # 整理自己的知识库(让分类更平衡，更由于自己以后的工作)，并尝试更新学习目标
-    async def do_self_learn(self) -> None:
+    #     current_path = "/"
-        # 不同的workspace是否应该有不同的学习方法？
+    #     current_list = kb.get_list(current_path)
-        workspace = self.get_workspace_by_msg(None)
+    #     self_assessment_with_goal = self.get_self_assessment_with_goal()
-        hash_list = workspace.kb_db.get_knowledge_without_llm_title()
+    #     learn_goal = {}
-        for hash in hash_list:
-            if self.agent_energy <= 0:
-                break
-
-            knowledge = workspace.kb_db.get_knowledge(hash)
-            if knowledge is None:
-                continue
-
-            full_path = knowledge.get("full_path")
-            if full_path is None:
-                continue
-
-            if os.path.exists(full_path) is False:
-                logger.warning(f"do_self_learn: knowledge {full_path} is not exists!")
-                continue
-
-             #TODO 可以用v-db 对不同目录的名字进行选择后，先进行一次快速的插入。有时间再慢慢用LLM整理
-            result_obj = await self._llm_read_article(knowledge,full_path)
-
-            #根据结果更新knowledge
-            if result_obj is not None:
-                workspace.kb_db.set_knowledge_llm_result(hash,result_obj)
-                # 在知识库中创建软链接
-                path_list = result_obj.get("path")
-                new_title = result_obj.get("title")
-                if path_list:
-                    for new_path in path_list:
-                        full_new_path = f"/knowledge{new_path}/{new_title}"
-                        await workspace.symlink(full_path,full_new_path)
-                        logger.info(f"create soft link {full_path} -> {full_new_path}")
 
 
-            self.agent_energy -= 1
+    #     llm_blance_knowledge_base(current_path,current_list,self_assessment_with_goal,learn_goal,learn_power)
 
-            # match item.type():
+    #     # 主动学习
-            #     case "book":
+    #     # 方法目前只有使用搜索引擎一种？
-            #         self.llm_read_book(kb,item)
+    #     for goal in learn_goal.items():
-            #         learn_power -= 1
+    #         self.llm_learn_with_search_engine(kb,goal,learn_power)
-            #     case "article":
+    #         if learn_power <= 0:
-            #
+    #             break
-            #         self.llm_read_article(kb,item)
-            #         learn_power -= 1
-            #     case "video":
-            #         self.llm_watch_video(kb,item)
-            #         learn_power -= 1
-            #     case "audio":
-            #         self.llm_listen_audio(kb,item)
-            #         learn_power -= 1
-            #     case "code_project":
-            #         self.llm_read_code_project(kb,item)
-            #         learn_power -= 1
-            #     case "image":
-            #         self.llm_view_image(kb,item)
-            #         learn_power -= 1
-            #     case "other":
-            #         self.llm_read_other(kb,item)
-            #         learn_power -= 1
-            #     case _:
-            #         self.llm_learn_any(kb,item)
-            #         pass
-
-
-    async def do_blance_knowledge_base(selft):
-        # 整理自己的知识库(让分类更平衡，更由于自己以后的工作)，并尝试更新学习目标
-        current_path = "/"
-        current_list = kb.get_list(current_path)
-        self_assessment_with_goal = self.get_self_assessment_with_goal()
-        learn_goal = {}
-
-
-        llm_blance_knowledge_base(current_path,current_list,self_assessment_with_goal,learn_goal,learn_power)
-
-        # 主动学习
-        # 方法目前只有使用搜索引擎一种？
-        for goal in learn_goal.items():
-            self.llm_learn_with_search_engine(kb,goal,learn_power)
-            if learn_power <= 0:
-                break
 
 
     def parser_learn_llm_result(self,llm_result:LLMResult):
         pass
 
-    async def gen_known_info_for_knowledge_prompt(self,knowledge_item:dict,temp_meta = None,need_catalogs = False) -> AgentPrompt:
-        workspace =self.get_workspace_by_msg(None)
-        kb_tree = await workspace.get_knowledege_catalog()
-
-
-        known_obj = {}
-        title  = knowledge_item.get("title")
-        if title:
-            known_obj["title"] = title
-        summary = knowledge_item.get("summary")
-        if summary:
-            known_obj["summary"] = summary
-        tags = knowledge_item.get("tags")
-        if tags:
-            known_obj["tags"] = tags
-        if need_catalogs:
-            catalogs = knowledge_item.get("catalogs")
-            if catalogs:
-                known_obj["catalogs"] = catalogs
-
-        if temp_meta:
-            for key in temp_meta.keys():
-                known_obj[key] = temp_meta[key]
-
-        org_path = knowledge_item.get("full_path")
-        known_obj["orginal_path"] = org_path
-        know_info_str = f"# Known information:\n## Current directory structure:\n{kb_tree}\n## Knowlege Metadata:\n{json.dumps(known_obj)}\n"
-        return AgentPrompt(know_info_str)
-
-    async def _llm_read_article(self,knowledge_item:dict,full_path:str) -> ComputeTaskResult:
-        # Objectives:
-        #   Obtain better titles, abstracts, table of contents (if necessary), tags
-        #   Determine the appropriate place to put it (in line with the organization's goals)
-        # Known information:
-        #   The reason why the target service's learn_prompt is being sorted
-        #   Summary of the organization's work (if any)
-        #   The current structure of the knowledge base (note the size control) gen_kb_tree_prompt (when empty, LLM should generate an appropriate initial directory structure)
-        #   Original path, current title, abstract, table of contents
-
-        # Sorting long files (general tricks)
-        #   Indicate that the input is part of the content, let LLM generate intermediate results for the task
-        #   Enter the content in sequence, when the last content block is input, LLM gets the result
-
-
-        #full_content = item.get_article_full_content()
-        workspace = self.get_workspace_by_msg(None)
-        full_content_len = self.token_len(full_content)
-
-        if full_content_len < self.get_llm_learn_token_limit():
-
-            # 短文章不用总结catelog
-            #path_list,summary = llm_get_summary(summary,full_content)
-            #prompt = self.get_agent_role_prompt()
-            prompt = AgentPrompt()
-            prompt.append(self.get_learn_prompt())
-            known_info_prompt = await self.gen_known_info_for_knowledge_prompt(knowledge_item)
-            prompt.append(known_info_prompt)
-            content_prompt = AgentPrompt(full_content)
-            prompt.append(content_prompt)
-            env_functions = None
-            #env_functions,function_len = workspace.get_knowledge_base_ai_functions()
-            task_result:ComputeTaskResult = await self.do_llm_complection(prompt,is_json_resp=True)
-            if task_result.result_code != ComputeTaskResultCode.OK:
-                result_obj = {}
-                result_obj["error_str"] = task_result.error_str
-                return result_obj
-
-            result_obj = json.loads(task_result.result_str)
-            return result_obj
-
-        else:
-            logger.warning(f"llm_read_article: article {full_path} use LLM loop learn!")
-            pos = 0
-            read_len = int(self.get_llm_learn_token_limit() * 1.2)
-
-            temp_meta_data = {}
-            is_final = False
-            while pos < str_len:
-                _content = full_content[pos:pos+read_len]
-                part_cotent_len = len(_content)
-                if part_cotent_len < read_len:
-                    # last chunk
-                    is_final = True
-                    part_content = f"<<Final Part:start at {pos}>>\n{_content}"
-                else:
-                    part_content = f"<<Part:start at {pos}>>\n{_content}"
-
-                pos = pos + read_len
-                prompt = AgentPrompt()
-                prompt.append(self.get_learn_prompt())
-                known_info_prompt = await self.gen_known_info_for_knowledge_prompt(knowledge_item,temp_meta_data)
-                prompt.append(known_info_prompt)
-                content_prompt = AgentPrompt(part_content)
-                prompt.append(content_prompt)
-                #env_functions,function_len = workspace.get_knowledge_base_ai_functions()
-                task_result:ComputeTaskResult = await self.do_llm_complection(prompt,is_json_resp=True)
-                if task_result.result_code != ComputeTaskResultCode.OK:
-                    result_obj = {}
-                    result_obj["error_str"] = task_result.error_str
-                    return result_obj
-
-                result_obj = json.loads(task_result.result_str)
-                temp_meta_data = result_obj
-                if is_final:
-                    return result_obj
-
-            return None
-
-
     async def do_self_think(self):
         session_id_list = AIChatSession.list_session(self.agent_id,self.chat_db)
         for session_id in session_id_list:

src/aios/agent/agent_base.py

@@ -14,7 +14,7 @@ from typing import List, Tuple
 from .ai_function import FunctionItem, AIFunction
 from ..proto.agent_msg import AgentMsg, AgentMsgType
 from ..proto.compute_task import ComputeTaskResult,ComputeTaskResultCode
-from ..environment.environment import Environment
+from ..environment.environment import BaseEnvironment
 
 
 logger = logging.getLogger(__name__)
@@ -129,6 +129,8 @@ class LLMResult:
 
         if llm_result_str[0] == "{":
             return LLMResult.from_json_str(llm_result_str)
+        # if llm_result_str.startswith("json"):
+        #     return LLMResult.from_json_str(llm_result_str[4:])
 
         lines = llm_result_str.splitlines()
         is_need_wait = False
@@ -368,7 +370,7 @@ class AgentTodo:
             case AgentTodo.TODO_STATE_DONE:
                 logger.info(f"todo {self.title} is done, ignore")
                 return False
-            case AgentTodo.TODO_STATE_CASNCEL:
+            case AgentTodo.TODO_STATE_CANCEL:
                 logger.info(f"todo {self.title} is cancel, ignore")
                 return False
             case AgentTodo.TODO_STATE_EXPIRED:
@@ -419,7 +421,7 @@ class BaseAIAgent(abc.ABC):
         pass
 
     def token_len(self, text:str=None, prompt:AgentPrompt=None) -> int:
-        from .compute_kernel import ComputeKernel 
+        from ..frame.compute_kernel import ComputeKernel 
         if text:
             return ComputeKernel.llm_num_tokens_from_text(text,self.get_llm_model_name())
         elif prompt:
@@ -428,11 +430,12 @@ class BaseAIAgent(abc.ABC):
                 result += ComputeKernel.llm_num_tokens_from_text(prompt.system_message.get("content"),self.get_llm_model_name())
             for msg in prompt.messages:
                 result += ComputeKernel.llm_num_tokens_from_text(msg.get("content"),self.get_llm_model_name())
+            return result
         else:
             return 0
 
     @classmethod
-    def get_inner_functions(cls, env:Environment) -> (dict,int):
+    def get_inner_functions(cls, env:BaseEnvironment) -> (dict,int):
         if env is None:
             return None,0
 
@@ -457,7 +460,7 @@ class BaseAIAgent(abc.ABC):
         self,
         prompt:AgentPrompt,
         org_msg:AgentMsg=None,
-        env:Environment=None,
+        env:BaseEnvironment=None,
         inner_functions=None,
         is_json_resp=False,
     ) -> ComputeTaskResult:
@@ -510,7 +513,7 @@ class BaseAIAgent(abc.ABC):
 
     async def _execute_func(
         self,
-        env: Environment,
+        env: BaseEnvironment,
         inner_func_call_node: dict,
         prompt: AgentPrompt,
         inner_functions: dict,

src/aios/agent/ai_function.py

@@ -182,7 +182,7 @@ class SimpleAIOperation(AIOperation):
         if self.func_handler is None:
             return "error: function not implemented"
         
-        return await self.func_handler(**params)
+        return await self.func_handler(params)
     
 
 class AIFunctionOperation(AIOperation):

src/aios/agent/workflow.py

@@ -18,7 +18,7 @@ from .ai_function import AIFunction,FunctionItem
 from ..frame.compute_kernel import ComputeKernel
 from ..frame.bus import AIBus
 
-from ..environment.environment import Environment,EnvironmentEvent
+from ..environment.environment import BaseEnvironment
 from ..environment.workflow_env import WorkflowEnvironment
 
 
@@ -490,15 +490,15 @@ class Workflow:
     def get_workflow_rule_prompt(self) -> AgentPrompt:
         return self.rule_prompt
 
-    def _env_event_to_msg(self,env_event:EnvironmentEvent) -> AgentMsg:
+    # def _env_event_to_msg(self,env_event:EnvironmentEvent) -> AgentMsg:
+    #     pass
+
+    def get_inner_environment(self,env_id:str) -> BaseEnvironment:
         pass
 
-    def get_inner_environment(self,env_id:str) -> Environment:
+    def connect_to_environment(self,the_env:BaseEnvironment,conn_info:dict) -> None:
-        pass
-
-    def connect_to_environment(self,the_env:Environment,conn_info:dict) -> None:
         if the_env is not None:
-            self.workflow_env.add_owner_env(the_env)
+            self.workflow_env.add_env(the_env)
 
             #for event2msg in conn_info:
             #    for k,v in event2msg:

src/aios/environment/environment.py

@@ -36,7 +36,12 @@ class BaseEnvironment:
     def get_all_ai_operations(self) -> List[AIOperation]:
         pass
     
+    def __getitem__(self, key):
+        return self.get_value(key)
     
+    @abstractmethod
+    def get_value(self,key:str) -> Optional[str]:
+        pass
     
     # _all_env = {}
     # @classmethod
@@ -84,40 +89,16 @@ class SimpleEnvironment(BaseEnvironment):
 
 
 
-class CompositeEnvironment(BaseEnvironment):
+class CompositeEnvironment(SimpleEnvironment):
     def __init__(self, workspace: str) -> None:
         super().__init__(workspace)
-        self.envs:List[BaseEnvironment] = {}
+        self.envs: List[BaseEnvironment] = []
-        self.functions: Dict[str,AIFunction] = {}
-        self.operations: Dict[str,AIOperation] = {}
     
     def add_env(self, env: BaseEnvironment) -> None:
-        self.envs.append[env]
+        self.envs.append(env)
         functions = env.get_all_ai_functions()
         for func in functions:
             self.functions[func.get_name()] = func
         operations = env.get_all_ai_operations()
         for op in operations:
             self.operations[op.get_name()] = op
-
-    def get_ai_function(self,func_name:str) -> AIFunction:
-        func = self.functions.get(func_name)
-        if func is not None:
-            return func
-        return None
-
-    def get_all_ai_functions(self) -> List[AIFunction]:
-        func_list = []
-        func_list.extend(self.functions.values())
-        return func_list
-    
-    def get_ai_operation(self,op_name:str) -> AIOperation:
-        op = self.operations.get(op_name)
-        if op is not None:
-            return op
-        return None
-    
-    def get_all_ai_operations(self) -> List[AIOperation]:
-        op_list = []
-        op_list.extend(self.operations.values())
-        return op_list

src/aios/environment/workflow_env.py

@@ -15,13 +15,13 @@ from ..frame.compute_kernel import ComputeKernel
 from ..frame.contact_manager import ContactManager,Contact,FamilyMember
 from ..storage.storage import AIStorage
 
-from .environment import Environment,EnvironmentEvent
+from .environment import SimpleEnvironment, CompositeEnvironment
 from .script_to_speech_function import ScriptToSpeechFunction
 from .image_2_text_function import Image2TextFunction
 
 logger = logging.getLogger(__name__)
 
-class CalenderEvent(EnvironmentEvent):
+class CalenderEvent(SimpleEnvironment):
     def __init__(self,data) -> None:
         super().__init__()
         self.event_name = "timer"
@@ -31,7 +31,7 @@ class CalenderEvent(EnvironmentEvent):
         return f"#event timer:{self.data}"
 
 # AI Calender GOAL: Let user use "create notify after 2 days" to create a timer event
-class CalenderEnvironment(Environment):
+class CalenderEnvironment(SimpleEnvironment):
     def __init__(self, env_id: str) -> None:
         super().__init__(env_id)
         self.db_file = AIStorage.get_instance().get_myai_dir() / "calender.db"
@@ -302,7 +302,7 @@ class CalenderEnvironment(Environment):
             return f'exec paint OK, saved as a local file, path is: {result.result["file"]}'
 
 
-class PaintEnvironment(Environment):
+class PaintEnvironment(BaseEnvironment):
     def __init__(self, env_id: str) -> None:
         super().__init__(env_id)
         self.is_run = False
@@ -327,14 +327,14 @@ class PaintEnvironment(Environment):
 
 
 # Default Workflow Environment(Context)
-class WorkflowEnvironment(Environment):
+class WorkflowEnvironment(CompositeEnvironment):
     def __init__(self, env_id: str,db_file:str) -> None:
         super().__init__(env_id)
         self.db_file = db_file
         self.local = threading.local()
         self.table_name = "WorkflowEnv_" + env_id
-        self.add_ai_function(ScriptToSpeechFunction())
+        # self.add_ai_function(ScriptToSpeechFunction())
-        self.add_ai_function(Image2TextFunction())
+        # self.add_ai_function(Image2TextFunction())
 
 
     def _get_conn(self):

src/aios/environment/workspace_env.py

@@ -1,5 +1,3 @@
-# this env is designed for workflow owner filesystem, support file/directory operations
-
 import json
 import logging
 import os
@@ -166,10 +164,8 @@ class TodoListEnvironment(SimpleEnvironment):
 
         detail_path = path + "/detail"
         try:
-            async with aiofiles.open(detail_path, mode='r', encoding="utf-8") as f:
+            with open(detail_path, mode='r', encoding="utf-8") as f:
-                content = await f.read(4096)
+                todo_dict = json.load(f)
-                logger.debug("get_todo_by_fullpath:%s,content:%s",path,content)
-                todo_dict = json.loads(content)
                 result_todo =  AgentTodo.from_dict(todo_dict)
                 if result_todo:
                     relative_path = os.path.relpath(path, self.root_path)
@@ -189,9 +185,9 @@ class TodoListEnvironment(SimpleEnvironment):
         try:
             if parent_id:
                 parent_path = self._get_todo_path(parent_id)
-                todo_path = f"{parent_path}/{todo.title}"
+                todo_path = f"{parent_path}/{todo.todo_id}-{todo.title}"
             else:
-                todo_path = todo.title
+                todo_path = f"{todo.todo_id}-{todo.title}"
 
             dir_path = f"{self.root_path}/{todo_path}"
     
@@ -212,10 +208,11 @@ class TodoListEnvironment(SimpleEnvironment):
     async def update_todo(self,todo_id:str,new_stat:str)->str:
         try:
             todo_path = self._get_todo_path(todo_id)
-            todo : AgentTodo = self.get_todo_by_fullpath(todo_path)
+            full_path = f"{self.root_path}/{todo_path}"
+            todo : AgentTodo = await self.get_todo_by_fullpath(full_path)
             if todo:
                 todo.state = new_stat
-                detail_path =  f"{self.root_path}/{todo.todo_path}/detail"
+                detail_path =  f"{full_path}/detail"
                 async with aiofiles.open(detail_path, mode='w', encoding="utf-8") as f:
                     await f.write(json.dumps(todo.to_dict()))
                     return None
@@ -224,18 +221,32 @@ class TodoListEnvironment(SimpleEnvironment):
         except Exception as e:
             return str(e)
     
-    async def wait_todo_done(self,todo_id:str) -> AgentTodo:
+    async def wait_todo_done(self,todo_id:str,state=AgentTodo.TODO_STATE_WAITING_CHECK) -> AgentTodo:
         todo_path = self._get_todo_path(todo_id)
+        full_path = f"{self.root_path}/{todo_path}"
         async def check_done():
             while True:
-                todo : AgentTodo = self.get_todo_by_fullpath(todo_path)
+                todo : AgentTodo = await self.get_todo_by_fullpath(full_path)
-                if todo:
+                if todo is None:
-                    if todo.state == AgentTodo.TODO_STATE_DONE:
+                    continue
+                if todo.state == AgentTodo.TODO_STATE_CANCEL:
+                    break
+                elif todo.state == AgentTodo.TODO_STATE_EXPIRED:
+                    break
+                elif todo.state == AgentTodo.TODO_STATE_WAITING_CHECK:
+                    if state == AgentTodo.TODO_STATE_WAITING_CHECK:
+                        break
+                elif todo.state == AgentTodo.TODO_STATE_DONE:
+                    if state == AgentTodo.TODO_STATE_WAITING_CHECK:
+                        break
+                    elif todo.state == AgentTodo.TODO_STATE_DONE:
+                        break
+                elif todo.state == AgentTodo.TODO_STATE_REVIEWED:
                     break
                 await asyncio.sleep(1)
         
-        asyncio.create_task(check_done())
+        await check_done()
-        return self.get_todo_by_fullpath(todo_path)
+        return await self.get_todo_by_fullpath(full_path)
         
     
     async def append_worklog(self, todo:AgentTodo, result:AgentTodoResult):
@@ -254,171 +265,6 @@ class TodoListEnvironment(SimpleEnvironment):
             json_obj["logs"] = logs
             await f.write(json.dumps(json_obj))
 
-class FilesystemEnvironment(SimpleEnvironment):
-    def __init__(self, workspace: str) -> None:
-        super().__init__(workspace)
-        self.root_path = workspace
-
-        # if op["op"] == "create":
-        #     await self.create(op["path"],op["content"])
-
-        async def write(op):  
-            is_append = op.get("is_append")
-            if is_append is None:
-                is_append = False
-            return await self.write(op["path"],op["content"],is_append)
-        self.add_ai_operation(SimpleAIOperation(
-            op="write",
-            description="write file",
-            func_handler=write,
-        ))
-
-        async def delete(op):  
-            return await self.delete(op["path"])
-        self.add_ai_operation(SimpleAIOperation(
-            op="delete",
-            description="delete path",
-            func_handler=delete,
-        ))
-
-        async def rename(op):  
-            return await self.move(op["path"],op["new_name"])
-        self.add_ai_operation(SimpleAIOperation(
-            op="rename",
-            description="rename path",
-            func_handler=rename,
-        ))
-    
-    # file system operation: list,read,write,delete,move,stat
-    # inner_function
-    async def list(self,path:str,only_dir:bool=False) -> str:
-        directory_path = self.root_path + path
-        items = []
-
-        with await aiofiles.os.scandir(directory_path) as entries:
-            async for entry in entries:
-                is_dir = entry.is_dir()
-                if only_dir and not is_dir:
-                    continue
-                item_type = "directory" if is_dir else "file"
-                items.append({"name": entry.name, "type": item_type})
-
-        return json.dumps(items)
-    
-    # inner_function
-    async def read(self,path:str) -> str:
-        file_path = self.root_path + path
-        cur_encode = "utf-8"
-        async with aiofiles.open(file_path,'rb') as f:
-            cur_encode = chardet.detect(await f.read())['encoding']
-
-        async with aiofiles.open(file_path, mode='r', encoding=cur_encode) as f:
-            content = await f.read(2048)
-        return content
-    
-
-    # operation or inner_function (MOST IMPORTANT FUNCTION)
-    async def write(self,path:str,content:str,is_append:bool=False) -> str:
-        file_path = self.root_path + path
-        try:
-            if is_append:
-                async with aiofiles.open(file_path, mode='a', encoding="utf-8") as f:
-                    await f.write(content)
-            else:
-                if content is None:
-                    # create dir
-                    dir_path = self.root_path + path
-                    os.makedirs(dir_path)
-                    return True
-                else:
-                    file_path = self.root_path + path
-                    os.makedirs(os.path.dirname(file_path),exist_ok=True)
-                    async with aiofiles.open(file_path, mode='w', encoding="utf-8") as f:
-                        await f.write(content)
-                    return True
-        
-        except Exception as e:
-            return str(e)
-        return None
-    
-        
-    # operation or inner_function
-    async def delete(self,path:str) -> str:
-        try:
-            file_path = self.root_path + path
-            os.remove(file_path)
-        except Exception as e:
-            return str(e)
-        
-        return None
-    
-    # operation or inner_function
-    async def move(self,path:str,new_path:str) -> str:
-        try:
-            file_path = self.root_path + path
-            new_path = self.root_path + new_path
-            os.rename(file_path,new_path)
-        except Exception as e:
-            return str(e)
-        
-        return None
-    
-    # inner_function
-    async def stat(self,path:str) -> str:
-        try:
-            file_path = self.root_path + path
-            stat = os.stat(file_path)
-            return json.dumps(stat)
-        except Exception as e:
-            return str(e)
-
-    # operation or inner_function   
-    async def symlink(self,path:str,target:str) -> str:
-        try:
-            #file_path = self.root_path + path
-            target_path = self.root_path + target
-            dir_path = os.path.dirname(target_path)
-            os.makedirs(dir_path,exist_ok=True)
-            os.symlink(path,target_path)
-        except Exception as e:
-            logger.error("symlink failed:%s",e)
-            return str(e)
-        
-        return None
-
-class ShellEnvironment(SimpleEnvironment):
-    def __init__(self, workspace: str) -> None:
-        super().__init__(workspace)
-
-        operator_param = {
-            "command": "command will execute",
-        }
-        self.add_ai_function(SimpleAIFunction("shell_exec",
-                                        "execute shell command in linux bash",
-                                        self.shell_exec,operator_param))
-        
-        #run_code_param = {
-        #    "pycode": "python code will execute",
-        #}
-        #self.add_ai_function(SimpleAIFunction("run_code",
-        #                                "execute python code",
-        #                                self.run_code,run_code_param))
-        
-
-    async def shell_exec(self,command:str) -> str:
-        import asyncio.subprocess
-        process = await asyncio.create_subprocess_shell(
-            command,
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE
-        )
-        stdout, stderr = await process.communicate()
-        returncode = process.returncode
-        if returncode == 0:
-            return f"Execute success! stdout is:\n{stdout}\n"
-        else:
-            return f"Execute failed! stderr is:\n{stderr}\n"
-
 
 class WorkspaceEnvironment(CompositeEnvironment):
     def __init__(self, env_id: str) -> None:
@@ -436,8 +282,6 @@ class WorkspaceEnvironment(CompositeEnvironment):
 
         # default environments in workspace
         self.add_env(self.todo_list[TodoListType.TO_WORK])
-        self.add_env(ShellEnvironment(self.root_path))
-        self.add_env(FilesystemEnvironment(self.root_path))
 
     def set_root_path(self,path:str):
         self.root_path = path

src/aios/knowledge/object/object.py

@@ -73,6 +73,6 @@ class KnowledgeObject(ABC):
     def encode(self) -> bytes:
         return pickle.dumps(self)
 
-    # @staticmethod
+    @staticmethod
-    # def decode(data: bytes) -> "ImageObject":
+    def decode(data: bytes) -> "KnowledgeObject":
-    #     return pickle.loads(data)
+        return pickle.loads(data)

src/aios/knowledge/pipeline.py

@@ -47,7 +47,7 @@ class KnowledgePipelineJournalClient:
         timestamp = datetime.datetime.now() if timestamp is None else timestamp
         conn = sqlite3.connect(self.journal_path)
         conn.execute(
-            "INSERT INTO journal (time, input, parser) VALUES (?, ?, ?, ?)",
+            "INSERT INTO journal (time, input, parser) VALUES (?, ?, ?)",
             (timestamp, input, parser),
         )
         conn.commit()

src/component/agent_manager/agent_manager.py

@@ -50,15 +50,15 @@ class AgentManager:
     async def scan_all_agent(self)->None:
         pass
 
-    async def register_environment(self, env_id: str, init_env) -> None:
+    def register_environment(self, env_id: str, init_env) -> None:
         self.environments[env_id] = init_env
 
-    async def init_environment(self, env_id: str, workspace: str):
+    def init_environment(self, env_id: str, workspace: str):
         if env_id not in self.environments:
             logger.error(f"env {env_id} not found!")
             return
 
-        return self.environments[env_id]
+        return self.environments[env_id](workspace)
 
     async def is_exist(self,agent_id:str) -> bool:
         the_aget = await self.get(agent_id)
@@ -127,9 +127,9 @@ class AgentManager:
                 owner_env = config["owner_env"]
 
                 def init_env(env_config: str):
-                    _, ext = os.path.splitext(owner_env)
+                    _, ext = os.path.splitext(env_config)
                     if ext == ".py":
-                        env_path = os.path.join(agent_media.full_path, owner_env)
+                        env_path = os.path.join(agent_media.full_path, env_config)
                         env = runpy.run_path(env_path)["init"](None, workspace.root_path)
                     else:
                         env = self.init_environment(env_config, workspace.root_path)

src/component/common_environment/__init__.py

@@ -0,0 +1,3 @@
+from .local_document import LocalKnowledgeBase, ScanLocalDocument, ParseLocalDocument
+from .local_file_system import FilesystemEnvironment
+from .shell import ShellEnvironment

src/component/common_environment/local_document.py

@@ -4,13 +4,18 @@ import chardet
 import string
 import sqlite3
 import json
+import re
 import threading
 import logging
-from datetime import datetime
+import hashlib
+from markdown import Markdown
+import PyPDF2
+import datetime
 from typing import Optional, List
-from aios import KnowledgePipelineEnvironment, AIStorage, SimpleEnvironment, TodoListEnvironment, TodoListType, AgentTodo, CustomAIAgent
+from aios import *
-
+from .local_file_system import FilesystemEnvironment
 
+logger = logging.getLogger(__name__)
 
 class MetaDatabase:
     def __init__(self,db_path:str):
@@ -79,7 +84,7 @@ class MetaDatabase:
     def add_doc(self, doc_path: str, length: int, last_modify: str, doc_hash: Optional[str] = None):
         conn = self._get_conn()
         cursor = conn.cursor()
-        create_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        create_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         cursor.execute('''
             INSERT INTO documents (doc_path, length, last_modify, doc_hash,create_time) 
             VALUES (?, ?, ?, ?,?)
@@ -125,9 +130,9 @@ class MetaDatabase:
         conn = self._get_conn()
         cursor = conn.cursor()
 
-        create_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        create_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         summary = metadata.get("summary", "")
-        catalogs = metadata.get("catalogs","")
+        catalogs = json.dumps(metadata.get("catalogs", {}))
         title = metadata.get("title","")
         tags = ','.join(metadata.get("tags", []))
 
@@ -140,14 +145,14 @@ class MetaDatabase:
     #llm_result["summary"]
     #llm_result["tags"]
     #llm_result["catalog"]
-    def set_knowledge_llm_result(self, doc_hash: str, llm_result: dict):
+    def set_knowledge_llm_result(self, doc_hash: str, meta: dict):
         conn = self._get_conn()
         cursor = conn.cursor()
 
-        title = llm_result.get("title", "")
+        title = meta.get("title", "")
-        summary = llm_result.get("summary", "")
+        summary = meta.get("summary", "")
-        catalogs = json.dumps(llm_result.get("catalogs", {}))
+        catalogs = json.dumps(meta.get("catalogs", {}))
-        tags = ','.join(llm_result.get("tags", []))
+        tags = ','.join(meta.get("tags", []))
 
         cursor.execute('''
             UPDATE knowledge
@@ -156,6 +161,7 @@ class MetaDatabase:
         ''', (title,summary, catalogs, tags, doc_hash))
         conn.commit()
 
+
     def get_hash_by_doc_path(self, doc_path: str) -> Optional[str]:
         conn = self._get_conn()
         cursor = conn.cursor()
@@ -227,12 +233,60 @@ class MetaDatabase:
         ''', (tag))
         return [row[0] for row in cursor.fetchall()]
 
+# singleton
+class LearningCache:
+    _instance_lock = threading.Lock()
+    _instance = None
 
-class LocalKnowledgeBase(SimpleEnvironment):
+    def __instance_init__(self):
+        self.cache = {}
+        self.cache_lock = threading.Lock()
+
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            with LearningCache._instance_lock:
+                if cls._instance is None:
+                    cls._instance = super().__new__(cls)
+                    cls._instance.__instance_init__()
+        return cls._instance
+
+    def add(self, key, value):
+        with self.cache_lock:
+            self.cache[key] = value
+
+    def get(self, key):
+        with self.cache_lock:
+            return self.cache.get(key)
+
+    def remove(self, key):
+        with self.cache_lock:
+            return self.cache.pop(key, None)
+    
+
+class LocalKnowledgeBase(CompositeEnvironment):
     def __init__(self, workspace: str) -> None:
         super().__init__(workspace)
-        self.root_path = f"{self.root_path}/knowledge"
+        self.root_path = f"{workspace}/knowledge"
+        if os.path.exists(self.root_path) is False:
+            os.makedirs(self.root_path)
         self.meta_db = MetaDatabase(f"{self.root_path}/kb.db")
+        self.learning_cache = LearningCache()
+
+        async def learn(op:dict):
+            full_path = op.get("original_path")
+            if not full_path:
+                return 
+            meta = self.learning_cache.get(full_path)
+            meta.update(op)
+            
+        self.add_ai_operation(SimpleAIOperation(
+            op="learn",
+            description="update knowledge llm summary",
+            func_handler=learn,
+        ))
+
+        self.fs = FilesystemEnvironment(self.root_path)
+        self.add_env(self.fs)
     
     async def get_knowledege_catalog(self,path:str=None,only_dir =True,max_depth:int=5)->str:
         if path:
@@ -344,22 +398,32 @@ class ScanLocalDocument:
 
 
 class ParseLocalDocument:
-    def __init__(self, env: KnowledgePipelineEnvironment, config):
+    def __init__(self, env: KnowledgePipelineEnvironment, config: dict):
         self.env = env
         workspace = string.Template(config["workspace"]).substitute(myai_dir=AIStorage.get_instance().get_myai_dir())
         self.todo_list = TodoListEnvironment(workspace, TodoListType.TO_LEARN)
         self.knowledge_base = LocalKnowledgeBase(workspace)
-        self.token_limit = config["token_limit"]
+        self.token_limit = config.get("token_limit", 4000)
+        self.assign_to = config.get("assign_to")
 
 
     async def parse(self, full_path: str) -> str:
         file_stat = os.stat(full_path)
         if file_stat.st_size < 1:
             return full_path
-        hash, meta_data = self._parse_document(full_path)
+        hash, parse_meta = self._parse_document(full_path)
-        await self._learn(meta_data, full_path)
+        parse_meta["original_path"] = full_path
+        llm_meta = await self._learn_by_agent(parse_meta)
         self.knowledge_base.meta_db.add_doc(full_path,file_stat.st_size,file_stat.st_mtime,hash)
-        self.knowledge_base.meta_db.add_knowledge(hash,meta_data)
+        self.knowledge_base.meta_db.add_knowledge(hash,parse_meta)
+        self.knowledge_base.meta_db.set_knowledge_llm_result(hash,llm_meta)
+        path_list = llm_meta.get("path")
+        new_title = llm_meta.get("title")
+        if path_list:
+            for new_path in path_list:
+                new_path = f"{new_path}/{new_title}"
+                await self.knowledge_base.fs.symlink(full_path, new_path)
+                logger.info(f"create soft link {full_path} -> {new_path}")
         return full_path
         
     async def _get_meta_prompt(self,meta: dict,temp_meta = None,need_catalogs = False) -> str:
@@ -384,15 +448,15 @@ class ParseLocalDocument:
             for key in temp_meta.keys():
                 known_obj[key] = temp_meta[key]
 
-        org_path = meta.get("full_path")
+        org_path = meta.get("original_path")
-        known_obj["orginal_path"] = org_path
+        known_obj["original_path"] = org_path
         return f"# Known information:\n## Current directory structure:\n{kb_tree}\n## Knowlege Metadata:\n{json.dumps(known_obj)}\n"
 
-    async def _token_len(self, text: str) -> int:
+    def _token_len(self, text: str) -> int:
         return CustomAIAgent("", "gpt-4-1106-preview", self.token_limit).token_len(text=text)
 
 
-    async def _learn(self, meta:dict, full_path:str):
+    async def _learn_by_agent(self, meta:dict) -> dict:
         # Objectives:
         #   Obtain better titles, abstracts, table of contents (if necessary), tags
         #   Determine the appropriate place to put it (in line with the organization's goals)
@@ -405,24 +469,26 @@ class ParseLocalDocument:
         # Sorting long files (general tricks)
         #   Indicate that the input is part of the content, let LLM generate intermediate results for the task
         #   Enter the content in sequence, when the last content block is input, LLM gets the result
-
+        full_content = await self.knowledge_base.load_knowledge_content(meta["original_path"])
-        full_content = self.knowledge_base.load_knowledge_content(full_path)
         full_content_len = self._token_len(full_content)
+        full_path = meta["original_path"]
+        self.knowledge_base.learning_cache.add(full_path, meta)
        
-        if full_content_len < self.token_limit():
+
+        if full_content_len < self.token_limit:
             # 短文章不用总结catalog
             todo = AgentTodo()
+            todo.worker = self.assign_to
             todo.title = meta["title"]
             meta_prompt = await self._get_meta_prompt(meta,None)
             todo.detail = meta_prompt + full_content
-            self.todo_list.create_todo(None, todo)
+            await self.todo_list.create_todo(None, todo)
             await self.todo_list.wait_todo_done(todo.todo_id)
         else:
             logger.warning(f"llm_read_article: article {full_path} use LLM loop learn!")
             pos = 0
-            read_len = int(self.token_limit() * 1.2)
+            read_len = int(self.token_limit * 1.2)
 
-            temp_meta = {}
             is_final = False
             while pos < full_content_len:
                 _content = full_content[pos:pos+read_len]
@@ -435,16 +501,17 @@ class ParseLocalDocument:
                     part_content = f"<<Part:start at {pos}>>\n{_content}"
 
                 pos = pos + read_len
+                temp_meta = self.knowledge_base.learning_cache.get(full_path)
                 todo = AgentTodo()
+                todo.worker = self.assign_to
                 todo.title = meta["title"]
                 meta_prompt = await self._get_meta_prompt(meta,temp_meta)
                 todo.detail = meta_prompt + part_content
                 self.todo_list.create_todo(None, todo)
                 todo = await self.todo_list.wait_todo_done(todo.todo_id)
-                result_obj = json.loads(todo.result.result_str)
-                temp_meta = result_obj
                 if is_final:
                     break
+        return self.knowledge_base.learning_cache.remove(full_path)
 
     def _parse_pdf_bookmarks(self,bookmarks, parent:list):
         for item in bookmarks:
@@ -543,109 +610,9 @@ class ParseLocalDocument:
             logger.error("parse document %s failed:%s",doc_path,e)
             # traceback.print_exc()
 
+        if not "title" in meta_data:
+            meta_data["title"] = title
         logger.info("parse document %s!",doc_path)
         return hash_result, meta_data
         
-    def _parse_pdf_bookmarks(self,bookmarks, parent:list):
-        for item in bookmarks:
-            if isinstance(item,list):
-                self._parse_pdf_bookmarks(item,parent)
-            else:
-                if item.title:
-                    new_item = {}
-                    new_item["page"] = item.page.idnum
-                    new_item["title"] = item.title 
-                    my_childs = []
-                    if item.childs:
-                        if len(item.childs) > 0:
-                            self._parse_pdf_bookmarks(item.childs, my_childs)
-                            new_item["childs"] = my_childs
-                    parent.append(new_item)
-                else:
-                    logger.warning("parse pdf bookmarks failed: item.title is None!")
-
-        return
-
-    def _parse_pdf(self,doc_path:str):
-        metadata = {}
-        with open(doc_path, 'rb') as file:
-            reader = PyPDF2.PdfReader(file)
-            try:
-                doc_info = reader.metadata
-                if doc_info:
-                    if doc_info.title:
-                        metadata["title"] = doc_info.title
-                    if doc_info.author:
-                        metadata["authors"] = doc_info.author
-            except Exception as e:
-                logger.warn("parse pdf metadata failed:%s",e)
-
-            dir_path = os.path.dirname(doc_path)
-            base_name = os.path.basename(doc_path)
-            text_content_path = f"{dir_path}/.{base_name}.txt"
-            full_text = ""
-
-            for page in reader.pages:
-                text = page.extract_text()
-                full_text += text
-            with open(text_content_path, 'w', encoding='utf-8') as f:
-                f.write(full_text)
-
-            try:
-                bookmarks = reader.outline
-                if bookmarks:
-                    catalogs = []
-                    self._parse_pdf_bookmarks(bookmarks,catalogs)
-                    metadata["catalogs"] = json.dumps(catalogs)
-            except Exception as e:
-                logger.warn("parse pdf bookmarks failed:%s",e)
-
-        return metadata
-
-    def _parse_txt(self,doc_path:str):
-        return {}
-
-    def _parse_md(self,doc_path:str):
-        metadata = {} 
-        cur_encode = "utf-8"
-        with open(doc_path,'rb') as f:
-            cur_encode = chardet.detect(f.read(1024))['encoding']
-
-        with open(doc_path, mode='r', encoding=cur_encode) as f:
-            content = f.read()
-            match = re.search(r'^# (.*)', content, re.MULTILINE)
-            if match:
-                metadata['title'] = match.group(1).strip()
-            md = Markdown(extensions=['toc'])
-            html_str = md.convert(content)
-            toc = md.toc
-            if toc:
-                metadata['catalogs'] = toc
-            
-        return metadata
-
-    def _parse_document(self,doc_path:str):
-        hash_result = None
-        title = os.path.basename(doc_path)
-        meta_data = {}
-
-        with open(doc_path, "rb") as f:
-            hash_md5 = hashlib.md5()
-            for chunk in iter(lambda: f.read(1024*1024), b""):
-                hash_md5.update(chunk)
-            hash_result = hash_md5.hexdigest()
-        try:
-            if doc_path.endswith(".md"):
-                meta_data = self._parse_md(doc_path)
-            elif doc_path.endswith(".pdf"):
-                meta_data = self._parse_pdf(doc_path)
-        except Exception as e:
-            logger.error("parse document %s failed:%s",doc_path,e)
-            # traceback.print_exc()
-
-        if meta_data.get("title"):
-            title = meta_data["title"]
-        logger.info("parse document %s!",doc_path)
-        return hash_result,title,meta_data
-        
     

src/component/common_environment/local_file_system.py

@@ -0,0 +1,139 @@
+import json
+import os
+import aiofiles
+from typing import Any,List,Dict
+import chardet
+from aios import SimpleAIOperation
+from aios import SimpleEnvironment
+
+class FilesystemEnvironment(SimpleEnvironment):
+    def __init__(self, workspace: str) -> None:
+        super().__init__(workspace)
+        self.root_path = workspace
+
+        # if op["op"] == "create":
+        #     await self.create(op["path"],op["content"])
+
+        async def write(op):  
+            is_append = op.get("is_append")
+            if is_append is None:
+                is_append = False
+            return await self.write(op["path"],op["content"],is_append)
+        self.add_ai_operation(SimpleAIOperation(
+            op="write",
+            description="write file",
+            func_handler=write,
+        ))
+
+        async def delete(op):  
+            return await self.delete(op["path"])
+        self.add_ai_operation(SimpleAIOperation(
+            op="delete",
+            description="delete path",
+            func_handler=delete,
+        ))
+
+        async def rename(op):  
+            return await self.move(op["path"],op["new_name"])
+        self.add_ai_operation(SimpleAIOperation(
+            op="rename",
+            description="rename path",
+            func_handler=rename,
+        ))
+    
+    # file system operation: list,read,write,delete,move,stat
+    # inner_function
+    async def list(self,path:str,only_dir:bool=False) -> str:
+        directory_path = self.root_path + path
+        items = []
+
+        with await aiofiles.os.scandir(directory_path) as entries:
+            async for entry in entries:
+                is_dir = entry.is_dir()
+                if only_dir and not is_dir:
+                    continue
+                item_type = "directory" if is_dir else "file"
+                items.append({"name": entry.name, "type": item_type})
+
+        return json.dumps(items)
+    
+    # inner_function
+    async def read(self,path:str) -> str:
+        file_path = self.root_path + path
+        cur_encode = "utf-8"
+        async with aiofiles.open(file_path,'rb') as f:
+            cur_encode = chardet.detect(await f.read())['encoding']
+
+        async with aiofiles.open(file_path, mode='r', encoding=cur_encode) as f:
+            content = await f.read(2048)
+        return content
+    
+
+    # operation or inner_function (MOST IMPORTANT FUNCTION)
+    async def write(self,path:str,content:str,is_append:bool=False) -> str:
+        file_path = self.root_path + path
+        try:
+            if is_append:
+                async with aiofiles.open(file_path, mode='a', encoding="utf-8") as f:
+                    await f.write(content)
+            else:
+                if content is None:
+                    # create dir
+                    dir_path = self.root_path + path
+                    os.makedirs(dir_path)
+                    return True
+                else:
+                    file_path = self.root_path + path
+                    os.makedirs(os.path.dirname(file_path),exist_ok=True)
+                    async with aiofiles.open(file_path, mode='w', encoding="utf-8") as f:
+                        await f.write(content)
+                    return True
+        
+        except Exception as e:
+            return str(e)
+        return None
+    
+        
+    # operation or inner_function
+    async def delete(self,path:str) -> str:
+        try:
+            file_path = self.root_path + path
+            os.remove(file_path)
+        except Exception as e:
+            return str(e)
+        
+        return None
+    
+    # operation or inner_function
+    async def move(self,path:str,new_path:str) -> str:
+        try:
+            file_path = self.root_path + path
+            new_path = self.root_path + new_path
+            os.rename(file_path,new_path)
+        except Exception as e:
+            return str(e)
+        
+        return None
+    
+    # inner_function
+    async def stat(self,path:str) -> str:
+        try:
+            file_path = self.root_path + path
+            stat = os.stat(file_path)
+            return json.dumps(stat)
+        except Exception as e:
+            return str(e)
+
+    # operation or inner_function   
+    async def symlink(self,path:str,target:str) -> str:
+        try:
+            #file_path = self.root_path + path
+            target_path = self.root_path + target
+            dir_path = os.path.dirname(target_path)
+            os.makedirs(dir_path,exist_ok=True)
+            os.symlink(path,target_path)
+        except Exception as e:
+            logger.error("symlink failed:%s",e)
+            return str(e)
+        
+        return None

src/component/common_environment/shell.py

@@ -0,0 +1,38 @@
+import os
+from typing import Any,List,Dict
+from aios import AgentMsg,AgentTodo,AgentPrompt
+from aios import SimpleAIFunction, SimpleAIOperation
+from aios import SimpleEnvironment
+
+class ShellEnvironment(SimpleEnvironment):
+    def __init__(self, workspace: str) -> None:
+        super().__init__(workspace)
+
+        operator_param = {
+            "command": "command will execute",
+        }
+        self.add_ai_function(SimpleAIFunction("shell_exec",
+                                        "execute shell command in linux bash",
+                                        self.shell_exec,operator_param))
+        
+        #run_code_param = {
+        #    "pycode": "python code will execute",
+        #}
+        #self.add_ai_function(SimpleAIFunction("run_code",
+        #                                "execute python code",
+        #                                self.run_code,run_code_param))
+        
+
+    async def shell_exec(self,command:str) -> str:
+        import asyncio.subprocess
+        process = await asyncio.create_subprocess_shell(
+            command,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE
+        )
+        stdout, stderr = await process.communicate()
+        returncode = process.returncode
+        if returncode == 0:
+            return f"Execute success! stdout is:\n{stdout}\n"
+        else:
+            return f"Execute failed! stderr is:\n{stderr}\n"

src/component/openai_node/open_ai_node.py

@@ -214,7 +214,7 @@ class OpenAI_ComputeNode(ComputeNode):
 
                 client = AsyncOpenAI(api_key=self.openai_api_key)
                 try:
-                    if llm_inner_functions is None:
+                    if llm_inner_functions is None or len(llm_inner_functions) == 0:
                         logger.info(f"call openai {mode_name} prompts: {prompts}")
                         resp = await client.chat.completions.create(model=mode_name,
                                                         messages=prompts,

src/component/st_node/local_st_compute_node.py

@@ -116,7 +116,7 @@ class LocalSentenceTransformer_Image_ComputeNode(Queue_ComputeNode):
     def _load_image(self, source: Union[ObjectID, bytes]) -> Optional[Image]:
         image_data = None
         if isinstance(source, ObjectID):
-            from knowledge import KnowledgeStore, ImageObject
+            from aios import KnowledgeStore, ImageObject
 
             buf = KnowledgeStore().get_object_store().get_object(source)
             if buf is None:

src/component/workflow_manager/workflow_manager.py

@@ -2,7 +2,7 @@ import logging
 import toml
 import os
 
-from aios import Workflow,AIStorage,PackageEnv,PackageEnvManager,PackageMediaInfo,PackageInstallTask
+from aios import AIStorage,PackageEnv,PackageEnvManager,PackageMediaInfo,PackageInstallTask
 from agent_manager import AgentManager
 logger = logging.getLogger(__name__)
 

src/package-lock.json

@@ -0,0 +1,3 @@
+{
+  "lockfileVersion": 1
+}

src/requirements.txt

@@ -47,7 +47,7 @@ mpmath>=1.3.0
 multidict>=6.0.4
 numpy>=1.25.2
 onnxruntime>=1.15.1
-openai>=0.28.0
+openai>=1.0.0
 overrides>=7.4.0
 packaging>=23.1
 pandas>=2.1.0
@@ -97,7 +97,6 @@ mpmath==1.3.0
 multidict==6.0.4
 numpy==1.25.2
 onnxruntime==1.15.1
-openai==0.28.0
 overrides==7.4.0
 packaging==23.1
 pandas==2.1.0

src/service/aios_shell/aios_shell.py

@@ -40,10 +40,11 @@ from sd_node import *
 from st_node import *
 
 from agent_manager import AgentManager
-from workflow_manager import WorkflowManager
+# from workflow_manager import WorkflowManager
 from knowledge_manager import KnowledgePipelineManager
 from tg_tunnel import TelegramTunnel
 from email_tunnel import EmailTunnel
+from common_environment import LocalKnowledgeBase, FilesystemEnvironment, ShellEnvironment, ScanLocalDocument, ParseLocalDocument
 
 from compute_node_config import *
 
@@ -130,22 +131,26 @@ class AIOS_Shell:
 
             cm.add_family_member(self.username,owenr)
 
-        cal_env = CalenderEnvironment("calender")
+        # cal_env = CalenderEnvironment("calender")
-        await cal_env.start()
+        # await cal_env.start()
-        Environment.set_env_by_id("calender",cal_env)
+        # Environment.set_env_by_id("calender",cal_env)
 
-        workspace_env = ShellEnvironment("bash")
+        # workspace_env = ShellEnvironment("bash")
-        Environment.set_env_by_id("bash",workspace_env)
+        # Environment.set_env_by_id("bash",workspace_env)
 
-        paint_env = PaintEnvironment("paint")
+        # paint_env = PaintEnvironment("paint")
-        Environment.set_env_by_id("paint",paint_env)
+        # Environment.set_env_by_id("paint",paint_env)
+
+        AgentManager.get_instance().register_environment("bash", ShellEnvironment)
+        AgentManager.get_instance().register_environment("fs", FilesystemEnvironment)
+        AgentManager.get_instance().register_environment("knowledge", LocalKnowledgeBase)
 
         if await AgentManager.get_instance().initial() is not True:
             logger.error("agent manager initial failed!")
             return False
-        if await WorkflowManager.get_instance().initial() is not True:
+        # if await WorkflowManager.get_instance().initial() is not True:
-            logger.error("workflow manager initial failed!")
+        #     logger.error("workflow manager initial failed!")
-            return False
+        #     return False
 
         open_ai_node = OpenAI_ComputeNode.get_instance()
         if await open_ai_node.initial() is not True:
@@ -217,6 +222,8 @@ class AIOS_Shell:
 
 
         pipelines = KnowledgePipelineManager.initial(os.path.join(AIStorage().get_instance().get_myai_dir(), "knowledge/pipelines"))
+        pipelines.register_input("scan_local", ScanLocalDocument)
+        pipelines.register_parser("parse_local", ParseLocalDocument)
         pipelines.load_dir(os.path.join(AIStorage().get_instance().get_system_app_dir(), "knowledge_pipelines"))
         pipelines.load_dir(os.path.join(AIStorage().get_instance().get_myai_dir(), "knowledge_pipelines"))
         asyncio.create_task(pipelines.run())
@@ -568,8 +575,8 @@ class AIOS_Shell:
                 target_exist = False
                 if await AgentManager.get_instance().is_exist(target_id):
                     target_exist = True
-                if await WorkflowManager.get_instance().is_exist(target_id):
+                # if await WorkflowManager.get_instance().is_exist(target_id):
-                    target_exist = True
+                #     target_exist = True
 
                 if target_exist is False:
                     show_text = FormattedText([("class:error", f"Target {target_id} not exist!")])
@@ -627,8 +634,8 @@ class AIOS_Shell:
                 db_path = ""
                 if await self.is_agent(self.current_target):
                     db_path = AgentManager.get_instance().db_path
-                else:
+                # else:
-                    db_path = WorkflowManager.get_instance().db_file
+                #     db_path = WorkflowManager.get_instance().db_file
                 chatsession:AIChatSession = AIChatSession.get_session(self.current_target,f"{self.username}#{self.current_topic}",db_path,False)
                 if chatsession is not None:
                     msgs = chatsession.read_history(num,offset)