Refactor the code directory structure to better suit the current complexity

src/aios/__init__.py

@@ -0,0 +1,32 @@
+
+from .proto.agent_msg import *
+from .proto.compute_task import *
+
+from .agent.agent_base import AgentPrompt,CustomAIAgent
+from .agent.chatsession import AIChatSession
+from .agent.agent import AIAgent,AIAgentTemplete, BaseAIAgent
+from .agent.role import AIRole,AIRoleGroup
+from .agent.workflow import Workflow
+from .agent.ai_function import SimpleAIFunction
+
+from .frame.compute_kernel import ComputeKernel,ComputeTask,ComputeTaskResult,ComputeTaskState,ComputeTaskType
+from .frame.compute_node import ComputeNode,LocalComputeNode
+from .frame.bus import AIBus
+from .frame.tunnel import AgentTunnel
+from .frame.contact_manager import ContactManager,Contact,FamilyMember
+from .frame.queue_compute_node import Queue_ComputeNode
+
+from .environment.environment import Environment,EnvironmentEvent
+from .environment.workflow_env import WorkflowEnvironment,CalenderEnvironment,CalenderEvent,PaintEnvironment
+from .environment.text_to_speech_function import TextToSpeechFunction
+from .environment.image_2_text_function import Image2TextFunction
+from .environment.workspace_env import ShellEnvironment,WorkspaceEnvironment
+
+from .storage.storage import ResourceLocation,AIStorage,UserConfig,UserConfigItem
+
+from .net import *
+from .knowledge import *
+from .package_manager import *
+
+
+AIOS_Version = "0.5.2, build 2023-11-30"

src/aios/agent/agent_base.py

@@ -0,0 +1,539 @@
+import abc
+import copy
+from abc import abstractmethod
+from datetime import datetime, timedelta
+import logging
+from enum import Enum
+import uuid
+import time
+import re
+import shlex
+import json
+from typing import List
+
+from .ai_function import FunctionItem, AIFunction
+from ..proto.agent_msg import AgentMsg, AgentMsgType
+from ..proto.compute_task import ComputeTaskResult,ComputeTaskResultCode
+from ..environment.environment import Environment
+
+
+logger = logging.getLogger(__name__)
+
+
+class AgentPrompt:
+    def __init__(self,prompt_str = None) -> None:
+        self.messages = []
+        if prompt_str:
+            self.messages.append({"role":"user","content":prompt_str})
+        self.system_message = None
+
+    def as_str(self)->str:
+        result_str = ""
+        if self.system_message:
+            result_str += self.system_message.get("role") + ":" + self.system_message.get("content") + "\n"
+        if self.messages:
+            for msg in self.messages:
+                result_str += msg.get("role") + ":" + msg.get("content") + "\n"
+
+        return result_str
+
+    def to_message_list(self):
+        result = []
+        if self.system_message:
+            result.append(self.system_message)
+        result.extend(self.messages)
+        return result
+
+    def append(self,prompt):
+        if prompt is None:
+            return
+
+        if prompt.system_message is not None:
+            if self.system_message is None:
+                self.system_message = copy.deepcopy(prompt.system_message)
+            else:
+                self.system_message["content"] += prompt.system_message.get("content")
+
+        self.messages.extend(prompt.messages)
+
+    def get_prompt_token_len(self):
+        result = 0
+
+        if self.system_message:
+            result += len(self.system_message.get("content"))
+        for msg in self.messages:
+            result += len(msg.get("content"))
+
+        return result
+
+    def load_from_config(self,config:list) -> bool:
+        if isinstance(config,list) is not True:
+            logger.error("prompt is not list!")
+            return False
+        self.messages = []
+        for msg in config:
+            if msg.get("content"):
+                if msg.get("role") == "system":
+                    self.system_message = msg
+                else:
+                    self.messages.append(msg)
+            else:
+                logger.error("prompt message has no content!")
+        return True
+
+class LLMResult:
+    def __init__(self) -> None:
+        self.state : str = "ignore"
+        self.resp : str = ""
+        self.raw_resp = None
+        self.paragraphs : dict[str,FunctionItem] = []
+
+
+        self.post_msgs : List[AgentMsg] = []
+        self.send_msgs : List[AgentMsg] = []
+        self.calls : List[FunctionItem] = []
+        self.post_calls : List[FunctionItem] = []
+        self.op_list : List[FunctionItem] = [] # op_list is a optimize design for saving token
+    @classmethod
+    def from_json_str(self,llm_json_str:str) -> 'LLMResult':
+        r = LLMResult()
+        if llm_json_str is None:
+            r.state = "ignore"
+            return r
+        if llm_json_str == "ignore":
+            r.state = "ignore"
+            return r
+
+        llm_json = json.loads(llm_json_str)
+        r.state = llm_json.get("state")
+        r.resp = llm_json.get("resp")
+        r.raw_resp = llm_json
+
+        post_msgs = llm_json.get("post_msg")
+        r.post_msgs = []
+        if post_msgs:
+            for msg in post_msgs:
+                new_msg = AgentMsg()
+                target_id = msg.get("target")
+                msg_content = msg.get("content")
+                new_msg.set("",target_id,msg_content)
+                r.post_msgs.append(new_msg)
+                #new_msg.msg_type = AgentMsgType.TYPE_MSG
+
+        r.calls = llm_json.get("calls")
+        r.post_calls = llm_json.get("post_calls")
+        r.op_list = llm_json.get("op_list")
+
+        return r
+
+    @classmethod
+    def from_str(self,llm_result_str:str,valid_func:List[str]=None) -> 'LLMResult':
+        r = LLMResult()
+
+        if llm_result_str is None:
+            r.state = "ignore"
+            return r
+        if llm_result_str == "ignore":
+            r.state = "ignore"
+            return r
+
+        if llm_result_str[0] == "{":
+            return LLMResult.from_json_str(llm_result_str)
+
+        lines = llm_result_str.splitlines()
+        is_need_wait = False
+
+        def check_args(func_item:FunctionItem):
+            match func_name:
+                case "send_msg":# /send_msg $target_id
+                    if len(func_args) != 1:
+                        return False
+
+                    new_msg = AgentMsg()
+                    target_id = func_item.args[0]
+                    msg_content = func_item.body
+                    new_msg.set("",target_id,msg_content)
+
+                    r.send_msgs.append(new_msg)
+                    is_need_wait = True
+                    return True
+
+                case "post_msg":# /post_msg $target_id
+                    if len(func_args) != 1:
+                        return False
+
+                    new_msg = AgentMsg()
+                    target_id = func_item.args[0]
+                    msg_content = func_item.body
+                    new_msg.set("",target_id,msg_content)
+                    r.post_msgs.append(new_msg)
+                    return True
+
+                case "call":# /call $func_name $args_str
+                    r.calls.append(func_item)
+                    is_need_wait = True
+                    return True
+                case "post_call": # /post_call $func_name,$args_str
+                    r.post_calls.append(func_item)
+                    return True
+                case _:
+                    if valid_func is not None:
+                        if func_name in valid_func:
+                            r.paragraphs[func_name] = func_item
+                            return True
+
+            return False
+
+
+        current_func : FunctionItem = None
+        for line in lines:
+            if line.startswith("##/"):
+                if current_func:
+                    if check_args(current_func) is False:
+                        r.resp += current_func.dumps()
+
+                func_name,func_args = AgentMsg.parse_function_call(line[3:])
+                current_func = FunctionItem(func_name,func_args)
+            else:
+                if current_func:
+                    current_func.append_body(line + "\n")
+                else:
+                    r.resp += line + "\n"
+
+        if current_func:
+            if check_args(current_func) is False:
+                r.resp += current_func.dumps()
+
+        if len(r.send_msgs) > 0 or len(r.calls) > 0:
+            r.state = "waiting"
+        else:
+            r.state = "reponsed"
+
+        return r
+
+class AgentReport:
+    def __init__(self):
+        pass
+
+class AgentTodoResult:
+    TODO_RESULT_CODE_OK = 0,
+    TODO_RESULT_CODE_LLM_ERROR = 1,
+    TODO_RESULT_CODE_EXEC_OP_ERROR = 2
+
+
+    def __init__(self) -> None:
+        self.result_code = AgentTodoResult.TODO_RESULT_CODE_OK
+        self.result_str = None
+        self.error_str = None
+        self.op_list = None
+
+    def to_dict(self) -> dict:
+        result = {}
+        result["result_code"] = self.result_code
+        result["result_str"] = self.result_str
+        result["error_str"] = self.error_str
+        result["op_list"] = self.op_list
+        return result
+
+
+class AgentTodo:
+    TODO_STATE_WAIT_ASSIGN = "wait_assign"
+    TODO_STATE_INIT = "init"
+
+    TODO_STATE_PENDING = "pending"
+    TODO_STATE_WAITING_CHECK = "wait_check"
+    TODO_STATE_EXEC_FAILED = "exec_failed"
+    TDDO_STATE_CHECKFAILED = "check_failed"
+
+    TODO_STATE_CASNCEL = "cancel"
+    TODO_STATE_DONE = "done"
+    TODO_STATE_EXPIRED = "expired"
+
+    def __init__(self):
+        self.todo_id = "todo#" + uuid.uuid4().hex
+        self.title = None
+        self.detail = None
+        self.todo_path = None # get parent todo,sub todo by path
+        #self.parent = None
+        self.create_time = time.time()
+
+        self.state = "wait_assign"
+        self.worker = None
+        self.checker = None
+        self.createor = None
+
+        self.need_check = True
+        self.due_date = time.time() + 3600 * 24 * 2
+        self.last_do_time = None
+        self.last_check_time = None
+        self.last_review_time = None
+
+        self.depend_todo_ids = []
+        self.sub_todos = {}
+
+        self.result : AgentTodoResult = None
+        self.last_check_result = None
+        self.retry_count = 0
+        self.raw_obj = None
+
+    @classmethod
+    def from_dict(cls,json_obj:dict) -> 'AgentTodo':
+        todo = AgentTodo()
+        if json_obj.get("id") is not None:
+            todo.todo_id = json_obj.get("id")
+
+        todo.title = json_obj.get("title")
+        todo.state = json_obj.get("state")
+        create_time = json_obj.get("create_time")
+        if create_time:
+            todo.create_time = datetime.fromisoformat(create_time).timestamp()
+
+        todo.detail = json_obj.get("detail")
+        due_date = json_obj.get("due_date")
+        if due_date:
+            todo.due_date = datetime.fromisoformat(due_date).timestamp()
+
+        last_do_time = json_obj.get("last_do_time")
+        if last_do_time:
+            todo.last_do_time = datetime.fromisoformat(last_do_time).timestamp()
+        last_check_time = json_obj.get("last_check_time")
+        if last_check_time:
+            todo.last_check_time = datetime.fromisoformat(last_check_time).timestamp()
+        last_review_time = json_obj.get("last_review_time")
+        if last_review_time:
+            todo.last_review_time = datetime.fromisoformat(last_review_time).timestamp()
+
+        todo.depend_todo_ids = json_obj.get("depend_todo_ids")
+        todo.need_check = json_obj.get("need_check")
+        #todo.result = json_obj.get("result")
+        #todo.last_check_result = json_obj.get("last_check_result")
+        todo.worker = json_obj.get("worker")
+        todo.checker = json_obj.get("checker")
+        todo.createor = json_obj.get("createor")
+        if json_obj.get("retry_count"):
+            todo.retry_count = json_obj.get("retry_count")
+
+        todo.raw_obj = json_obj
+
+        return todo
+
+    def to_dict(self) -> dict:
+        if self.raw_obj:
+            result = self.raw_obj
+        else:
+            result = {}
+
+        result["id"] = self.todo_id
+        #result["parent_id"] = self.parent_id
+        result["title"] = self.title
+        result["state"] = self.state
+        result["create_time"] = datetime.fromtimestamp(self.create_time).isoformat()
+        result["detail"] = self.detail
+        result["due_date"] = datetime.fromtimestamp(self.due_date).isoformat()
+        result["last_do_time"] = datetime.fromtimestamp(self.last_do_time).isoformat() if self.last_do_time else None
+        result["last_check_time"] = datetime.fromtimestamp(self.last_check_time).isoformat() if self.last_check_time else None
+        result["last_review_time"] = datetime.fromtimestamp(self.last_review_time).isoformat() if self.last_review_time else None
+        result["depend_todo_ids"] = self.depend_todo_ids
+        result["need_check"] = self.need_check
+        result["worker"] = self.worker
+        result["checker"] = self.checker
+        result["createor"] = self.createor
+        result["retry_count"] = self.retry_count
+
+        return result
+
+    def can_check(self)->bool:
+        if self.state != AgentTodo.TODO_STATE_WAITING_CHECK:
+            return False
+
+        now = datetime.now().timestamp()
+        if self.last_check_time:
+            time_diff = now - self.last_check_time
+            if time_diff < 60*15:
+                logger.info(f"todo {self.title} is already checked, ignore")
+                return False
+
+        return True
+
+    def can_do(self) -> bool:
+        match self.state:
+            case AgentTodo.TODO_STATE_DONE:
+                logger.info(f"todo {self.title} is done, ignore")
+                return False
+            case AgentTodo.TODO_STATE_CASNCEL:
+                logger.info(f"todo {self.title} is cancel, ignore")
+                return False
+            case AgentTodo.TODO_STATE_EXPIRED:
+                logger.info(f"todo {self.title} is expired, ignore")
+                return False
+            case AgentTodo.TODO_STATE_EXEC_FAILED:
+                if self.retry_count > 3:
+                    logger.info(f"todo {self.title} retry count ({self.retry_count}) is too many, ignore")
+                    return False
+
+        now = datetime.now().timestamp()
+        time_diff = self.due_date - now
+        if time_diff < 0:
+            logger.info(f"todo {self.title} is expired, ignore")
+            self.state = AgentTodo.TODO_STATE_EXPIRED
+            return False
+
+        if time_diff > 7*24*3600:
+            logger.info(f"todo {self.title} is far before due date, ignore")
+            return False
+
+        if self.last_do_time:
+            time_diff = now - self.last_do_time
+            if time_diff < 60*15:
+                logger.info(f"todo {self.title} is already do ignore")
+                return False
+
+        logger.info(f"todo {self.title} can do.")
+        return True
+
+
+class AgentWorkLog:
+    def __init__(self) -> None:
+        pass
+
+
+class BaseAIAgent(abc.ABC):
+    @abstractmethod
+    def get_id(self) -> str:
+        pass
+
+    @abstractmethod
+    def get_llm_model_name(self) -> str:
+        pass
+
+    @abstractmethod
+    def get_max_token_size(self) -> int:
+        pass
+
+    @classmethod
+    def get_inner_functions(cls, env:Environment) -> (dict,int):
+        if env is None:
+            return None,0
+
+        all_inner_function = env.get_all_ai_functions()
+        if all_inner_function is None:
+            return None,0
+
+        result_func = []
+        result_len = 0
+        for inner_func in all_inner_function:
+            func_name = inner_func.get_name()
+            this_func = {}
+            this_func["name"] = func_name
+            this_func["description"] = inner_func.get_description()
+            this_func["parameters"] = inner_func.get_parameters()
+            result_len += len(json.dumps(this_func)) / 4
+            result_func.append(this_func)
+
+        return result_func,result_len
+
+    async def do_llm_complection(
+        self,
+        prompt:AgentPrompt,
+        org_msg:AgentMsg=None,
+        env:Environment=None,
+        inner_functions=None,
+        is_json_resp=False,
+    ) -> ComputeTaskResult:
+        from ..frame.compute_kernel import ComputeKernel
+
+        #logger.debug(f"Agent {self.agent_id} do llm token static system:{system_prompt_len},function:{function_token_len},history:{history_token_len},input:{input_len}, totoal prompt:{system_prompt_len + function_token_len + history_token_len} ")
+        if inner_functions is None and env is not None:
+            inner_functions,_ = BaseAIAgent.get_inner_functions(env)
+        if is_json_resp:
+            task_result:ComputeTaskResult = await ComputeKernel.get_instance().do_llm_completion(prompt,resp_mode="json",mode_name=self.get_llm_model_name(),max_token=self.get_max_token_size(),inner_functions=inner_functions,timeout=None)
+        else:
+            task_result:ComputeTaskResult = await ComputeKernel.get_instance().do_llm_completion(prompt,resp_mode="text",mode_name=self.get_llm_model_name(),max_token=self.get_max_token_size(),inner_functions=inner_functions,timeout=None)
+        if task_result.result_code != ComputeTaskResultCode.OK:
+            logger.error(f"_do_llm_complection llm compute error:{task_result.error_str}")
+            #error_resp = msg.create_error_resp(task_result.error_str)
+            return task_result
+
+        result_message = task_result.result.get("message")
+        inner_func_call_node = None
+        if result_message:
+            inner_func_call_node = result_message.get("function_call")
+
+        if inner_func_call_node:
+            call_prompt : AgentPrompt = copy.deepcopy(prompt)
+            func_msg = copy.deepcopy(result_message)
+            del func_msg["tool_calls"]
+            call_prompt.messages.append(func_msg)
+            task_result = await self._execute_func(env,inner_func_call_node,call_prompt,inner_functions,org_msg)
+
+        return task_result
+
+    async def _execute_func(
+        self,
+        env: Environment,
+        inner_func_call_node: dict,
+        prompt: AgentPrompt,
+        inner_functions: dict,
+        org_msg:AgentMsg,
+        stack_limit = 5
+    ) -> ComputeTaskResult:
+        arguments = None
+        try:
+            func_name = inner_func_call_node.get("name")
+            arguments = json.loads(inner_func_call_node.get("arguments"))
+            logger.info(f"llm execute inner func:{func_name} ({json.dumps(arguments)})")
+
+            func_node : AIFunction = env.get_ai_function(func_name)
+            if func_node is None:
+                result_str = f"execute {func_name} error,function not found"
+            else:
+                result_str:str = await func_node.execute(**arguments)
+        except Exception as e:
+            result_str = f"execute {func_name} error:{str(e)}"
+            logger.error(f"llm execute inner func:{func_name} error:{e}")
+
+
+        logger.info("llm execute inner func result:" + result_str)
+
+        prompt.messages.append({"role":"function","content":result_str,"name":func_name})
+        task_result:ComputeTaskResult = await ComputeKernel.get_instance().do_llm_completion(prompt,mode_name=self.get_llm_model_name(),max_token=self.get_max_token_size(),inner_functions=inner_functions)
+        if task_result.result_code != ComputeTaskResultCode.OK:
+            logger.error(f"llm compute error:{task_result.error_str}")
+            return task_result
+
+        if org_msg:
+            internal_call_record = AgentMsg.create_internal_call_msg(func_name,arguments,org_msg.get_msg_id(),org_msg.target)
+            internal_call_record.result_str = task_result.result_str
+            internal_call_record.done_time = time.time()
+            org_msg.inner_call_chain.append(internal_call_record)
+
+        inner_func_call_node = None
+        if stack_limit > 0:
+            result_message : dict = task_result.result.get("message")
+            if result_message:
+                inner_func_call_node = result_message.get("function_call")
+                if inner_func_call_node:
+                    func_msg = copy.deepcopy(result_message)
+                    del func_msg["tool_calls"]
+                    prompt.messages.append(func_msg)
+
+        if inner_func_call_node:
+            return await self._execute_func(env,inner_func_call_node,prompt,inner_functions,org_msg,stack_limit-1)
+        else:
+            return task_result
+
+
+class CustomAIAgent(BaseAIAgent):
+    def __init__(self, agent_id: str, llm_model_name: str, max_token_size: int) -> None:
+        self.agent_id = agent_id
+        self.llm_model_name = llm_model_name
+        self.max_token_size = max_token_size
+
+    def get_id(self) -> str:
+        return self.agent_id
+
+    def get_llm_model_name(self) -> str:
+        return self.llm_model_name
+
+    def get_max_token_size(self) -> int:
+        return self.max_token_size

src/aios/agent/ai_function.py

@@ -0,0 +1,144 @@
+from abc import ABC, abstractmethod
+from typing import Dict,Coroutine,Callable
+
+class ParameterDefine:
+    def __init__(self) -> None:
+        self.name = None
+        self.type = None
+        self.description = None
+        
+
+class AIFunction:
+    def __init__(self) -> None:
+        self.description : str = None
+    
+    @abstractmethod
+    def get_name(self) -> str:
+        """
+        return the name of the function (should be snake case)
+        """
+        pass
+
+    @abstractmethod
+    def get_description(self) -> str:
+        """
+        return a detailed description of what the function does
+        """
+        return self.description
+
+    @abstractmethod
+    def get_parameters(self) -> Dict:
+        """
+        Return the list of parameters to execute this function in the form of
+        JSON schema as specified in the OpenAI documentation:
+        https://platform.openai.com/docs/api-reference/chat/create#chat/create-parameters
+
+        str = run_code(code:str)
+        parameters = {
+            "type": "object",
+            "properties": {
+                "code": {
+                    "type": "string",
+                    "description": "Python code which needs to be executed"
+                }
+            }
+        }
+
+        """
+        pass
+    
+    @abstractmethod
+    async def execute(self, **kwargs) -> str:
+        """
+        Execute the function and return a JSON serializable dict.
+        The parameters are passed in the form of kwargs
+        """
+        pass
+
+    @abstractmethod
+    def is_local(self) -> bool:
+        """
+        is this function call need network?
+        """
+        pass
+
+    @abstractmethod
+    def is_in_zone(self) -> bool:
+        """
+        is this function call in Lan?
+        """
+        pass
+
+    @abstractmethod  
+    def is_ready_only(self) -> bool:
+        pass
+
+    #def load_from_config(self,config:dict) -> bool:
+    #    pass
+
+class FunctionItem:
+    def __init__(self,name,args) -> None:
+        self.name = name
+        self.args = args
+        self.body = None
+
+    def append_body(self,body:str) -> None:
+        if self.body is None:
+            self.body = body
+        else:
+            self.body += body
+
+    def dumps(self) -> str:
+        pass
+
+# call chain is a combination of ai_function,group of ai_function.
+class CallChain:
+    def __init__(self) -> None:
+        pass
+
+    def load_from_config(self,config:dict) -> bool:
+        pass
+
+    async def execute(self):
+        pass
+
+class SimpleAIFunction(AIFunction):
+    def __init__(self,func_id:str,description:str,func_handler:Coroutine,parameters:Dict = None) -> None:
+        self.func_id = func_id
+        self.description = description
+        self.func_handler = func_handler
+        self.parameters = parameters
+
+    def get_name(self) -> str: 
+        return self.func_id
+    
+    def get_parameters(self) -> Dict:
+        if self.parameters is not None:
+            result = {}
+            result["type"] = "object"
+            parm_defines = {}
+            for parm,desc in self.parameters.items():
+                parm_item = {}
+                parm_item["type"] = "string"
+                parm_item["description"] = desc
+                parm_defines[parm] = parm_item
+            result["properties"] = parm_defines
+            return result
+        return {"type": "object", "properties": {}}
+
+
+    async def execute(self,**kwargs) -> str:
+        if self.func_handler is None:
+            return "error: function not implemented"
+        
+        return await self.func_handler(**kwargs)
+
+    def is_local(self) -> bool:
+        return True
+
+    def is_in_zone(self) -> bool:
+        return True
+    
+    def is_ready_only(self) -> bool:
+        return False
+

src/aios/agent/behavior.py

@@ -0,0 +1,4 @@
+# TODO: let agent develolp custmized behavior easily
+class AgentBehavior:
+    def __init__(self) -> None:
+        pass

src/aios/agent/chatsession.py

@@ -0,0 +1,406 @@
+
+import sqlite3 # Because sqlite3 IO operation is small, so we can use sqlite3 directly.(so we don't need to use async sqlite3 now)
+from sqlite3 import Error
+import logging
+import threading
+import datetime
+import uuid
+import json
+
+from ..proto.agent_msg import AgentMsgType, AgentMsg, AgentMsgStatus
+
+class ChatSessionDB:
+    def __init__(self, db_file):
+        """ initialize db connection """
+        self.db_file = db_file
+        self._get_conn()
+
+    def _get_conn(self):
+        """ get db connection """
+        local = threading.local()
+        if not hasattr(local, 'conn'):
+            local.conn = self._create_connection(self.db_file)
+        return local.conn
+
+    def _create_connection(self, db_file):
+        """ create a database connection to a SQLite database """
+        conn = None
+        try:
+            conn = sqlite3.connect(db_file)
+        except Error as e:
+            logging.error("Error occurred while connecting to database: %s", e)
+            return None
+
+        if conn:
+            self._create_table(conn)
+
+        return conn
+
+    def close(self):
+        if not hasattr(self.local, 'conn'):
+            return
+        self.local.conn.close()
+
+    def _create_table(self, conn):
+        """ create table """
+        try:
+            # create sessions table
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS ChatSessions (
+                    SessionID TEXT PRIMARY KEY,
+                    SessionOwner TEXT,
+                    SessionTopic TEXT,
+                    StartTime TEXT,
+                    SummarizePos INTEGER,
+                    Summary TEXT,
+                    ThreadID TEXT
+                );
+            """)
+
+            # create messages table
+            # reciver_id could be None
+
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS Messages (
+                    MessageID TEXT PRIMARY KEY,
+                    SessionID TEXT,
+                    MsgType INTEGER,
+                    PrevMsgID TEXT,
+                    QuoteMsgID TEXT,
+                    RelyMsgID TEXT,
+                    
+                    SenderID TEXT, 
+                    ReceiverID TEXT,
+                    Timestamp TEXT,
+                    
+                    Topic TEXT,
+                    Mentions TEXT,
+                    ContentMIME TEXT,
+                    Content TEXT,
+                    
+                    ActionName TEXT,
+                    ActionParams TEXT,
+                    ActionResult TEXT,
+                    DoneTime TEXT,     
+                         
+                    Status INTEGER
+                );
+            """)
+            conn.commit()
+        except Error as e:
+            logging.error("Error occurred while creating tables: %s", e)
+
+    def insert_chatsession(self, session_id, session_owner,session_topic, start_time,thread_id = ""):
+        """ insert a new session into the ChatSessions table """
+        try:
+            conn = self._get_conn()
+            conn.execute("""
+                INSERT INTO ChatSessions (SessionID, SessionOwner,SessionTopic, StartTime,SummarizePos,Summary,ThreadID)
+                VALUES (?,?, ?, ?,0,"",?)
+            """, (session_id, session_owner,session_topic, start_time,thread_id))
+            conn.commit()
+            return 0  # return 0 if successful
+        except Error as e:
+            logging.error("Error occurred while inserting session: %s", e)
+            return -1  # return -1 if an error occurs
+
+    def insert_message(self, msg:AgentMsg):
+        """ insert a new message into the Messages table """
+        try:
+            action_name = None
+            action_params = None
+            action_result = None
+            mentions = None
+            if msg.mentions:
+                mentions = json.dumps(msg.mentions)
+
+            match msg.msg_type:
+                case AgentMsgType.TYPE_MSG:
+                    pass
+                case AgentMsgType.TYPE_ACTION:
+                    action_name = msg.func_name
+                    action_params = json.dumps(msg.args)
+                    action_result = msg.result_str
+                case AgentMsgType.TYPE_INTERNAL_CALL:
+                    action_name = msg.func_name
+                    action_params = json.dumps(msg.args)
+                    action_result = msg.result_str
+                case AgentMsgType.TYPE_EVENT:
+                    action_name = msg.event_name
+                    action_params = json.dumps(msg.event_args)
+
+
+            conn = self._get_conn()
+            conn.execute("""
+                INSERT INTO Messages (MessageID, SessionID, MsgType, PrevMsgID, SenderID, ReceiverID, Timestamp, Topic,Mentions,ContentMIME,Content,ActionName,ActionParams,ActionResult,DoneTime,Status)
+                VALUES (?, ?, ?, ?, ?, ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """, (msg.msg_id, msg.session_id, msg.msg_type.value, msg.prev_msg_id, msg.sender, msg.target, msg.create_time, msg.topic,mentions,msg.body_mime,msg.body,action_name,action_params,action_result,msg.done_time,msg.status.value))
+            conn.commit()
+
+            if msg.inner_call_chain:
+                for inner_call in msg.inner_call_chain:
+                    self.insert_message(inner_call)
+
+            return 0  # return 0 if successful
+        except Error as e:
+            logging.error("Error occurred while inserting message: %s", e)
+            return -1  # return -1 if an error occurs
+
+    def get_chatsession_by_id(self, session_id):
+        """Get a message by its ID"""
+        conn = self._get_conn()
+        c = conn.cursor()
+        c.execute("SELECT * FROM ChatSessions WHERE SessionID = ?", (session_id,))
+        chatsession = c.fetchone()
+        return chatsession
+
+    def get_chatsession_by_owner_topic(self, owner_id, topic):
+        """Get a chatsession by its owner and topic"""
+        conn = self._get_conn()
+        c = conn.cursor()
+        c.execute("SELECT * FROM ChatSessions WHERE SessionOwner = ? AND SessionTopic = ?", (owner_id,topic))
+        chatsession = c.fetchone()
+        return chatsession
+
+    def list_chatsessions(self, owner_id, limit, offset):
+        """ retrieve sessions with pagination """
+        try:
+            conn = self._get_conn()
+            cursor = conn.cursor()
+            cursor.execute("""
+                SELECT SessionID FROM ChatSessions
+                WHERE SessionOwner = ?           
+                ORDER BY StartTime DESC
+                LIMIT ? OFFSET ? 
+            """, (owner_id,limit, offset))
+            results = cursor.fetchall()
+            #self.close()
+            return results  # return 0 and the result if successful
+        except Error as e:
+            logging.error("Error occurred while getting sessions: %s", e)
+            return -1, None  # return -1 and None if an error occurs
+
+    def get_message_by_id(self, message_id):
+        """Get a message by its ID"""
+        conn =self._get_conn()
+        c = conn.cursor()
+        c.execute("SELECT MessageID, SessionID, MsgType, PrevMsgID, SenderID, ReceiverID, Timestamp, Topic,Mentions,ContentMIME,Content,ActionName,ActionParams,ActionResult,DoneTime,Status FROM Messages WHERE MessageID = ?", (message_id,))
+        message = c.fetchone()
+        return message
+
+    # read message from begin->now
+    def read_message(self,session_id,limit,offset):
+        try:
+            conn = self._get_conn()
+            cursor = conn.cursor()
+            cursor.execute("""
+                SELECT MessageID, SessionID, MsgType, PrevMsgID, SenderID, ReceiverID, Timestamp, Topic,Mentions,ContentMIME,Content,ActionName,ActionParams,ActionResult,DoneTime,Status FROM Messages
+                WHERE SessionID = ?
+                ORDER BY Timestamp 
+                LIMIT ? OFFSET ?
+            """, (session_id, limit, offset))
+            results = cursor.fetchall()
+            #self.close()
+            return results  # return 0 and the result if successful
+        except Error as e:
+            logging.error("Error occurred while getting messages: %s", e)
+            return -1, None  # return -1 and None if an error occurs
+        
+    # read message from  now->beign    
+    def get_messages(self, session_id, limit, offset):
+        """ retrieve messages of a session with pagination """
+        try:
+            conn = self._get_conn()
+            cursor = conn.cursor()
+            cursor.execute("""
+                SELECT MessageID, SessionID, MsgType, PrevMsgID, SenderID, ReceiverID, Timestamp, Topic,Mentions,ContentMIME,Content,ActionName,ActionParams,ActionResult,DoneTime,Status FROM Messages
+                WHERE SessionID = ?
+                ORDER BY Timestamp DESC
+                LIMIT ? OFFSET ?
+            """, (session_id, limit, offset))
+            results = cursor.fetchall()
+            #self.close()
+            return results  # return 0 and the result if successful
+        except Error as e:
+            logging.error("Error occurred while getting messages: %s", e)
+            return -1, None  # return -1 and None if an error occurs
+
+    def update_message_status(self, message_id, status):
+        """ update the status of a message """
+        try:
+            conn = self._get_conn()
+            conn.execute("""
+                UPDATE Messages
+                SET Status = ?
+                WHERE MessageID = ?
+            """, (status, message_id))
+            conn.commit()
+            return 0  # return 0 if successful
+        except Error as e:
+            logging.error("Error occurred while updating message status: %s", e)
+            return -1  # return -1 if an error occurs
+
+    def update_session_summary(self, session_id, summarize_pos, summary):
+        """ update the summary of a session """
+        try:
+            conn = self._get_conn()
+            conn.execute("""
+                UPDATE ChatSessions
+                SET SummarizePos = ?, Summary = ?
+                WHERE SessionID = ?
+            """, (summarize_pos, summary, session_id))
+            conn.commit()
+            return 0  # return 0 if successful
+        except Error as e:
+            logging.error("Error occurred while updating session summary: %s", e)
+            return -1
+        
+    def update_session_thread_id(self, session_id, thread_id):
+        """ update the threadid of a session """
+        try:
+            conn = self._get_conn()
+            conn.execute("""
+                UPDATE ChatSessions
+                SET ThreadID = ?
+                WHERE SessionID = ?
+            """, (thread_id, session_id))
+            conn.commit()
+            return 0  # return 0 if successful
+        except Error as e:
+            logging.error("Error occurred while updating session threadid: %s", e)
+            return -1
+
+# chat session store the chat history between owner and agent
+# chat session might be large, so can read / write at stream mode.
+class AIChatSession:
+    _dbs = {}
+    #@classmethod
+    #async def get_session_by_id(cls,session_id:str,db_path:str):
+    #    db = cls._dbs.get(db_path)
+    #    if db is None:
+    #        db = ChatSessionDB(db_path)
+    #        cls._dbs[db_path] = db
+    #    db.get_chatsession_by_id(session_id)
+    #    #result = AIChatSession()
+
+    @classmethod
+    def get_session(cls,owner_id:str,session_topic:str,db_path:str,auto_create = True) -> 'AIChatSession':
+        db = cls._dbs.get(db_path)
+        if db is None:
+            db = ChatSessionDB(db_path)
+            cls._dbs[db_path] = db
+
+        result = None
+        session = db.get_chatsession_by_owner_topic(owner_id,session_topic)
+        if session is None:
+            if auto_create:
+                session_id = "CS#" + uuid.uuid4().hex
+                db.insert_chatsession(session_id,owner_id,session_topic,datetime.datetime.now())
+                result = AIChatSession(owner_id,session_id,db)
+        else:
+            result = AIChatSession(owner_id,session[0],db)
+            result.topic = session_topic
+            result.summarize_pos = session[4]
+            result.summary = session[5]
+            result.openai_thread_id = session[6]
+
+        return result
+    
+    @classmethod
+    def get_session_by_id(cls,session_id:str,db_path:str)->'AIChatSession':
+        db = cls._dbs.get(db_path)
+        if db is None:
+            db = ChatSessionDB(db_path)
+            cls._dbs[db_path] = db
+
+        result = None
+        session = db.get_chatsession_by_id(session_id)
+        if session is None:
+            return None
+        else:
+            result = AIChatSession(session[1],session[0],db)
+            result.topic = session[2]
+            result.summarize_pos = session[4]
+            result.summary = session[5]
+            result.openai_thread_id = session[6]
+
+        return result        
+
+    @classmethod
+    def list_session(cls,owner_id:str,db_path:str) -> list[str]:
+        db = cls._dbs.get(db_path)
+        if db is None:
+            db = ChatSessionDB(db_path)
+            cls._dbs[db_path] = db
+
+        result = db.list_chatsessions(owner_id,16,0)
+        result_ids = []
+        for r in result:
+            result_ids.append(r[0])
+        return result_ids    
+
+
+    def __init__(self,owner_id:str, session_id:str, db:ChatSessionDB) -> None:
+        self.owner_id :str = owner_id
+        self.session_id : str = session_id
+        self.db : ChatSessionDB = db
+
+        self.topic : str = None
+        self.start_time : str = None
+        self.summarize_pos : int = 0
+        self.summary = None
+        self.openai_thread_id = None
+
+    def get_owner_id(self) -> str:
+        return self.owner_id
+
+    def read_history(self, number:int=10,offset=0,order="revers") -> [AgentMsg]:
+        if order == "revers":
+            msgs = self.db.get_messages(self.session_id, number, offset)
+        else:
+            msgs = self.db.read_message(self.session_id, number, offset)
+            
+        result = []
+        for msg in msgs:
+            agent_msg = AgentMsg()
+            agent_msg.msg_id = msg[0]
+            agent_msg.session_id = msg[1]
+            agent_msg.msg_type = AgentMsgType(msg[2])
+            agent_msg.prev_msg_id = msg[3]
+            agent_msg.sender = msg[4]
+            agent_msg.target = msg[5]
+            agent_msg.create_time = msg[6]
+            agent_msg.topic = msg[7]
+            if msg[8] is not None:
+                agent_msg.mentions = json.loads(msg[8])
+            agent_msg.body_mime = msg[9]
+            agent_msg.body = msg[10]
+            agent_msg.func_name = msg[11]
+            if msg[12] is not None:
+                agent_msg.args = json.loads(msg[12])
+            agent_msg.result_str = msg[13]
+            agent_msg.done_time = msg[14]
+            agent_msg.status = AgentMsgStatus(msg[15])
+
+            result.append(agent_msg)
+        return result
+
+    def append(self,msg:AgentMsg) -> None:
+        msg.session_id = self.session_id
+        self.db.insert_message(msg)
+
+
+    def update_think_progress(self,progress:int,new_summary:str) -> None:
+        self.db.update_session_summary(self.session_id,progress,new_summary)
+        self.summarize_pos = progress
+        self.summary = new_summary
+
+    def update_openai_thread_id(self,thread_id:str) -> None:
+        self.db.update_session_thread_id(self.session_id,thread_id)
+        self.openai_thread_id = thread_id
+
+    #def attach_event_handler(self,handler) -> None:
+    #    """chat session changed event handler"""
+    #    pass
+
+    #TODO : add iterator interface for read chat history

src/aios/agent/role.py

@@ -0,0 +1,80 @@
+import logging
+
+from .agent_base import AgentPrompt
+
+class AIRole:
+    def __init__(self) -> None:
+        self.agent_instance_id : str = None
+        self.role_name : str = None
+        self.role_id :str = None # $workflow_id.$sub_workflow_id.$role_name
+        self.fullname : str = None
+        self.agent_name : str = None
+        self.prompt : AgentPrompt = None
+        self.introduce : str = None
+        self.agent = None
+        self.enable_function_list : list[str] = None
+        self.history_len = 10
+
+    def load_from_config(self,config:dict) -> bool:
+        name_node = config.get("name")
+        if name_node is None:
+            logging.error("role name is not found!")
+            return False
+        self.role_name = name_node
+
+
+        agent_id_node = config.get("agent")
+        if agent_id_node is None:
+            logging.error("agent id is not found!")
+            return False
+        self.agent_name = agent_id_node
+
+        prompt_node = config.get("prompt")
+        if prompt_node:
+            self.prompt = AgentPrompt()
+            if self.prompt.load_from_config(prompt_node) is False:
+                logging.error("load prompt failed!")
+                return False
+
+        intro_node = config.get("intro")
+        if intro_node is not None:
+            self.introduce = intro_node
+
+        history_node = config.get("history_len")
+        if history_node is not None:
+            self.history_len = int(history_node)
+
+        if config.get("enable_function") is not None:
+            self.enable_function_list = config["enable_function"]
+
+    def get_role_id(self) -> str:
+        return self.role_id
+
+    def get_intro(self) -> str:
+        return self.introduce
+
+    def get_name(self) -> str:
+        return self.role_name
+
+    def get_prompt(self) -> AgentPrompt:
+        return self.prompt
+
+class AIRoleGroup:
+    def __init__(self) -> None:
+        self.roles : dict[str,AIRole] = {}
+        self.owner_name : str = None
+
+    def load_from_config(self,config:dict) -> bool:
+        for k,v in config.items():
+            role = AIRole()
+            if role.load_from_config(v) is False:
+                logging.error(f"load role {k} failed!")
+                return False
+            role.role_id = self.owner_name + "." + k
+            self.roles[k] = role
+
+        return True
+
+    def get(self,role_name:str) -> AIRole:
+        return self.roles.get(role_name)
+

src/aios/agent/workflow.py

@@ -0,0 +1,519 @@
+import logging
+import asyncio
+import json
+import os
+import time
+from asyncio import Queue
+from typing import Optional,Tuple,List
+from abc import ABC, abstractmethod
+
+from ..proto.compute_task import *
+from ..proto.agent_msg import *
+
+from .agent_base import *
+from .chatsession import AIChatSession
+from .role import AIRole,AIRoleGroup
+from .ai_function import AIFunction,FunctionItem
+
+from ..frame.compute_kernel import ComputeKernel
+from ..frame.bus import AIBus
+
+from ..environment.environment import Environment,EnvironmentEvent
+from ..environment.workflow_env import WorkflowEnvironment
+
+
+logger = logging.getLogger(__name__)
+
+class MessageFilter:
+    def __init__(self) -> None:
+        self.filters = {}
+
+    def select(self,msg:AgentMsg) -> str:
+        star_target = self.filters.get("*")
+        if star_target is not None:
+            return star_target
+
+        # TODO: add more filter
+        return None
+
+    def load_from_config(self,config:dict) -> bool:
+        self.filters = config
+        return True
+
+
+
+
+
+class Workflow:
+    def __init__(self) -> None:
+        self.workflow_name : str = None
+        self.workflow_id : str = None
+        self.rule_prompt : AgentPrompt = None
+        self.workflow_config = None
+        self.role_group : dict = None
+        self.input_filter : MessageFilter= None
+        self.connected_environment = {}
+        self.sub_workflows = {}
+        self.owner_workflow = None
+        self.db_file = None
+        self.env_db_file = None
+        self.workflow_env:WorkflowEnvironment = None
+
+        self.is_start = False
+        self.msg_queue = Queue()
+
+    def get_bus(self) -> AIBus:
+        return AIBus.get_default_bus()
+
+    def set_owner(self,owner):
+        self.owner_workflow = owner
+
+    def load_from_config(self,config:dict) -> bool:
+        if config is None:
+            return False
+
+        if config.get("name") is None:
+            logger.error("workflow config must have name")
+            return False
+        self.workflow_name = config.get("name")
+        if self.owner_workflow is None:
+            self.workflow_id = self.workflow_name
+        else:
+            self.workflow_id = self.owner_workflow.workflow_id + "." + self.workflow_name
+            self.db_file = self.owner_workflow.db_file
+
+        if config.get("prompt") is not None:
+            self.rule_prompt = AgentPrompt()
+            if self.rule_prompt.load_from_config(config.get("prompt")) is False:
+                logger.error("Workflow load prompt failed")
+                return False
+
+        if config.get("roles") is None:
+            logger.error("workflow config must have roles")
+            return False
+        self.role_group = AIRoleGroup()
+        self.role_group.owner_name = self.workflow_id
+        if self.role_group.load_from_config(config.get("roles")) is False:
+            logger.error("Workflow load role_group failed")
+            return False
+
+        if config.get("filter") is not None:
+            self.input_filter = MessageFilter()
+            if self.input_filter.load_from_config(config.get("filter")) is False:
+                logger.error("Workflow load input_filter failed")
+                return False
+
+        if self.owner_workflow is None:
+            self.env_db_file = os.path.dirname(self.db_file) + "/" + self.workflow_id + "_env.db"
+        else:
+            self.env_db_file = self.owner_workflow.env_db_file
+        self.workflow_env = WorkflowEnvironment(self.workflow_id,self.env_db_file)
+
+        env_ndoe = config.get("enviroment")
+        if  env_ndoe is not None:
+            if self._load_env_from_config(env_ndoe) is False:
+                logger.error("Workflow load env failed")
+                return False
+
+        connected_env_ndoe = config.get("connected_env")
+        if  connected_env_ndoe is not None:
+           for _node in connected_env_ndoe:
+                env_id = _node.get("env_id")
+                if env_id is None:
+                    continue
+
+                remote_env = Environment.get_env_by_id(env_id)
+                if remote_env is None:
+                     logger.error(f"Workflow load connected_env failed, env {env_id} not found!")
+                     return False
+                self.connect_to_environment(remote_env,_node.get("event2msg"))
+
+        sub_workflows = config.get("sub_workflows")
+        if sub_workflows is not None:
+            if self._load_sub_workflows(sub_workflows) is False:
+                logger.error("Workflow load sub workflows failed")
+                return False
+
+        return True
+
+    def _load_env_from_config(self,config:dict) -> bool:
+        for k,v in config.items():
+            self.workflow_env.set_value(k,v,False)
+
+    def _load_sub_workflows(self,config:dict) -> bool:
+        for k,v in config.items():
+            sub_workflow = Workflow()
+            sub_workflow.set_owner(self)
+
+            if sub_workflow.load_from_config(v) is False:
+                logger.error(f"load sub workflow {k} failed!")
+                return False
+            self.sub_workflows[k] = sub_workflow
+        return True
+
+    def _parse_msg_target(self,s:str)->list[str]:
+        return s.split(".")
+
+    async def _forword_msg(self,inner_obj_id,msg):
+        i : int = 1
+        current_workflow = self
+        while i < len(inner_obj_id):
+            if i == len(inner_obj_id) - 1:
+                the_role : AIRole = current_workflow.role_group.get(inner_obj_id[i])
+                current_workflow_chatsession = AIChatSession.get_session(current_workflow.workflow_id,msg.sender + "#" + msg.topic,current_workflow.db_file)
+                if the_role is not None:
+                    return await current_workflow.role_process_msg(msg,the_role,current_workflow_chatsession)
+                sub_workflow = current_workflow.sub_workflows.get(inner_obj_id[i])
+                if sub_workflow is not None:
+                    return await sub_workflow._process_msg(msg)
+                logger.error(f"{msg.target} not found! forword message failed!")
+                return None
+            else:
+                current_workflow = current_workflow.sub_workflows.get(inner_obj_id[i])
+                if current_workflow is None:
+                    logger.error(f"sub workflow {inner_obj_id[i]} not found!")
+                    return None
+
+            i += 1
+
+        logger.error(f"{msg.target} not found! forword message failed!")
+        return None
+
+    def get_workflow_id_from_target(self,target:str) -> str:
+        target_list = target.split(".")
+        if len(target_list) == 0:
+            return target
+        else:
+            result_str = ""
+            p = 0
+            for s in target_list:
+                p = p + 1
+                result_str += s
+                if p < len(target_list)-1:
+                    result_str += "."
+                else:
+                    return result_str
+
+    async def _process_msg(self,msg:AgentMsg) -> AgentMsg:
+        real_target = msg.target.split(".")[0]
+        targets = self._parse_msg_target(msg.target)
+        if len(targets) > 1:
+            return await self._forword_msg(targets,msg)
+
+        #0 we don't support workflow join a group right now, this cloud be a feture in future
+        if msg.mentions is not None:
+            logger.warn(f"workflow {self.workflow_id} recv a group chat message,not support ignore!")
+            error_resp = msg.create_error_resp(f"workflow {self.workflow_id} recv a group chat message,not support ignore!")
+            return error_resp
+
+        #1. workflow start process message
+        # this is workflow's group_chat session
+        session_topic = msg.sender + "#" + msg.topic
+        chatsesssion = AIChatSession.get_session(self.workflow_id,session_topic,self.db_file)
+
+        #2. find role by msg.mentions or workflow's selector logic
+        if msg.mentions is not None:
+            if not self.workflow_id in msg.mentions:
+                chatsesssion.append(msg)
+                logger.info(f"workflow {self.workflow_id} recv a group chat message from {msg.sender},but is not mentioned,ignore!")
+                return None
+
+            for mention in msg.mentions:
+                this_role = self.role_group.get(mention)
+                if this_role is not None:
+                    return await self.role_process_msg(msg,this_role,chatsesssion)
+
+        if self.input_filter is not None:
+            select_role_id = self.input_filter.select(msg)
+            if select_role_id is not None:
+                select_role = self.role_group.get(select_role_id)
+                if select_role is None:
+                    logger.error(f"input_filter return invalid role id:{select_role_id}, role not found in role_group")
+                    return None
+
+                return await self.role_process_msg(msg,select_role,chatsesssion)
+            else:
+                logger.error(f"input_filter return None for :{msg.body}")
+                return None
+
+        err_str = f"{self.workflow_id}:no role can process this msg:{msg.body}"
+        logger.error(err_str)
+        error_resp = msg.create_error_resp(err_str)
+        return error_resp
+
+    async def role_post_msg(self,msg:AgentMsg,the_role:AIRole,workflow_chat_session:AIChatSession):
+        msg.sender = the_role.get_role_id()
+
+        target_role = self.role_group.get(msg.target)
+        if target_role:
+            msg.target = target_role.get_role_id()
+            logger.info(f"{msg.sender} post message {msg.msg_id} to inner role: {msg.target}")
+            asyncio.create_task(self.role_process_msg(msg,target_role,workflow_chat_session))
+            return
+
+        target_workflow = self.sub_workflows.get(msg.target)
+        if target_workflow:
+            msg.target = target_workflow.workflow_id
+            logger.info(f"{msg.sender} post message {msg.msg_id} to sub workflow: {msg.target}")
+            asyncio.create_task(target_workflow._process_msg(msg))
+
+        logger.info(f"{msg.sender} post message {msg.msg_id} to AIBus: {msg.target}")
+        await self.get_bus().post_message(msg,msg.target)
+        return
+
+
+    async def role_send_msg(self,msg:AgentMsg,the_role:AIRole,workflow_chat_session:AIChatSession):
+        msg.sender = the_role.get_role_id()
+        target_role = self.role_group.get(msg.target)
+        if target_role:
+            # msg.target = target_role.get_role_id()
+            logger.info(f"{msg.sender} send message {msg.msg_id} to inner role: {msg.target}")
+            return await self.role_process_msg(msg,target_role,workflow_chat_session)
+
+        target_workflow = self.sub_workflows.get(msg.target)
+        if target_workflow:
+            # msg.target = target_workflow.workflow_id
+            logger.info(f"{msg.sender} send message {msg.msg_id} to sub workflow: {msg.target}")
+            return await target_workflow._process_msg(msg)
+
+        logger.info(f"{msg.sender} post message {msg.msg_id} to AIBus: {msg.target}")
+        return await self.get_bus().send_message(msg)
+
+    async def role_call(self,func_item:FunctionItem,the_role:AIRole):
+        logger.info(f"{the_role.role_id} call {func_item.name} ")
+        arguments = func_item.args
+
+        func_node : AIFunction = self.workflow_env.get_ai_function(func_item.name)
+        if func_node is None:
+            return "execute failed,function not found"
+
+        result_str:str = await func_node.execute(**arguments)
+        return result_str
+
+    async def role_post_call(self,func_item:FunctionItem,the_role:AIRole):
+        logger.info(f"{the_role.role_id} post call {func_item.name} ")
+        return await self.role_call(func_item,the_role)
+
+    def _format_msg_by_env_value(self,prompt:AgentPrompt):
+        if self.workflow_env is None:
+            return
+
+        for msg in prompt.messages:
+            old_content = msg.get("content")
+            msg["content"] = old_content.format_map(self.workflow_env)
+
+    def _get_inner_functions(self,the_role:AIRole) -> dict:
+        all_inner_function = self.workflow_env.get_all_ai_functions()
+        if all_inner_function is None:
+            return None
+
+        result_func = []
+        for inner_func in all_inner_function:
+            func_name = inner_func.get_name()
+            if the_role.enable_function_list is not None:
+                if len(the_role.enable_function_list) > 0:
+                    if func_name not in the_role.enable_function_list:
+                        logger.debug(f"agent {the_role.agent.agent_id} ignore inner func:{func_name}")
+                        continue
+                else:
+                    continue
+            this_func = {}
+            this_func["name"] = func_name
+            this_func["description"] = inner_func.get_description()
+            this_func["parameters"] = inner_func.get_parameters()
+            result_func.append(this_func)
+        if len(result_func) > 0:
+            return result_func
+        return None
+
+    async def _role_execute_func(self,the_role:AIRole,inenr_func_call_node:dict,prompt:AgentPrompt,org_msg:AgentMsg,stack_limit = 5) -> [str,int]:
+
+        func_name = inenr_func_call_node.get("name")
+        arguments = json.loads(inenr_func_call_node.get("arguments"))
+        ineternal_call_record = AgentMsg.create_internal_call_msg(func_name,arguments,org_msg.get_msg_id(),org_msg.target)
+        func_node : AIFunction = self.workflow_env.get_ai_function(func_name)
+        result_str : str = ""
+        if func_node is None:
+            result_str = f"execute {func_name} failed,function not found"
+        else:
+            try:
+                result_str = await func_node.execute(**arguments)
+            except Exception as e:
+                result_str = f"execute {func_name} error:{str(e)}"
+                logger.error(f"llm execute inner func:{func_name} error:{e}")
+                logger.exception(e)
+
+
+        inner_functions = self._get_inner_functions(the_role)
+        prompt.messages.append({"role":"function","content":result_str,"name":func_name})
+        task_result:ComputeTaskResult = await ComputeKernel.get_instance().do_llm_completion(prompt,
+                                                                                the_role.agent.llm_model_name,the_role.agent.max_token_size,
+                                                                                inner_functions)
+        if task_result.result_code != ComputeTaskResultCode.OK:
+            logger.error(f"llm compute error:{task_result.error_str}")
+            return task_result.error_str,1
+
+        ineternal_call_record.result_str = task_result.result_str
+        ineternal_call_record.done_time = time.time()
+        org_msg.inner_call_chain.append(ineternal_call_record)
+        if stack_limit > 0:
+            result_message = task_result.result.get("message")
+            if result_message:
+                inner_func_call_node = result_message.get("function_call")
+
+        if inner_func_call_node:
+            return await self._role_execute_func(the_role,inner_func_call_node,prompt,org_msg,stack_limit-1)
+        else:
+            return task_result.result_str,0
+
+    def _is_in_same_workflow(self,msg) -> bool:
+        pass
+
+    async def role_process_msg(self,msg:AgentMsg,the_role:AIRole,workflow_chat_session:AIChatSession) -> AgentMsg:
+        msg.target = the_role.get_role_id()
+
+        prompt = AgentPrompt()
+        prompt.append(the_role.agent.agent_prompt)
+        prompt.append(self.get_workflow_rule_prompt())
+        prompt.append(the_role.get_prompt())
+        # prompt.append(self._get_function_prompt(the_role.get_name()))
+        # prompt.append(self._get_knowlege_prompt(the_role.get_name()))
+
+        #support group chat, user content include sender name!
+        prompt.append(await self._get_prompt_from_session(the_role,workflow_chat_session))
+
+        msg_prompt = AgentPrompt()
+        msg_prompt.messages = [{"role":"user","content":f"user name is {msg.sender}, his question is :{msg.body}"}]
+        prompt.append(msg_prompt)
+
+        self._format_msg_by_env_value(prompt)
+        inner_functions = self._get_inner_functions(the_role)
+
+        async def _do_process_msg():
+            #TODO: send msg to agent might be better?
+            task_result:ComputeTaskResult = await ComputeKernel.get_instance().do_llm_completion(prompt,the_role.agent.get_llm_model_name(),the_role.agent.get_max_token_size(),inner_functions)
+            if task_result.result_code != ComputeTaskResultCode.OK:
+                logger.error(f"llm compute error:{task_result.error_str}")
+                error_resp = msg.create_error_resp(task_result.error_str)
+                return error_resp
+
+            result_str = task_result.result_str
+            logger.info(f"{the_role.role_id} process {msg.sender}:{msg.body},llm str is :{result_str}")
+
+            result_message = task_result.result.get("message")
+            if result_message:
+                inner_func_call_node = result_message.get("function_call")
+
+            if inner_func_call_node:
+                #TODO to save more token ,can i use msg_prompt?
+                result_str,r_code = await self._role_execute_func(the_role,inner_func_call_node,prompt,msg)
+                if r_code != 0:
+                    error_resp = msg.create_error_resp(result_str)
+                    return error_resp
+
+            result : LLMResult = LLMResult.from_str(result_str)
+            for postmsg in result.post_msgs:
+                postmsg.prev_msg_id = msg.get_msg_id()
+                # might be craete a new msg.topic for this postmsg
+                postmsg.topic = msg.topic
+
+                await self.role_post_msg(postmsg,the_role,workflow_chat_session)
+                if not self._is_in_same_workflow(postmsg):
+                    role_sesion = AIChatSession.get_session(the_role.get_role_id(),f"{postmsg.target}#{msg.topic}",self.db_file)
+                    role_sesion.append(postmsg)
+                else:
+                    # message will be saved in role.process_message
+                    pass
+
+
+            for post_call in result.post_calls:
+                action_msg = msg.create_action_msg(post_call[0],post_call[1],the_role.get_role_id())
+                workflow_chat_session.append(action_msg)
+                await self.role_post_call(post_call,the_role)
+                #save post_call
+
+            result_prompt_str = ""
+            match result.state:
+                case "ignore":
+                    return None
+                case "reponsed":
+                    resp_msg = msg.create_resp_msg(result.resp)
+                    resp_msg.sender = the_role.get_role_id()
+                    # It is always the person handling the messages who puts them into the session.
+                    workflow_chat_session.append(msg)
+                    workflow_chat_session.append(resp_msg)
+                    #await self.get_bus().resp_message(resp_msg)
+                    return resp_msg
+                case "waiting":
+                    for sendmsg in result.send_msgs:
+                        target = sendmsg.target
+                        sendmsg.topic = msg.topic
+                        sendmsg.prev_msg_id = msg.get_msg_id()
+                        send_resp = await self.role_send_msg(sendmsg,the_role,workflow_chat_session)
+                        if send_resp is not None:
+                            result_prompt_str += f"\n# {target} response is : \n{send_resp.body}"
+
+                        if not self._is_in_same_workflow(sendmsg):
+                            role_sesion = AIChatSession.get_session(the_role.get_role_id(),f"{sendmsg.target}#{sendmsg.topic}",self.db_file)
+                            role_sesion.append(sendmsg)
+                            role_sesion.append(send_resp)
+                        else:
+                             # message will be saved in role.process_message
+                            pass
+
+                    this_llm_resp_prompt = AgentPrompt()
+                    this_llm_resp_prompt.messages = [{"role":"assistant","content":result_str}]
+                    prompt.append(this_llm_resp_prompt)
+
+                    result_prompt = AgentPrompt()
+                    result_prompt.messages = [{"role":"user","content":result_prompt_str}]
+                    prompt.append(result_prompt)
+                    return await _do_process_msg()
+
+        return await _do_process_msg()
+
+    async def _get_prompt_from_session(self,the_role:AIRole,chatsession:AIChatSession) -> AgentPrompt:
+        messages = chatsession.read_history(the_role.history_len) # read last 10 message
+        result_prompt = AgentPrompt()
+        
+        for msg in reversed(messages):
+            if msg.sender == the_role.role_id:
+                result_prompt.messages.append({"role":"assistant","content":msg.body})
+            else:
+                result_prompt.messages.append({"role":"user","content":f"{msg.body}"})
+
+        return result_prompt
+
+    def _get_knowlege_prompt(self,role_name:str) -> AgentPrompt:
+        pass
+
+    def get_workflow_rule_prompt(self) -> AgentPrompt:
+        return self.rule_prompt
+
+    def _env_event_to_msg(self,env_event:EnvironmentEvent) -> AgentMsg:
+        pass
+
+    def get_inner_environment(self,env_id:str) -> Environment:
+        pass
+
+    def connect_to_environment(self,the_env:Environment,conn_info:dict) -> None:
+        if the_env is not None:
+            self.workflow_env.add_owner_env(the_env)
+
+            #for event2msg in conn_info:
+            #    for k,v in event2msg:
+            #        if k == "role":
+            #            continue
+            #        else:
+            #
+            #            def _env_msg_handler(env_event:EnvironmentEvent) -> None:
+            #                the_msg:AgentMsg= self._env_event_to_msg(env_event)
+            #                self.role_post_msg
+
+            #            the_env.attach_event_handler(k,_env_msg_handler)
+            #            break
+
+
+
+
+

src/aios/environment/asr_function.py

@@ -0,0 +1,55 @@
+import logging
+from typing import Dict
+
+from ..frame.compute_kernel import ComputeKernel
+from ..agent.ai_function import AIFunction
+
+logger = logging.getLogger(__name__)
+
+
+class AsrFunction(AIFunction):
+    def __init__(self):
+        self.func_id = "speech_to_text"
+        self.description = "语音识别，将语音转换为文字"
+
+    def get_name(self) -> str:
+        return self.func_id
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {
+            "type": "object",
+            "properties": {
+                "audio_file": {"type": "string", "description": "音频文件路径"},
+                "model": {"type": "string", "description": "识别模型", "enum": ["openai-whisper"]},
+                "prompt": {"type": "string", "description": "提示语句,可以为None"},
+                "response_format": {"type": "string", "description": "返回格式", "enum": ["text", "json", "srt", "verbose_json", "vtt"]},
+            }
+        }
+
+    async def execute(self, **kwargs) -> str:
+        logger.info(f"execute asr function: {kwargs}")
+
+        audio_file = kwargs.get("audio_file")
+        model = kwargs.get("model")
+        prompt = kwargs.get("prompt")
+        response_format = kwargs.get("response_format")
+        if response_format is None:
+            response_format = "text"
+
+        result = await ComputeKernel.get_instance().do_speech_to_text(audio_file, model, prompt, response_format)
+        if result is not None:
+            return f"exec speech_to_text Ok. {response_format} is\n```\n{result.result_str}\n```"
+        else:
+            return "exec speech_to_text failed"
+
+    def is_local(self) -> bool:
+        return True
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False

src/aios/environment/code_interpreter.py

@@ -0,0 +1,426 @@
+import logging
+import os
+import pathlib
+import shutil
+import subprocess
+import sys
+import re
+import time
+import ast
+from concurrent.futures import ThreadPoolExecutor
+from hashlib import md5
+from typing import Optional, Union, List, Tuple
+from generic_escape import GenericEscape
+
+from ..storage.storage import AIStorage
+
+try:
+    import docker
+except ImportError:
+    docker = None
+
+CODE_BLOCK_PATTERN = r"```[ \t]*(\w+)?[ \t]*\r?\n(.*?)\r?\n[ \t]*```"
+UNKNOWN = "unknown"
+TIMEOUT_MSG = "Timeout"
+DEFAULT_TIMEOUT = 600
+WIN32 = sys.platform == "win32"
+PATH_SEPARATOR = WIN32 and "\\" or "/"
+
+logger = logging.getLogger(__name__)
+
+
+BUILT_IN_MODULES = set(
+    [
+        "sys",
+        "os",
+        "math",
+        "random",
+        "datetime",
+        "json",
+        "re",
+        "subprocess",
+        "time",
+        "threading",
+        "logging",
+        "collections",
+        "itertools",
+        "functools",
+        "operator",
+        "pathlib",
+        "shutil",
+        "tempfile",
+        "pickle",
+        "io",
+        "argparse",
+        "typing",
+        "unittest",
+        "contextlib",
+        "abc",
+        "heapq",
+        "bisect",
+        "copy",
+        "decimal",
+        "fractions",
+        "hashlib",
+        "secrets",
+        "statistics",
+        "difflib",
+        "doctest",
+        "enum",
+        "inspect",
+        "traceback",
+        "weakref",
+        "gc",
+        "mmap",
+        "msvcrt",
+        "winreg",
+        "array",
+        "audioop",
+        "binascii",
+        "cProfile",
+        "concurrent.futures",
+        "configparser",
+        "csv",
+        "ctypes",
+        "dateutil",
+        "dis",
+        "fnmatch",
+        "getopt",
+        "glob",
+        "gzip",
+        "pdb",
+        "pprint",
+        "profile",
+        "pstats",
+        "queue",
+        "socket",
+        "sqlite3",
+        "ssl",
+        "struct",
+        "tarfile",
+        "telnetlib",
+        "timeit",
+        "tokenize",
+        "uuid",
+        "xml",
+        "zipfile",
+        "zlib",
+    ]
+)
+
+
+def get_imports(code: str) -> List[str]:
+    root = ast.parse(code)
+
+    imports = []
+    for node in ast.iter_child_nodes(root):
+        if isinstance(node, ast.Import):
+            module_names = [alias.name for alias in node.names]
+        elif isinstance(node, ast.ImportFrom):
+            module_names = [node.module]
+        else:
+            continue
+
+        for name in module_names:
+            # Exclude built-in modules
+            if name not in BUILT_IN_MODULES:
+                imports.append(name)
+
+    return imports
+
+
+def write_requirements(code: str, requirements_filepath: str):
+    imports = get_imports(code)
+
+    with open(requirements_filepath, "w") as file:
+        for module in imports:
+            file.write(module + "\n")
+
+
+def _cmd(lang):
+    if lang.startswith("python") or lang in ["bash", "sh", "powershell"]:
+        return lang
+    if lang in ["shell"]:
+        return "sh"
+    if lang in ["ps1"]:
+        return "powershell"
+    raise NotImplementedError(f"{lang} not recognized in code execution")
+
+
+def create_runner(code: str, timeout: int = 30) -> str:
+    """
+    Create a Python script that runs the code and prints the output
+    """
+    code = GenericEscape().escape(code)
+    # Create a runner script
+    runner = f"""
+import os
+import subprocess
+
+my_env = os.environ.copy()
+my_env["PYTHONIOENCODING"] = "utf-8"
+
+process = subprocess.Popen(
+    f"python -i -q -u".split(),
+    stdin=subprocess.PIPE,
+    stdout=subprocess.PIPE,
+    stderr=subprocess.PIPE,
+    text=True,
+    bufsize=0,
+    universal_newlines=True,
+    env=my_env
+)
+
+process.stdin.write("{code}" + "\\n")
+process.stdin.write("exit()\\n")
+process.stdin.flush()
+
+try:
+    process.wait({timeout})
+except Exception as e:
+    process.terminate()
+
+for line in iter(process.stdout.readline, ""):
+    print(line)
+    
+for line in iter(process.stderr.readline, ""):
+    if line.startswith(">>>"):
+        continue
+    print(line)
+"""
+    return runner
+
+
+def _run_cmd(cmd: [str], work_dir: str, timeout: int) -> str:
+    if WIN32:
+        logger.warning("SIGALRM is not supported on Windows. No timeout will be enforced.")
+        result = subprocess.run(
+            cmd,
+            cwd=work_dir,
+            capture_output=True,
+            text=True,
+        )
+    else:
+        with ThreadPoolExecutor(max_workers=1) as executor:
+            future = executor.submit(
+                subprocess.run,
+                cmd,
+                cwd=work_dir,
+                capture_output=True,
+                text=True,
+            )
+            result = future.result(timeout=timeout)
+    return result
+
+
+def execute_code(
+        code: Optional[str] = None,
+        timeout: Optional[int] = None,
+        filename: Optional[str] = None,
+        work_dir: Optional[str] = None,
+        use_docker: Optional[Union[List[str], str, bool]] = None,
+        lang: Optional[str] = "python",
+) -> Tuple[int, str]:
+    """Execute code in a docker container.
+    This function is not tested on MacOS.
+
+    Args:
+        code (Optional, str): The code to execute.
+            If None, the code from the file specified by filename will be executed.
+            Either code or filename must be provided.
+        timeout (Optional, int): The maximum execution time in seconds.
+            If None, a default timeout will be used. The default timeout is 600 seconds. On Windows, the timeout is not enforced when use_docker=False.
+        filename (Optional, str): The file name to save the code or where the code is stored when `code` is None.
+            If None, a file with a randomly generated name will be created.
+            The randomly generated file will be deleted after execution.
+            The file name must be a relative path. Relative paths are relative to the working directory.
+        work_dir (Optional, str): The working directory for the code execution.
+            If None, a default working directory will be used.
+            The default working directory is the "extensions" directory under
+            "path_to_autogen".
+        use_docker (Optional, list, str or bool): The docker image to use for code execution.
+            If a list or a str of image name(s) is provided, the code will be executed in a docker container
+            with the first image successfully pulled.
+            If None, False or empty, the code will be executed in the current environment.
+            Default is None, which will be converted into an empty list when docker package is available.
+            Expected behaviour:
+                - If `use_docker` is explicitly set to True and the docker package is available, the code will run in a Docker container.
+                - If `use_docker` is explicitly set to True but the Docker package is missing, an error will be raised.
+                - If `use_docker` is not set (i.e., left default to None) and the Docker package is not available, a warning will be displayed, but the code will run natively.
+            If the code is executed in the current environment,
+            the code must be trusted.
+        lang (Optional, str): The language of the code. Default is "python".
+
+    Returns:
+        int: 0 if the code executes successfully.
+        str: The error message if the code fails to execute; the stdout otherwise.
+    """
+    if all((code is None, filename is None)):
+        error_msg = f"Either {code=} or {filename=} must be provided."
+        logger.error(error_msg)
+        raise AssertionError(error_msg)
+
+    # Warn if use_docker was unspecified (or None), and cannot be provided (the default).
+    # In this case the current behavior is to fall back to run natively, but this behavior
+    # is subject to change.
+    if use_docker is None:
+        if docker is None:
+            use_docker = False
+            logger.warning(
+                "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change"
+            )
+        else:
+            # Default to true
+            use_docker = True
+
+    timeout = timeout or DEFAULT_TIMEOUT
+    original_filename = filename
+    if WIN32 and lang in ["sh", "shell"] and (not use_docker):
+        lang = "ps1"
+    if filename is None:
+        code_hash = md5(code.encode()).hexdigest()
+        # create a file with a automatically generated name
+        filename = f"tmp_code_{code_hash}.{'py' if lang.startswith('python') else lang}"
+    if work_dir is None:
+        WORKING_DIR = os.path.join(AIStorage.get_instance().get_myai_dir(), "tmp_code")
+        pathlib.Path(WORKING_DIR).mkdir(exist_ok=True)
+        work_dir = os.path.join(WORKING_DIR, code_hash)
+        pathlib.Path(work_dir).mkdir(exist_ok=True)
+    filepath = os.path.join(work_dir, filename)
+    file_dir = os.path.dirname(filepath)
+    os.makedirs(file_dir, exist_ok=True)
+    if code is not None:
+        write_requirements(code, os.path.join(file_dir, "requirements.txt"))
+        code = create_runner(code, 30)
+        with open(filepath, "w", encoding="utf-8") as fout:
+            fout.write(code)
+
+
+    # check if already running in a docker container
+    in_docker_container = os.path.exists("/.dockerenv")
+    if not use_docker or in_docker_container:
+        try:
+            env_cmd = ["python", "-m", "venv", os.path.join(file_dir, "venv")]
+            _run_cmd(env_cmd, file_dir, timeout)
+            if WIN32:
+                venv_path = os.path.join(file_dir, "venv", "Scripts")
+            else:
+                venv_path = os.path.join(file_dir, "venv", "bin")
+            pip_cmd = [os.path.join(venv_path, "python"), "-m", "pip", "install", "-r", "requirements.txt"]
+            _run_cmd(pip_cmd, file_dir, timeout)
+            # already running in a docker container
+            cmd = [
+                os.path.join(venv_path, "python"),
+                f".\\{filename}" if WIN32 else filename,
+            ]
+            result = _run_cmd(cmd, file_dir, timeout)
+        except TimeoutError:
+            if original_filename is None:
+                shutil.rmtree(os.path.join(file_dir, "venv"))
+                os.remove(filepath)
+                os.remove(os.path.join(file_dir, "requirements.txt"))
+                try:
+                    os.removedirs(file_dir)
+                except Exception:
+                    pass
+            return 1, TIMEOUT_MSG
+        if original_filename is None:
+            shutil.rmtree(os.path.join(file_dir, "venv"))
+            os.remove(filepath)
+            os.remove(os.path.join(file_dir, "requirements.txt"))
+            try:
+                os.removedirs(file_dir)
+            except Exception:
+                pass
+        if result.returncode:
+            logs = result.stderr
+            if original_filename is None:
+                abs_path = str(pathlib.Path(filepath).absolute())
+                logs = logs.replace(str(abs_path), "").replace(filename, "")
+            else:
+                abs_path = str(pathlib.Path(work_dir).absolute()) + PATH_SEPARATOR
+                logs = logs.replace(str(abs_path), "")
+        else:
+            logs = result.stdout
+        return result.returncode, logs
+
+    # create a docker client
+    client = docker.from_env()
+    image_list = (
+        ["python:3-alpine", "python:3", "python:3-windowsservercore"]
+        if use_docker is True
+        else [use_docker]
+        if isinstance(use_docker, str)
+        else use_docker
+    )
+    for image in image_list:
+        # check if the image exists
+        try:
+            client.images.get(image)
+            break
+        except docker.errors.ImageNotFound:
+            # pull the image
+            logger.info("Pulling image", image)
+            try:
+                client.images.pull(image, stream=True, decode=True)
+                break
+            except docker.errors.DockerException as e:
+                logger.error("Failed to pull image", image)
+                logger.exception(e)
+    # get a randomized str based on current time to wrap the exit code
+    exit_code_str = f"exitcode{time.time()}"
+    start_str = f'start{time.time()}'
+    abs_path = pathlib.Path(work_dir).absolute()
+    cmd = [
+        "sh",
+        "-c",
+        f"pip install --quiet -r requirements.txt; echo -n {start_str}; {_cmd(lang)} {filename}; exit_code=$?; echo -n {exit_code_str}; echo -n $exit_code; echo {exit_code_str};",
+    ]
+    # create a docker container
+    container = client.containers.run(
+        image,
+        command=cmd,
+        working_dir="/workspace",
+        detach=True,
+        # get absolute path to the working directory
+        volumes={abs_path: {"bind": "/workspace", "mode": "rw"}},
+    )
+    start_time = time.time()
+    while container.status != "exited" and time.time() - start_time < timeout:
+        # Reload the container object
+        container.reload()
+    if container.status != "exited":
+        container.stop()
+        container.remove()
+        if original_filename is None:
+            os.remove(filepath)
+        return 1, TIMEOUT_MSG, image
+    # get the container logs
+    logs: str = container.logs().decode("utf-8").rstrip()
+    start_pos = logs.find(start_str)
+    if start_pos != -1:
+        logs = logs[start_pos + len(start_str):]
+    # # commit the image
+    # tag = filename.replace("/", "")
+    # container.commit(repository="python", tag=tag)
+    # remove the container
+    container.remove()
+    # check if the code executed successfully
+    exit_code = container.attrs["State"]["ExitCode"]
+    if exit_code == 0:
+        # extract the exit code from the logs
+        pattern = re.compile(f"{exit_code_str}(\\d+){exit_code_str}")
+        match = pattern.search(logs)
+        exit_code = 1 if match is None else int(match.group(1))
+        # remove the exit code from the logs
+        logs = logs if match is None else pattern.sub("", logs)
+
+    if original_filename is None:
+        os.remove(filepath)
+        os.remove(os.path.join(file_dir, "requirements.txt"))
+        os.removedirs(file_dir)
+    if exit_code:
+        logs = logs.replace(f"/workspace/{filename if original_filename is None else ''}", "")
+    # return the exit code, logs and image
+    return exit_code, logs
+

src/aios/environment/code_interpreter_function.py

@@ -0,0 +1,41 @@
+from typing import Dict
+
+from ..agent.ai_function import AIFunction
+from .code_interpreter import execute_code
+
+
+class CodeInterpreterFunction(AIFunction):
+    def __init__(self):
+        self.func_id = "code_interpreter"
+        self.description = "execute python code"
+
+    def get_name(self) -> str:
+        return self.func_id
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {
+            "type": "object",
+            "properties": {
+                "code": {"type": "string", "description": "python code"}
+            }
+        }
+
+    async def execute(self, **kwargs) -> str:
+        code = kwargs.get("code")
+        ret_code, result = execute_code(code=code)
+        if ret_code == 0:
+            return result.strip()
+        else:
+            return result.strip()
+
+    def is_local(self) -> bool:
+        return True
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False

src/aios/environment/duckduckgo_text_search_function.py

@@ -0,0 +1,52 @@
+import json
+from typing import Dict
+
+from ..agent.ai_function import AIFunction
+from duckduckgo_search import AsyncDDGS
+
+
+class DuckDuckGoTextSearchFunction(AIFunction):
+    def __init__(self):
+        self.name = "duckduckgo_text_search"
+        self.description = "Search text from duckduckgo.com"
+        self.region = "wt-wt"
+        self.safesearch = "moderate"
+        self.time = "y"
+        self.max_results = 5
+
+    def get_name(self) -> str:
+        return self.name
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {"type": "object",
+                "properties": {
+                    "query": {"type": "string", "description": "The query to search for."}
+                }
+                }
+
+    async def execute(self, **kwargs) -> str:
+        query = kwargs.get("query")
+
+        async with AsyncDDGS() as ddgs:
+            results = [r async for r in ddgs.text(
+                query,
+                region=self.region,
+                safesearch=self.safesearch,
+                timelimit=self.time,
+                backend="api",
+                max_results=self.max_results
+            )]
+
+            return json.dumps(results)
+
+    def is_local(self) -> bool:
+        return True
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False

src/aios/environment/environment.py

@@ -0,0 +1,148 @@
+# basic environment class
+# we have some built-in environment: Calender(include timer),Home(connect to IoT device in your home), ,KnwoledgeBase,FileSystem,
+
+from abc import ABC, abstractmethod
+from typing import Any, Callable, Optional,Dict,Awaitable,List
+import logging
+
+from ..agent.ai_function import AIFunction
+
+logger = logging.getLogger(__name__)
+
+class EnvironmentEvent(ABC):
+    @abstractmethod
+    def display(self) -> str:
+        pass
+
+EnvironmentEventHandler = Callable[[str,EnvironmentEvent],Awaitable[Any]]
+
+class Environment:
+    _all_env = {}
+    @classmethod
+    def get_env_by_id(cls,env_id:str):
+        return cls._all_env.get(env_id)
+
+    @classmethod
+    def set_env_by_id(cls,id,env):
+        assert id == env.get_id()
+        cls._all_env[env.get_id()] = env
+
+    def __init__(self,env_id:str) -> None:
+        self.env_id = env_id
+        self.values:Dict[str,str] = {}
+        self.get_handlers:Dict[str,Callable] = {}
+        self.owner_env:Dict[str,Environment] = {}
+        # self.valid_keys:Dict[str,bool] = None
+        self.event_handlers:Dict[str,List[EnvironmentEventHandler]]= {}
+
+        self.functions : Dict[str,AIFunction] = {}
+
+    def get_id(self) -> str:
+        return self.env_id
+
+    def add_owner_env(self,env) -> None:
+        self.owner_env[env.get_id()] = env
+
+    #@abstractmethod
+    #TODO: how to use env? different env has different prompt
+    def get_env_prompt(self) -> str:
+       pass
+
+    def add_ai_function(self,func:AIFunction) -> None:
+        if self.functions.get(func.get_name()) is not None:
+            logger.warn(f"add ai_function {func.get_name()} in env {self.env_id}:function already exist")
+
+        self.functions[func.get_name()] = func
+
+    def get_ai_function(self,func_name:str) -> AIFunction:
+        func = self.functions.get(func_name)
+        if func is not None:
+            return func
+
+        for owner_env in self.owner_env.values():
+            func = owner_env.get_ai_function(func_name)
+            if func is not None:
+                return func
+
+        return None
+
+    #def enable_ai_function(self,func_name:str) -> None:
+    #    pass
+
+    #def disable_ai_function(self,func_name:str) -> None:
+    #    pass
+
+    def get_all_ai_functions(self) -> List[AIFunction]:
+        func_list = []
+        func_list.extend(self.functions.values())
+        for owner_env in self.owner_env.values():
+            func_list.extend(owner_env.get_all_ai_functions())
+        return func_list
+
+    @abstractmethod
+    def _do_get_value(self,key:str) -> Optional[str]:
+        pass
+
+    def register_get_handler(self,key:str,handler:Callable) -> None:
+        h = self.get_handlers.get(key)
+        if h is not None:
+            logger.warn(f"register get_handler {key} in env {self.env_id}:handler already exist")
+
+        self.get_handlers[key] = handler
+
+
+    def attach_event_handler(self,event_id:str,handler:Callable) -> None:
+        handler_list = self.event_handlers.get(event_id)
+        if handler_list is None:
+            handler_list = []
+            self.event_handlers[event_id] = handler_list
+
+        handler_list.append(handler)
+
+    def remove_event_handler(self,event_id:str,handler:Callable) -> None:
+        handler_list = self.event_handlers.get(event_id)
+        if handler is not None:
+            handler_list.remove(handler)
+            return
+
+        logger.warn(f"remove event_handler {event_id} in env {self.env_id}:handler not found")
+
+    async def fire_event(self,event_id:str,event:EnvironmentEvent) -> None:
+        handler_list = self.event_handlers.get(event_id)
+        if handler_list is not None:
+            for handler in handler_list:
+                await handler(self.env_id,event)
+        else:
+            logger.debug(f"fire event {event_id} in env {self.env_id}:handler not found")
+        return
+
+    def __getitem__(self, key):
+        return self.get_value(key)
+
+    def get_value(self,key:str) -> Optional[str]:
+        handler = self.get_handlers.get(key)
+        if handler is not None:
+            return handler()
+
+        s = self.values.get(key)
+        if isinstance(s,str):
+            return s
+        else:
+            logger.warn(f"get value {key} in env {self.env_id} failed!,type is not str")
+
+        s = self._do_get_value(key)
+        if s is not None:
+            return s
+        if self.owner_env is not None:
+            for env in self.owner_env.values():
+                s = env.get_value(key)
+                if s is not None:
+                    return s
+
+        logger.warn(f"get value {key} in env {self.env_id} failed!,not found")
+        return None
+
+    def set_value(self, key: str, str_value: str,is_storage:bool = True):
+        logger.info(f"set value {key} in env {self.env_id} to {str_value}")
+        self.values[key] = str_value
+

src/aios/environment/image_2_text_function.py

@@ -0,0 +1,46 @@
+import logging
+from typing import Dict
+
+from ..frame.compute_kernel import ComputeKernel
+from ..agent.ai_function import AIFunction
+
+logger = logging.getLogger(__name__)
+
+class Image2TextFunction(AIFunction):
+    def __init__(self):
+        self.func_id = "image_2_text"
+        self.description = "According to the input image file address, return the description of the image content"
+        logger.info(f"init Image2TextFunction")
+
+    def get_name(self) -> str:
+        return self.func_id
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {
+        }
+
+    async def execute(self, **kwargs) -> str:
+        logger.info(f"execute image_2_text function: {kwargs}")
+        image_path = kwargs.get("image_path")
+        data = await ComputeKernel.get_instance().do_image_2_text(image_path, '')
+        try:
+            result = data['message']['choices'][0]['message']['content']
+        except (KeyError, TypeError, IndexError):
+            logger.error(f"image_2_text error: {data}")
+            result = ""
+        return result
+
+
+    def is_local(self) -> bool:
+        return False
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False
+
+

src/aios/environment/script_to_speech_function.py

@@ -0,0 +1,109 @@
+import io
+import logging
+import os
+import random
+from pathlib import Path
+from typing import Dict
+
+from ..agent.ai_function import AIFunction
+from ..frame.compute_kernel import ComputeKernel
+from ..storage.storage import AIStorage
+
+
+from pydub import AudioSegment
+
+logger = logging.getLogger(__name__)
+
+
+class ScriptToSpeechFunction(AIFunction):
+    def __init__(self):
+        self.func_id = "script_to_speech"
+        self.description = "根据输入的剧本生成音频文件，成功时会返回音频文件路径"
+        self.speech_path = os.path.join(AIStorage.get_instance().get_myai_dir(), "tts")
+        Path(self.speech_path).mkdir(exist_ok=True)
+
+    def get_name(self) -> str:
+        return self.func_id
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {
+            "type": "object",
+            "properties": {
+                "language": {"type": "string", "description": "演播语言", "enum": ["zh", "en"]},
+                "model": {"type": "string", "description": "演播模型", "enum": ["tts-1", "tts-1-hd"]},
+                "roles": {"type": "array", "items": {
+                    "type": "object",
+                    "properties": {
+                        "name": {"type": "string", "description": "角色名字"},
+                        "gender": {"type": "string", "description": "角色性别", "enum": ["man", "female"]},
+                        "age": {"type": "string", "description": "年龄", "enum": ["child", "adult"]},
+                    }}},
+                "lines": {"type": "array", "items": {
+                    "type": "object",
+                    "properties": {
+                        "name": {"type": "string", "description": "角色名字"},
+                        "tone": {"type": "string", "description": "演播情感",
+                                 "enum": ["happy", "sad", "angry", "fear", "disgust", "surprise", "neutral"]},
+                        "text": {"type": "string", "description": "台词"},
+                    }
+                }}
+            }
+        }
+
+    async def execute(self, **kwargs) -> str:
+        logger.info(f"execute text_to_speech function: {kwargs}")
+
+        language = kwargs.get("language")
+        if language is None:
+            language = "zh"
+        model = kwargs.get("model")
+        roles = kwargs.get("roles")
+        lines = kwargs.get("lines")
+
+        audio = None
+        for line in lines:
+            name = line.get("name")
+            tone = line.get("tone")
+            text = line.get("text")
+            gender = None
+            age = None
+            for role in roles:
+                role_name = role.get("name")
+                if role_name == name:
+                    gender = role.get("gender")
+                    age = role.get("age")
+                    break
+            i = 0
+            while i < 3:
+                try:
+                    data = await ComputeKernel.get_instance().do_text_to_speech(text, language, gender, age, name, tone, model_name=model)
+                    if audio is None:
+                        audio = AudioSegment.from_mp3(io.BytesIO(data))
+                    else:
+                        audio = audio + AudioSegment.from_mp3(io.BytesIO(data))
+                    break
+                except Exception as e:
+                    logger.error(f"do_text_to_speech failed: {e}")
+                    i += 1
+                    continue
+
+        if audio is not None:
+            path = os.path.join(self.speech_path, "{}.mp3".format(''.join(random.sample('zyxwvutsrqponmlkjihgfedcba', 10))))
+            audio.export(path, format="mp3")
+            return "exec text_to_speech OK，speech file store at ```{}```".format(path)
+        else:
+            return "exec text_to_speech failed"
+
+    def is_local(self) -> bool:
+        return True
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False
+
+

src/aios/environment/simple_kb_db.py

@@ -0,0 +1,228 @@
+
+import sqlite3
+import json
+import threading
+import logging
+from datetime import datetime
+
+from typing import Optional, List
+
+logger = logging.getLogger(__name__)
+
+class SimpleKnowledgeDB:
+    def __init__(self,db_path:str):
+        self.db_path = db_path
+        self._get_conn()
+        
+    def _get_conn(self):
+        """ get db connection """
+        local = threading.local()
+        if not hasattr(local, 'conn'):
+            local.conn = self._create_connection(self.db_path)
+        return local.conn
+
+
+    def _create_connection(self, db_file):
+        """ create a database connection to a SQLite database """
+        conn = None
+        try:
+            conn = sqlite3.connect(db_file)
+        except Exception as e:
+            logger.error("Error occurred while connecting to database: %s", e)
+            return None
+
+        if conn:
+            self._create_tables(conn)
+
+        return conn
+    
+    def _create_tables(self,conn):
+        cursor = conn.cursor()
+        cursor.execute('''
+            CREATE TABLE IF NOT EXISTS documents (
+                doc_path TEXT PRIMARY KEY,
+                length INTEGER,
+                last_modify TEXT,
+                doc_hash TEXT,
+                create_time TEXT
+            )
+        ''')
+        cursor.execute('''
+            CREATE TABLE IF NOT EXISTS knowledge (
+                doc_hash TEXT PRIMARY KEY,
+                title TEXT,
+                summary TEXT,
+                content TEXT,
+                catalogs TEXT,
+                tags TEXT,
+                llm_title TEXT,
+                llm_summary TEXT,
+                create_time TEXT
+            )
+        ''')
+            
+        cursor.execute('''
+            CREATE INDEX IF NOT EXISTS idx_documents_doc_hash
+            ON documents (doc_hash)
+        ''')
+
+        cursor.execute('''
+            CREATE INDEX IF NOT EXISTS idx_knowledge_tags
+            ON knowledge (tags)
+        ''')
+
+        conn.commit()
+
+    def add_doc(self, doc_path: str, length: int, last_modify: str, doc_hash: Optional[str] = None):
+        conn = self._get_conn()
+        cursor = conn.cursor()
+        create_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        cursor.execute('''
+            INSERT INTO documents (doc_path, length, last_modify, doc_hash,create_time) 
+            VALUES (?, ?, ?, ?,?)
+        ''', (doc_path, length, last_modify, doc_hash,create_time))
+        conn.commit()
+
+    def is_doc_exist(self, doc_path: str) -> bool:
+        conn = self._get_conn()
+        cursor = conn.cursor()
+        cursor.execute('''
+            SELECT doc_path
+            FROM documents
+            WHERE doc_path = ?
+        ''', (doc_path,))
+        return len(cursor.fetchall()) > 0
+
+    def set_doc_hash(self, doc_path: str, doc_hash: str):
+        conn = self._get_conn()
+        cursor = conn.cursor()
+        cursor.execute('''
+            UPDATE documents
+            SET doc_hash = ?
+            WHERE doc_path = ?
+        ''', (doc_hash, doc_path))
+        conn.commit()
+    
+    def get_docs_without_hash(self,limit:int=1024) -> List[str]:
+        conn = self._get_conn()
+        cursor = conn.cursor()
+        cursor.execute('''
+            SELECT doc_path
+            FROM documents
+            WHERE doc_hash IS NULL OR doc_hash = ''
+            ORDER BY create_time DESC
+            LIMIT ?
+        ''',(limit,))
+        return [row[0] for row in cursor.fetchall()]
+
+    #metadata["summary"]
+    #metadata["catelogs"]
+    #metadata["tags"]
+    def add_knowledge(self, doc_hash: str, title: str, metadata: dict,content:str = None,):
+        conn = self._get_conn()
+        cursor = conn.cursor()
+
+        create_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        summary = metadata.get("summary", "")
+        catalogs = metadata.get("catalogs","")
+        tags = ','.join(metadata.get("tags", []))
+
+        cursor.execute('''
+            INSERT INTO knowledge (doc_hash, title , summary , catalogs , tags,create_time) 
+            VALUES (?, ?, ?, ?, ?,?)
+        ''', (doc_hash, title, summary, catalogs, tags,create_time))
+        conn.commit()
+
+    #llm_result["summary"]
+    #llm_result["tags"]
+    #llm_result["catelog"]
+    def set_knowledge_llm_result(self, doc_hash: str, llm_result: dict):
+        conn = self._get_conn()
+        cursor = conn.cursor()
+
+        title = llm_result.get("title", "")
+        summary = llm_result.get("summary", "")
+        catalogs = json.dumps(llm_result.get("catalogs", {}))
+        tags = ','.join(llm_result.get("tags", []))
+
+        cursor.execute('''
+            UPDATE knowledge
+            SET llm_title = ?,llm_summary = ?, catalogs = ?, tags = ?
+            WHERE doc_hash = ?
+        ''', (title,summary, catalogs, tags, doc_hash))
+        conn.commit()
+
+    def get_hash_by_doc_path(self, doc_path: str) -> Optional[str]:
+        conn = self._get_conn()
+        cursor = conn.cursor()
+        cursor.execute('''
+            SELECT doc_hash
+            FROM documents
+            WHERE doc_path = ?
+        ''', (doc_path,))
+        row = cursor.fetchone()
+        if row is None:
+            return None
+        return row[0]
+
+    def get_knowledge(self, doc_hash: str) -> Optional[dict]:
+        conn = self._get_conn()
+        cursor = conn.cursor()
+        cursor.execute('''
+            SELECT title, summary, catalogs, tags, llm_title, llm_summary
+            FROM knowledge
+            WHERE doc_hash = ?
+        ''', (doc_hash,))
+        row = cursor.fetchone()
+        if row is None:
+            return None
+        
+        # get doc path
+        cursor.execute('''
+            SELECT doc_path
+            FROM documents
+            WHERE doc_hash = ?
+        ''', (doc_hash,))
+        row2 = cursor.fetchone()
+        if row2 is None:
+            return None
+        doc_path = row2[0]
+    
+
+        return {
+            "full_path": doc_path,
+            "title": row[0],
+            "summary": row[1],
+            "catalogs": row[2],
+            "tags": row[3],
+            "llm_title" : row[4],
+            "llm_summary" : row[5],
+        }
+
+    def get_knowledge_without_llm_title(self,limit:int=16) -> List[str]:
+        conn = self._get_conn()
+        cursor = conn.cursor()
+        cursor.execute('''
+            SELECT doc_hash
+            FROM knowledge
+            WHERE llm_title IS NULL OR llm_title = ''
+            ORDER BY create_time DESC
+            LIMIT ?
+        ''',(limit,))
+        return [row[0] for row in cursor.fetchall()]
+
+    def query_docs_by_tag(self, tag: str) -> List[str]:
+        conn = self._get_conn()
+        cursor = conn.cursor()
+        tag_json = json.dumps(tag)  # 将标签转换为 JSON 字符串
+        cursor.execute('''
+            SELECT documents.doc_path
+            FROM documents
+            JOIN knowledge ON documents.doc_hash = knowledge.doc_hash
+            WHERE json_extract(knowledge.tags, '$') LIKE ?
+        ''', (tag))
+        return [row[0] for row in cursor.fetchall()]
+    
+    def query(self,sql:str):
+        pass
+        #cursor = self.conn.cursor()

src/aios/environment/sql_database.py

@@ -0,0 +1,493 @@
+"""
+Taken from: langchain
+SQLAlchemy wrapper around a database.
+"""
+from __future__ import annotations
+import os
+
+import warnings
+from typing import Any, Dict, Iterable, List, Literal, Optional, Sequence, Union
+
+import sqlalchemy
+from sqlalchemy import MetaData, Table, create_engine, inspect, select, text
+from sqlalchemy.engine import Engine
+from sqlalchemy.exc import ProgrammingError, SQLAlchemyError
+from sqlalchemy.schema import CreateTable
+from sqlalchemy.types import NullType
+
+
+def get_from_env(key: str, env_key: str, default: Optional[str] = None) -> str:
+    """Get a value from a dictionary or an environment variable."""
+    if env_key in os.environ and os.environ[env_key]:
+        return os.environ[env_key]
+    elif default is not None:
+        return default
+    else:
+        raise ValueError(
+            f"Did not find {key}, please add an environment variable"
+            f" `{env_key}` which contains it, or pass"
+            f"  `{key}` as a named parameter."
+        )
+
+
+def _format_index(index: sqlalchemy.engine.interfaces.ReflectedIndex) -> str:
+    return (
+        f'Name: {index["name"]}, Unique: {index["unique"]},'
+        f' Columns: {str(index["column_names"])}'
+    )
+
+
+def truncate_word(content: Any, *, length: int, suffix: str = "...") -> str:
+    """
+    Truncate a string to a certain number of words, based on the max string
+    length.
+    """
+
+    if not isinstance(content, str) or length <= 0:
+        return content
+
+    if len(content) <= length:
+        return content
+
+    return content[: length - len(suffix)].rsplit(" ", 1)[0] + suffix
+
+
+class SQLDatabase:
+    """SQLAlchemy wrapper around a database."""
+
+    def __init__(
+            self,
+            engine: Engine,
+            schema: Optional[str] = None,
+            metadata: Optional[MetaData] = None,
+            ignore_tables: Optional[List[str]] = None,
+            include_tables: Optional[List[str]] = None,
+            sample_rows_in_table_info: int = 3,
+            indexes_in_table_info: bool = False,
+            custom_table_info: Optional[dict] = None,
+            view_support: bool = False,
+            max_string_length: int = 300,
+    ):
+        """Create engine from database URI."""
+        self._engine = engine
+        self._schema = schema
+        if include_tables and ignore_tables:
+            raise ValueError("Cannot specify both include_tables and ignore_tables")
+
+        self._inspector = inspect(self._engine)
+
+        # including view support by adding the views as well as tables to the all
+        # tables list if view_support is True
+        self._all_tables = set(
+            self._inspector.get_table_names(schema=schema)
+            + (self._inspector.get_view_names(schema=schema) if view_support else [])
+        )
+
+        self._include_tables = set(include_tables) if include_tables else set()
+        if self._include_tables:
+            missing_tables = self._include_tables - self._all_tables
+            if missing_tables:
+                raise ValueError(
+                    f"include_tables {missing_tables} not found in database"
+                )
+        self._ignore_tables = set(ignore_tables) if ignore_tables else set()
+        if self._ignore_tables:
+            missing_tables = self._ignore_tables - self._all_tables
+            if missing_tables:
+                raise ValueError(
+                    f"ignore_tables {missing_tables} not found in database"
+                )
+        usable_tables = self.get_usable_table_names()
+        self._usable_tables = set(usable_tables) if usable_tables else self._all_tables
+
+        if not isinstance(sample_rows_in_table_info, int):
+            raise TypeError("sample_rows_in_table_info must be an integer")
+
+        self._sample_rows_in_table_info = sample_rows_in_table_info
+        self._indexes_in_table_info = indexes_in_table_info
+
+        self._custom_table_info = custom_table_info
+        if self._custom_table_info:
+            if not isinstance(self._custom_table_info, dict):
+                raise TypeError(
+                    "table_info must be a dictionary with table names as keys and the "
+                    "desired table info as values"
+                )
+            # only keep the tables that are also present in the database
+            intersection = set(self._custom_table_info).intersection(self._all_tables)
+            self._custom_table_info = dict(
+                (table, self._custom_table_info[table])
+                for table in self._custom_table_info
+                if table in intersection
+            )
+
+        self._max_string_length = max_string_length
+
+        self._metadata = metadata or MetaData()
+        # including view support if view_support = true
+        self._metadata.reflect(
+            views=view_support,
+            bind=self._engine,
+            only=list(self._usable_tables),
+            schema=self._schema,
+        )
+
+    @classmethod
+    def from_uri(
+            cls, database_uri: str, engine_args: Optional[dict] = None, **kwargs: Any
+    ) -> SQLDatabase:
+        """Construct a SQLAlchemy engine from URI."""
+        _engine_args = engine_args or {}
+        return cls(create_engine(database_uri, **_engine_args), **kwargs)
+
+    @classmethod
+    def from_databricks(
+            cls,
+            catalog: str,
+            schema: str,
+            host: Optional[str] = None,
+            api_token: Optional[str] = None,
+            warehouse_id: Optional[str] = None,
+            cluster_id: Optional[str] = None,
+            engine_args: Optional[dict] = None,
+            **kwargs: Any,
+    ) -> SQLDatabase:
+        """
+        Class method to create an SQLDatabase instance from a Databricks connection.
+        This method requires the 'databricks-sql-connector' package. If not installed,
+        it can be added using `pip install databricks-sql-connector`.
+
+        Args:
+            catalog (str): The catalog name in the Databricks database.
+            schema (str): The schema name in the catalog.
+            host (Optional[str]): The Databricks workspace hostname, excluding
+                'https://' part. If not provided, it attempts to fetch from the
+                environment variable 'DATABRICKS_HOST'. If still unavailable and if
+                running in a Databricks notebook, it defaults to the current workspace
+                hostname. Defaults to None.
+            api_token (Optional[str]): The Databricks personal access token for
+                accessing the Databricks SQL warehouse or the cluster. If not provided,
+                it attempts to fetch from 'DATABRICKS_TOKEN'. If still unavailable
+                and running in a Databricks notebook, a temporary token for the current
+                user is generated. Defaults to None.
+            warehouse_id (Optional[str]): The warehouse ID in the Databricks SQL. If
+                provided, the method configures the connection to use this warehouse.
+                Cannot be used with 'cluster_id'. Defaults to None.
+            cluster_id (Optional[str]): The cluster ID in the Databricks Runtime. If
+                provided, the method configures the connection to use this cluster.
+                Cannot be used with 'warehouse_id'. If running in a Databricks notebook
+                and both 'warehouse_id' and 'cluster_id' are None, it uses the ID of the
+                cluster the notebook is attached to. Defaults to None.
+            engine_args (Optional[dict]): The arguments to be used when connecting
+                Databricks. Defaults to None.
+            **kwargs (Any): Additional keyword arguments for the `from_uri` method.
+
+        Returns:
+            SQLDatabase: An instance of SQLDatabase configured with the provided
+                Databricks connection details.
+
+        Raises:
+            ValueError: If 'databricks-sql-connector' is not found, or if both
+                'warehouse_id' and 'cluster_id' are provided, or if neither
+                'warehouse_id' nor 'cluster_id' are provided and it's not executing
+                inside a Databricks notebook.
+        """
+        try:
+            from databricks import sql  # noqa: F401
+        except ImportError:
+            raise ValueError(
+                "databricks-sql-connector package not found, please install with"
+                " `pip install databricks-sql-connector`"
+            )
+        context = None
+        try:
+            from dbruntime.databricks_repl_context import get_context
+
+            context = get_context()
+        except ImportError:
+            pass
+
+        default_host = context.browserHostName if context else None
+        if host is None:
+            host = get_from_env("host", "DATABRICKS_HOST", default_host)
+
+        default_api_token = context.apiToken if context else None
+        if api_token is None:
+            api_token = get_from_env("api_token", "DATABRICKS_TOKEN", default_api_token)
+
+        if warehouse_id is None and cluster_id is None:
+            if context:
+                cluster_id = context.clusterId
+            else:
+                raise ValueError(
+                    "Need to provide either 'warehouse_id' or 'cluster_id'."
+                )
+
+        if warehouse_id and cluster_id:
+            raise ValueError("Can't have both 'warehouse_id' or 'cluster_id'.")
+
+        if warehouse_id:
+            http_path = f"/sql/1.0/warehouses/{warehouse_id}"
+        else:
+            http_path = f"/sql/protocolv1/o/0/{cluster_id}"
+
+        uri = (
+            f"databricks://token:{api_token}@{host}?"
+            f"http_path={http_path}&catalog={catalog}&schema={schema}"
+        )
+        return cls.from_uri(database_uri=uri, engine_args=engine_args, **kwargs)
+
+    @classmethod
+    def from_cnosdb(
+            cls,
+            url: str = "127.0.0.1:8902",
+            user: str = "root",
+            password: str = "",
+            tenant: str = "cnosdb",
+            database: str = "public",
+    ) -> SQLDatabase:
+        """
+        Class method to create an SQLDatabase instance from a CnosDB connection.
+        This method requires the 'cnos-connector' package. If not installed, it
+        can be added using `pip install cnos-connector`.
+
+        Args:
+            url (str): The HTTP connection host name and port number of the CnosDB
+                service, excluding "http://" or "https://", with a default value
+                of "127.0.0.1:8902".
+            user (str): The username used to connect to the CnosDB service, with a
+                default value of "root".
+            password (str): The password of the user connecting to the CnosDB service,
+                with a default value of "".
+            tenant (str): The name of the tenant used to connect to the CnosDB service,
+                with a default value of "cnosdb".
+            database (str): The name of the database in the CnosDB tenant.
+
+        Returns:
+            SQLDatabase: An instance of SQLDatabase configured with the provided
+            CnosDB connection details.
+        """
+        try:
+            from cnosdb_connector import make_cnosdb_langchain_uri
+
+            uri = make_cnosdb_langchain_uri(url, user, password, tenant, database)
+            return cls.from_uri(database_uri=uri)
+        except ImportError:
+            raise ValueError(
+                "cnos-connector package not found, please install with"
+                " `pip install cnos-connector`"
+            )
+
+    @property
+    def dialect(self) -> str:
+        """Return string representation of dialect to use."""
+        return self._engine.dialect.name
+
+    def get_usable_table_names(self) -> Iterable[str]:
+        """Get names of tables available."""
+        if self._include_tables:
+            return sorted(self._include_tables)
+        return sorted(self._all_tables - self._ignore_tables)
+
+    def get_table_names(self) -> Iterable[str]:
+        """Get names of tables available."""
+        warnings.warn(
+            "This method is deprecated - please use `get_usable_table_names`."
+        )
+        return self.get_usable_table_names()
+
+    @property
+    def table_info(self) -> str:
+        """Information about all tables in the database."""
+        return self.get_table_info()
+
+    def get_table_info(self, table_names: Optional[List[str]] = None) -> str:
+        """Get information about specified tables.
+
+        Follows best practices as specified in: Rajkumar et al, 2022
+        (https://arxiv.org/abs/2204.00498)
+
+        If `sample_rows_in_table_info`, the specified number of sample rows will be
+        appended to each table description. This can increase performance as
+        demonstrated in the paper.
+        """
+        all_table_names = self.get_usable_table_names()
+        if table_names is not None:
+            missing_tables = set(table_names).difference(all_table_names)
+            if missing_tables:
+                raise ValueError(f"table_names {missing_tables} not found in database")
+            all_table_names = table_names
+
+        meta_tables = [
+            tbl
+            for tbl in self._metadata.sorted_tables
+            if tbl.name in set(all_table_names)
+               and not (self.dialect == "sqlite" and tbl.name.startswith("sqlite_"))
+        ]
+
+        tables = []
+        for table in meta_tables:
+            if self._custom_table_info and table.name in self._custom_table_info:
+                tables.append(self._custom_table_info[table.name])
+                continue
+
+            # Ignore JSON datatyped columns
+            for k, v in table.columns.items():
+                if type(v.type) is NullType:
+                    table._columns.remove(v)
+
+            # add create table command
+            create_table = str(CreateTable(table).compile(self._engine))
+            table_info = f"{create_table.rstrip()}"
+            has_extra_info = (
+                    self._indexes_in_table_info or self._sample_rows_in_table_info
+            )
+            if has_extra_info:
+                table_info += "\n\n/*"
+            if self._indexes_in_table_info:
+                table_info += f"\n{self._get_table_indexes(table)}\n"
+            if self._sample_rows_in_table_info:
+                table_info += f"\n{self._get_sample_rows(table)}\n"
+            if has_extra_info:
+                table_info += "*/"
+            tables.append(table_info)
+        tables.sort()
+        final_str = "\n\n".join(tables)
+        return final_str
+
+    def _get_table_indexes(self, table: Table) -> str:
+        indexes = self._inspector.get_indexes(table.name)
+        indexes_formatted = "\n".join(map(_format_index, indexes))
+        return f"Table Indexes:\n{indexes_formatted}"
+
+    def _get_sample_rows(self, table: Table) -> str:
+        # build the select command
+        command = select(table).limit(self._sample_rows_in_table_info)
+
+        # save the columns in string format
+        columns_str = "\t".join([col.name for col in table.columns])
+
+        try:
+            # get the sample rows
+            with self._engine.connect() as connection:
+                sample_rows_result = connection.execute(command)  # type: ignore
+                # shorten values in the sample rows
+                sample_rows = list(
+                    map(lambda ls: [str(i)[:100] for i in ls], sample_rows_result)
+                )
+
+            # save the sample rows in string format
+            sample_rows_str = "\n".join(["\t".join(row) for row in sample_rows])
+
+        # in some dialects when there are no rows in the table a
+        # 'ProgrammingError' is returned
+        except ProgrammingError:
+            sample_rows_str = ""
+
+        return (
+            f"{self._sample_rows_in_table_info} rows from {table.name} table:\n"
+            f"{columns_str}\n"
+            f"{sample_rows_str}"
+        )
+
+    def _execute(
+            self,
+            command: str,
+            fetch: Union[Literal["all"], Literal["one"]] = "all",
+    ) -> Sequence[Dict[str, Any]]:
+        """
+        Executes SQL command through underlying engine.
+
+        If the statement returns no rows, an empty list is returned.
+        """
+        with self._engine.begin() as connection:
+            if self._schema is not None:
+                if self.dialect == "snowflake":
+                    connection.exec_driver_sql(
+                        "ALTER SESSION SET search_path = %s", (self._schema,)
+                    )
+                elif self.dialect == "bigquery":
+                    connection.exec_driver_sql("SET @@dataset_id=?", (self._schema,))
+                elif self.dialect == "mssql":
+                    pass
+                elif self.dialect == "trino":
+                    connection.exec_driver_sql("USE ?", (self._schema,))
+                elif self.dialect == "duckdb":
+                    # Unclear which parameterized argument syntax duckdb supports.
+                    # The docs for the duckdb client say they support multiple,
+                    # but `duckdb_engine` seemed to struggle with all of them:
+                    # https://github.com/Mause/duckdb_engine/issues/796
+                    connection.exec_driver_sql(f"SET search_path TO {self._schema}")
+                elif self.dialect == "oracle":
+                    connection.exec_driver_sql(
+                        f"ALTER SESSION SET CURRENT_SCHEMA = {self._schema}"
+                    )
+                else:  # postgresql and other compatible dialects
+                    connection.exec_driver_sql("SET search_path TO %s", (self._schema,))
+            cursor = connection.execute(text(command))
+            if cursor.returns_rows:
+                if fetch == "all":
+                    result = [x._asdict() for x in cursor.fetchall()]
+                elif fetch == "one":
+                    first_result = cursor.fetchone()
+                    result = [] if first_result is None else [first_result._asdict()]
+                else:
+                    raise ValueError("Fetch parameter must be either 'one' or 'all'")
+                return result
+        return []
+
+    def run(
+            self,
+            command: str,
+            fetch: Union[Literal["all"], Literal["one"]] = "all",
+    ) -> str:
+        """Execute a SQL command and return a string representing the results.
+
+        If the statement returns rows, a string of the results is returned.
+        If the statement returns no rows, an empty string is returned.
+        """
+        result = self._execute(command, fetch)
+        # Convert columns values to string to avoid issues with sqlalchemy
+        # truncating text
+        res = [
+            tuple(truncate_word(c, length=self._max_string_length) for c in r.values())
+            for r in result
+        ]
+        if not res:
+            return ""
+        else:
+            return str(res)
+
+    def get_table_info_no_throw(self, table_names: Optional[List[str]] = None) -> str:
+        """Get information about specified tables.
+
+        Follows best practices as specified in: Rajkumar et al, 2022
+        (https://arxiv.org/abs/2204.00498)
+
+        If `sample_rows_in_table_info`, the specified number of sample rows will be
+        appended to each table description. This can increase performance as
+        demonstrated in the paper.
+        """
+        try:
+            return self.get_table_info(table_names)
+        except ValueError as e:
+            """Format the error message"""
+            return f"Error: {e}"
+
+    def run_no_throw(
+            self,
+            command: str,
+            fetch: Union[Literal["all"], Literal["one"]] = "all",
+    ) -> str:
+        """Execute a SQL command and return a string representing the results.
+
+        If the statement returns rows, a string of the results is returned.
+        If the statement returns no rows, an empty string is returned.
+
+        If the statement throws an error, the error message is returned.
+        """
+        try:
+            return self.run(command, fetch)
+        except SQLAlchemyError as e:
+            """Format the error message"""
+            return f"Error: {e}"

src/aios/environment/sql_database_function.py

@@ -0,0 +1,112 @@
+from datetime import timedelta, datetime
+from typing import Dict
+
+from cachetools import TLRUCache, cached
+
+from ..agent.ai_function import AIFunction
+from .sql_database import SQLDatabase, get_from_env
+
+
+def _my_ttu(_key, _value, now):
+    return now + timedelta(seconds=600)
+
+
+database_cache = TLRUCache(ttu=_my_ttu, maxsize=10000, timer=datetime.now)
+
+
+@cached(cache=database_cache)
+def get_database(uri: str) -> SQLDatabase:
+    return SQLDatabase.from_uri(uri)
+
+
+class GetTableInfosFunction(AIFunction):
+    def __init__(self):
+        super().__init__()
+        self.name = "get_table_infos"
+        self.description = "Get table informations in the database"
+
+    def get_name(self) -> str:
+        return self.name
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {
+            "type": "object",
+            "properties": {
+                "database_url": {"type": "string", "description": "Database URL,Can be set to None"},
+            }
+        }
+
+    async def execute(self, **kwargs) -> str:
+        database_url: str = kwargs.get("database_url")
+        if (database_url is None
+                or database_url.strip() == ""
+                or database_url.strip().lower() == "none"
+                or database_url.strip().lower() == "null"):
+            database_url = get_from_env(key="database url", env_key="DATABASE_URL")
+            if database_url is None:
+                return "error: database_url is None"
+        database = get_database(database_url)
+        tables = database.get_usable_table_names()
+        table_infos = database.get_table_info(tables)
+        return table_infos
+
+    def is_local(self) -> bool:
+        return True
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False
+
+
+class ExecuteSqlFunction(AIFunction):
+    def __init__(self):
+        super().__init__()
+        self.name = "execute_sql"
+        self.description = """
+        Input to this function is a detailed and correct SQL query, output is a result from the database.
+        If the query is not correct, an error message will be returned.
+        If an error is returned, rewrite the query, check the query, and try again.
+        """
+
+    def get_name(self) -> str:
+        return self.name
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {
+            "type": "object",
+            "properties": {
+                "database_url": {"type": "string", "description": "Database URL,Can be set to None"},
+                "sql": {"type": "string", "description": "SQL to execute"}
+            }
+        }
+
+    async def execute(self, **kwargs) -> str:
+        database_url = kwargs.get("database_url")
+        if (database_url is None
+                or database_url.strip() == ""
+                or database_url.strip().lower() == "none"
+                or database_url.strip().lower() == "null"):
+            database_url = get_from_env(key="database url", env_key="DATABASE_URL")
+            if database_url is None:
+                return "error: database_url is None"
+        sql = kwargs.get("sql")
+
+        database = get_database(database_url)
+        return database.run_no_throw(sql)
+
+    def is_local(self) -> bool:
+        return True
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False

src/aios/environment/text_to_speech_function.py

@@ -0,0 +1,79 @@
+import io
+import logging
+import os
+import random
+from pathlib import Path
+from typing import Dict
+
+from ..agent.ai_function import AIFunction
+from ..frame.compute_kernel import ComputeKernel
+from ..storage.storage import AIStorage
+
+
+from pydub import AudioSegment
+
+logger = logging.getLogger(__name__)
+
+
+class TextToSpeechFunction(AIFunction):
+    def __init__(self):
+        self.func_id = "text_to_speech"
+        self.description = "根据输入的文本生成音频文件，成功时会返回音频文件路径"
+        self.speech_path = os.path.join(AIStorage.get_instance().get_myai_dir(), "tts")
+        Path(self.speech_path).mkdir(exist_ok=True)
+
+    def get_name(self) -> str:
+        return self.func_id
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {
+            "type": "object",
+            "properties": {
+                "language": {"type": "string", "description": "演播语言", "enum": ["zh", "en"]},
+                "model": {"type": "string", "description": "演播模型", "enum": ["tts-1", "tts-1-hd"]},
+                "text": {"type": "string", "description": "文本内容"}
+            }
+        }
+
+    async def execute(self, **kwargs) -> str:
+        logger.info(f"execute text_to_speech function: {kwargs}")
+
+        language = kwargs.get("language")
+        if language is None:
+            language = "en"
+        model = kwargs.get("model")
+        text = kwargs.get("text")
+
+        i = 0
+        while i < 3:
+            try:
+                data = await ComputeKernel.get_instance().do_text_to_speech(text, language, None, None, None, None,
+                                                                            model_name=model)
+                if data is not None:
+                    audio = AudioSegment.from_mp3(io.BytesIO(data))
+                break
+            except Exception as e:
+                logger.error(f"do_text_to_speech failed: {e}")
+                i += 1
+                continue
+
+        if audio is not None:
+            path = os.path.join(self.speech_path, "{}.mp3".format(''.join(random.sample('zyxwvutsrqponmlkjihgfedcba', 10))))
+            audio.export(path, format="mp3")
+            return "exec text_to_speech OK，speech file store at ```{}```".format(path)
+        else:
+            return "exec text_to_speech failed"
+
+    def is_local(self) -> bool:
+        return True
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False
+
+

src/aios/environment/workflow_env.py

@@ -0,0 +1,421 @@
+
+from datetime import datetime
+import asyncio
+import json
+import sqlite3 # Because sqlite3 IO operation is small, so we can use sqlite3 directly.(so we don't need to use async sqlite3 now)
+from sqlite3 import Error
+import threading
+import logging
+from typing import Optional
+import aiosqlite
+
+from ..proto.compute_task import *
+from ..agent.ai_function import SimpleAIFunction
+from ..frame.compute_kernel import ComputeKernel
+from ..frame.contact_manager import ContactManager,Contact,FamilyMember
+from ..storage.storage import AIStorage
+
+from .environment import Environment,EnvironmentEvent
+from .script_to_speech_function import ScriptToSpeechFunction
+from .image_2_text_function import Image2TextFunction
+
+logger = logging.getLogger(__name__)
+
+class CalenderEvent(EnvironmentEvent):
+    def __init__(self,data) -> None:
+        super().__init__()
+        self.event_name = "timer"
+        self.data = data
+
+    def display(self) -> str:
+        return f"#event timer:{self.data}"
+
+# AI Calender GOAL: Let user use "create notify after 2 days" to create a timer event
+class CalenderEnvironment(Environment):
+    def __init__(self, env_id: str) -> None:
+        super().__init__(env_id)
+        self.db_file = AIStorage.get_instance().get_myai_dir() / "calender.db"
+        self.is_run = False
+
+        self.add_ai_function(SimpleAIFunction("get_time",
+                                        "get current time",
+                                        self._get_now))
+        get_param = {
+            "start_time": "start time (UTC) of event",
+            "end_time": "end time (UTC) of event"
+        }
+        self.add_ai_function(SimpleAIFunction("get_events",
+                                              "get events in calender by time range",
+                                              self._get_events_by_time_range,get_param))
+
+        add_param = {
+            "title": "title of event",
+            "start_time": "start time (UTC) of event",
+            "end_time": "end time (UTC) of event",
+            "participants": "participants of event",
+            "location": "location of event",
+            "details": "details of event"
+        }
+        self.add_ai_function(SimpleAIFunction("add_event",
+                                        "add event to calender",
+                                        self._add_event,add_param))
+
+        delete_param = {
+            "event_id": "id of event"
+        }
+        self.add_ai_function(SimpleAIFunction("delete_event",
+                                        "delete event from calender",
+                                        self._delete_event,delete_param))
+
+        update_param = {
+            "event_id": "id of event",
+            "new_title": "new title of event",
+            "new_participants": "new participants of event",
+            "new_location": "new location of event",
+            "new_details": "new details of event",
+            "start_time": "new start time (UTC) of event",
+            "end_time": "new end time (UTC) of event"
+        }
+        self.add_ai_function(SimpleAIFunction("update_event",
+                                        "update event in calender",
+                                        self._update_event,update_param))
+
+
+        #maybe this function should be in other env?
+        paint_param = {
+            "prompt": "A description of the content of the painting",
+            "model_name": "Which model to use to draw the picture, can be None"
+        }
+        self.add_ai_function(SimpleAIFunction("paint",
+                                        "Draw a picture according to the description",
+                                        self._paint,paint_param))
+
+        self.add_ai_function(SimpleAIFunction("get_contact",
+                                        "get contact info",
+                                        self._get_contact,{"name":"name of contact"}))
+
+        self.add_ai_function(SimpleAIFunction("set_contact",
+                                        "set contact info",
+                                        self._set_contact,{"name":"name of contact","contact_info":"A json to descrpit contact"}))
+
+
+
+
+        #self.add_ai_function(SimpleAIFunction("user_confirm",
+        #                                      "user confirm",
+        #                                      self._user_confirm))
+
+    async def init_db(self):
+        async with aiosqlite.connect(self.db_file) as db:
+            await db.execute("""
+                CREATE TABLE IF NOT EXISTS events (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    title TEXT,
+                    start_time DATETIME,
+                    end_time DATETIME,
+                    participants TEXT,
+                    location TEXT,
+                    details TEXT
+                );
+            """)
+            await db.commit()
+
+    async def _add_event(self,title, start_time, end_time, participants=None, location=None, details=None):
+        async with aiosqlite.connect(self.db_file) as db:
+            await db.execute("""
+                INSERT INTO events (title, start_time, end_time, participants, location, details)
+                VALUES (?, ?, ?, ?, ?, ?);
+            """, (title, start_time, end_time, participants, location, details))
+            await db.commit()
+            return f"execute add_event OK,event '{title}' already add to calender!"
+
+    async def _search_events(self,query):
+        async with aiosqlite.connect(self.db_file) as db:
+            cursor = await db.execute("""
+                SELECT id,title, start_time, end_time, participants, location, details FROM events
+                WHERE title LIKE ? OR participants LIKE ? OR location LIKE ? OR details LIKE ?;
+            """, (f"%{query}%", f"%{query}%", f"%{query}%", f"%{query}%"))
+            rows = await cursor.fetchall()
+
+            result = {}
+            for row in rows:
+                _event = {}
+                _event["title"] = row[1]
+                _event["start_time"] = row[2]
+                _event["end_time"] = row[3]
+                _event["participants"] = row[4]
+                _event["location"] = row[5]
+                _event["details"] = row[6]
+                result[row[0]] = _event
+            return json.dumps(result, indent=4, sort_keys=True)
+
+    async def _get_events_by_time_range(self,start_time, end_time):
+        async with aiosqlite.connect(self.db_file) as db:
+            cursor = await db.execute("""
+                SELECT id,title, start_time, end_time, participants, location, details FROM events
+                WHERE start_time >= ? AND end_time <= ?;
+            """, (start_time, end_time))
+            rows = await cursor.fetchall()
+
+            result = {}
+            have_result = False
+            for row in rows:
+                have_result = True
+                _event = {}
+                _event["title"] = row[1]
+                _event["start_time"] = row[2]
+                _event["end_time"] = row[3]
+                _event["participants"] = row[4]
+                _event["location"] = row[5]
+                _event["details"] = row[6]
+                result[row[0]] = _event
+
+            if not have_result:
+                return "No event."
+
+            return json.dumps(result, indent=4, sort_keys=True)
+
+    async def _update_event(self,event_id, new_title=None, new_participants=None, new_location=None, new_details=None ,start_time=None, end_time=None):
+        fields_to_update = []
+        values = []
+
+        if new_title is not None:
+            fields_to_update.append("title = ?")
+            values.append(new_title)
+
+        if new_participants is not None:
+            fields_to_update.append("participants = ?")
+            values.append(new_participants)
+
+        if new_location is not None:
+            fields_to_update.append("location = ?")
+            values.append(new_location)
+
+        if new_details is not None:
+            fields_to_update.append("details = ?")
+            values.append(new_details)
+
+        if start_time is not None:
+            fields_to_update.append("start_time = ?")
+            values.append(start_time)
+
+        if end_time is not None:
+            fields_to_update.append("end_time = ?")
+            values.append(end_time)
+
+        if not fields_to_update:
+            return "No fields to update."
+
+        sql_update_query = f"""
+            UPDATE events
+            SET {', '.join(fields_to_update)}
+            WHERE id = ?;
+        """
+
+        values.append(event_id)
+
+        async with aiosqlite.connect(self.db_file) as db:
+            await db.execute(sql_update_query, values)
+            await db.commit()
+            return "update ok"
+
+    async def _delete_event(self,event_id):
+        async with aiosqlite.connect(self.db_file) as db:
+            await db.execute("""
+                DELETE FROM events
+                WHERE id = ?;
+            """, (event_id,))
+            await db.commit()
+            return "Delete event ok"
+
+    def _do_get_value(self,key:str) -> Optional[str]:
+        return None
+
+    async def _get_contact(self,name:str) -> str:
+        cm = ContactManager.get_instance()
+        contact : Contact = cm.find_contact_by_name(name)
+        if contact:
+            s = json.dumps(contact.to_dict())
+            return f"Execute get_contact OK , contact {name} is {s}"
+        else:
+            return f"Execute get_contact OK , contact {name} not found!"
+
+    async def _set_contact(self,name:str,contact_info:str) -> str:
+        cm = ContactManager.get_instance()
+        contact = cm.find_contact_by_name(name)
+        contact_info = json.loads(contact_info)
+        if contact is None:
+            contact = Contact(name)
+            contact.email = contact_info.get("email")
+            contact.telegram = contact_info.get("telegram")
+            contact.notes = contact_info.get("notes")
+            contact.added_by = self.env_id
+
+            cm.add_contact(name,contact)
+
+            return f"Execute set_contact OK , new contact {name} added!"
+        else:
+            if contact_info.get("email") is not None:
+                contact.email = contact_info.get("email")
+            if contact_info.get("telegram") is not None:
+                contact.telegram = contact_info.get("telegram")
+            if contact_info.get("notes") is not None:
+                contact.notes = contact_info.get("notes")
+
+            contact.added_by = self.env_id
+            cm.set_contact(name,contact)
+
+            return f"Execute set_contact OK , contact {name} updated!"
+
+    async def start(self) -> None:
+        if self.is_run:
+            return
+        self.is_run = True
+        await self.init_db()
+
+        self.register_get_handler("now",self.get_now)
+        async def timer_loop():
+            while True:
+                if self.is_run == False:
+                    break
+
+                await asyncio.sleep(1.0)
+                now = datetime.now()
+                formatted_time = now.strftime('%Y-%m-%d %H:%M:%S')
+                env_event:CalenderEvent = CalenderEvent(formatted_time)
+                await self.fire_event("timer",env_event)
+
+            return
+
+        asyncio.create_task(timer_loop())
+
+    def stop(self):
+        self.is_run = False
+
+    def get_now(self)->str:
+        now = datetime.now()
+        formatted_time = now.strftime('%Y-%m-%d %H:%M:%S')
+        return formatted_time
+
+    async def _get_now(self) -> str:
+        now = datetime.now()
+        formatted_time = now.strftime('%Y-%m-%d %H:%M:%S')
+        return formatted_time
+
+
+    async def _paint(self, prompt, model_name = None) -> str:
+        result = await ComputeKernel.get_instance().do_text_2_image(prompt, model_name)
+        if result.result_code == ComputeTaskResultCode.ERROR:
+            return f"exec paint failed. err:{result.error_str}"
+        else:
+            return f'exec paint OK, saved as a local file, path is: {result.result["file"]}'
+
+
+class PaintEnvironment(Environment):
+    def __init__(self, env_id: str) -> None:
+        super().__init__(env_id)
+        self.is_run = False
+
+        paint_param = {
+            "prompt": "Keywords of the content of the painting",
+            "model_name": "Which model to use to draw the picture, can be None",
+            "negative_prompt": "Keywords that describe what is not to be drawn, can be None"
+        }
+        self.add_ai_function(SimpleAIFunction("paint",
+                                        "Draw a picture according to the keywords",
+                                        self._paint,paint_param))
+
+    def _do_get_value(self,key:str) -> Optional[str]:
+        return None
+
+
+    async def _paint(self, prompt, model_name = None, negative_prompt = None) -> str:
+        err, result = await ComputeKernel.get_instance().do_text_2_image(prompt, model_name, negative_prompt)
+        if err is not None:
+            return f"exec paint failed. err:{err}"
+        else:
+            return f'exec paint OK, saved as a local file, path is: {result.result["file"]}'
+
+
+# Default Workflow Environment(Context)
+class WorkflowEnvironment(Environment):
+    def __init__(self, env_id: str,db_file:str) -> None:
+        super().__init__(env_id)
+        self.db_file = db_file
+        self.local = threading.local()
+        self.table_name = "WorkflowEnv_" + env_id
+        self.add_ai_function(ScriptToSpeechFunction())
+        self.add_ai_function(Image2TextFunction())
+
+
+    def _get_conn(self):
+        """ get db connection """
+        if not hasattr(self.local, 'conn'):
+            self.local.conn = self._create_connection()
+        return self.local.conn
+
+    def _create_connection(self):
+        """ create a database connection to a SQLite database """
+        conn = None
+        try:
+            conn = sqlite3.connect(self.db_file)
+        except Error as e:
+            logging.error("Error occurred while connecting to database: %s", e)
+            return None
+
+        if conn:
+            self._create_table(conn)
+
+        return conn
+
+    def close(self):
+        if not hasattr(self.local, 'conn'):
+            return
+        self.local.conn.close()
+
+    def _create_table(self, conn):
+        """ create table """
+        try:
+            # create sessions table
+            conn.execute(f"""
+                CREATE TABLE IF NOT EXISTS """ + self.table_name + """ (
+                    EnvKey TEXT PRIMARY KEY,
+                    EnvValue TEXT,
+                    UpdateTime TEXT
+                );
+            """)
+            conn.commit()
+        except Error as e:
+            logging.error("Error occurred while creating tables: %s", e)
+
+    def _do_get_value(self, key: str) -> str | None:
+        try:
+            conn = self._get_conn()
+            c = conn.cursor()
+            c.execute("SELECT EnvValue FROM " + self.table_name +" WHERE EnvKey = ?", (key,))
+            value = c.fetchone()
+            if value is None:
+                return None
+            return value[0]
+        except Error as e:
+            logging.error(f"Error occurred while _do_get_value{key}: {e}")
+            return None
+
+    def set_value(self, key: str, str_value: str, is_storage:bool=True):
+        super().set_value(key,str_value)
+        if is_storage is False:
+            return
+
+        try:
+            conn = self._get_conn()
+            conn.execute("""
+                INSERT OR REPLACE INTO """ + self.table_name+ """ (EnvKey, EnvValue, UpdateTime)
+                VALUES (?, ?, ?) 
+            """, (key, str_value, datetime.now()))
+            conn.commit()
+            return 0  # return 0 if successful
+        except Error as e:
+            logging.error(f"Error occurred while update env{self.env_id}.{key} ,error:{e}")
+
+    def get_functions(self):
+        pass

src/aios/environment/workspace_env.py

@@ -0,0 +1,789 @@
+# this env is designed for workflow owner filesystem, support file/directory operations
+
+import hashlib
+import json
+import subprocess
+import logging
+import tempfile
+import threading
+import traceback
+import time
+import ast
+import sys
+import os
+import re
+import asyncio
+import aiofiles
+from typing import Any,List
+import os
+import chardet
+from markdown import Markdown
+import PyPDF2
+
+from ..proto.agent_msg import *
+from ..agent.agent_base import AgentTodo,AgentPrompt,AgentTodoResult
+from ..agent.ai_function import AIFunction,SimpleAIFunction
+from ..storage.storage import AIStorage,ResourceLocation
+from .simple_kb_db import SimpleKnowledgeDB
+from .environment import Environment,EnvironmentEvent
+
+logger = logging.getLogger(__name__)
+
+class WorkspaceEnvironment(Environment):
+    def __init__(self, env_id: str) -> None:
+        super().__init__(env_id)
+        myai_path = AIStorage.get_instance().get_myai_dir() 
+        self.root_path = f"{myai_path}/workspace/{env_id}"
+        if not os.path.exists(self.root_path):
+            os.makedirs(self.root_path+"/todos")
+
+        self.known_todo = {}
+        self.kb_db = SimpleKnowledgeDB(f"{self.root_path}/kb.db")
+        self.doc_dirs = {}
+        self._scan_thread = None 
+        self._scan_dirthread = None
+        
+
+    def set_root_path(self,path:str):
+        self.root_path = path
+
+    def get_prompt(self) -> AgentMsg:
+        return None
+    
+    def get_role_prompt(self,role_id:str) -> AgentPrompt:
+        return None
+
+    def get_knowledge_base(self,root_dir=None,indent=0) -> str:
+        pass
+
+
+    def get_do_prompt(self,todo:AgentTodo=None)->AgentPrompt:
+        return None
+
+    # result mean: list[op_error_str],have_error
+    async def exec_op_list(self,oplist:List,agent_id:str)->tuple[List[str],bool]:
+        result_str = "op list is none"
+        if oplist is None:
+            return None,False
+        
+        result_str = []
+        have_error = False
+        for op in oplist:
+            if op["op"] == "create":
+                await self.create(op["path"],op["content"])
+            elif op["op"] == "write_file":
+                is_append = op.get("is_append")
+                if is_append is None:
+                    is_append = False
+                error_str = await self.write(op["path"],op["content"],is_append)
+            elif op["op"] == "delete":
+                error_str = await self.delete(op["path"])
+            elif op["op"] == "rename":
+                error_str = await self.rename(op["path"],op["new_name"])
+            elif op["op"] == "mkdir":
+                error_str = await self.mkdir(op["path"])
+            elif op["op"] == "create_todo":
+                todoObj = AgentTodo.from_dict(op["todo"])
+                todoObj.worker = agent_id
+                todoObj.createor = agent_id
+                parent_id = op.get("parent")
+                error_str = await self.create_todo(parent_id,todoObj)
+            elif op["op"] == "update_todo":
+                todo_id = op["id"]
+                new_stat = op["state"]
+                error_str = await self.update_todo(todo_id,new_stat)
+            else:
+                logger.error(f"execute op list failed: unknown op:{op['op']}")
+                error_str = f"execute op list failed: unknown op:{op['op']}"
+            
+            if error_str:
+                have_error = True
+                result_str.append(error_str)
+            else:
+                result_str.append(f"execute success!")  
+    
+        
+        return result_str,have_error
+    
+    # file system operation: list,read,write,delete,move,stat
+    # inner_function
+    async def list(self,path:str,only_dir:bool=False) -> str:
+        directory_path = self.root_path + path
+        items = []
+
+        with await aiofiles.os.scandir(directory_path) as entries:
+            async for entry in entries:
+                is_dir = entry.is_dir()
+                if only_dir and not is_dir:
+                    continue
+                item_type = "directory" if is_dir else "file"
+                items.append({"name": entry.name, "type": item_type})
+
+        return json.dumps(items)
+    
+    # inner_function
+    async def read(self,path:str) -> str:
+        file_path = self.root_path + path
+        cur_encode = "utf-8"
+        async with aiofiles.open(file_path,'rb') as f:
+            cur_encode = chardet.detect(await f.read())['encoding']
+
+        async with aiofiles.open(file_path, mode='r', encoding=cur_encode) as f:
+            content = await f.read(2048)
+        return content
+    
+
+    # operation or inner_function (MOST IMPORTANT FUNCTION)
+    async def write(self,path:str,content:str,is_append:bool=False) -> str:
+        file_path = self.root_path + path
+        try:
+            if is_append:
+                async with aiofiles.open(file_path, mode='a', encoding="utf-8") as f:
+                    await f.write(content)
+            else:
+                if content is None:
+                    # create dir
+                    dir_path = self.root_path + path
+                    os.makedirs(dir_path)
+                    return True
+                else:
+                    file_path = self.root_path + path
+                    os.makedirs(os.path.dirname(file_path),exist_ok=True)
+                    async with aiofiles.open(file_path, mode='w', encoding="utf-8") as f:
+                        await f.write(content)
+                    return True
+        
+        except Exception as e:
+            return str(e)
+        return None
+    
+        
+    # operation or inner_function
+    async def delete(self,path:str) -> str:
+        try:
+            file_path = self.root_path + path
+            os.remove(file_path)
+        except Exception as e:
+            return str(e)
+        
+        return None
+    
+    # operation or inner_function
+    async def move(self,path:str,new_path:str) -> str:
+        try:
+            file_path = self.root_path + path
+            new_path = self.root_path + new_path
+            os.rename(file_path,new_path)
+        except Exception as e:
+            return str(e)
+        
+        return None
+    
+    # inner_function
+    async def stat(self,path:str) -> str:
+        try:
+            file_path = self.root_path + path
+            stat = os.stat(file_path)
+            return json.dumps(stat)
+        except Exception as e:
+            return str(e)
+
+    # operation or inner_function   
+    async def symlink(self,path:str,target:str) -> str:
+        try:
+            #file_path = self.root_path + path
+            target_path = self.root_path + target
+            dir_path = os.path.dirname(target_path)
+            os.makedirs(dir_path,exist_ok=True)
+            os.symlink(path,target_path)
+        except Exception as e:
+            logger.error("symlink failed:%s",e)
+            return str(e)
+        
+        return None
+        
+    # TODO use diff to update large file content
+    async def update_by_diff(self,path:str,diff):
+
+        pass 
+    
+    # doc system （read_only,agent cann't modify doc）
+
+    # inner_function
+    async def list_db(self) -> str:
+        pass
+    # inner_function
+    async def get_db_desc(self,db_name:str) -> str:
+        pass  
+    # inner_function
+    async def query(self,db_name:str,sql:str) -> str:
+        pass
+
+    # search (web)
+    # inner_function
+    async def google_search(self,keyword:str,opt=None) -> str:
+        pass
+
+    # inner_function
+    async def local_search(self,keyword:str,root_path=None ,opt=None) -> str:
+        pass
+    
+    # inner_function, might be return a image is better
+    async def web_get(self,url:str) -> str:
+        pass
+    
+    # inner_function
+    async def blockchain_get(self,chainid:str,query:dict) -> str:
+        pass
+
+    # code interpreter
+    # inner_function or operation
+    async def eval_code(self,pycode:str) -> str:
+        pass
+    
+    # operation or inner_function
+    async def improve_code(self,path:str):
+        pass
+    
+    # operation or inner_function
+    async def run(self,file_path:str)->str:
+        pass
+
+    # operation or inner_function
+    async def pub_service(self,project_path:str):
+        pass
+
+    # operation or inner_function
+    async def exec_tx(self,chain_id:str,tx:dict) -> str:
+        pass
+    
+    # social ability
+    # operation or inner_function
+    async def post_message(self,target:str,msg:AgentMsg,wait_time) -> AgentMsg:
+        pass
+    
+    # operation or inner_function
+    async def add_contact(self,name:str,contact_info) -> str:
+        pass
+
+    # inner_function , include contact realtime info
+    async def get_contact(self,name_list:List[str],opt:dict) -> List:
+        pass
+
+
+    # Task/todo system , create,update,delete,query
+    async def get_todo_tree(self,path:str = None,deep:int = 4):
+        if path:
+            directory_path = self.root_path + "/todos/" + path
+        else:
+            directory_path = self.root_path + "/todos"
+
+        
+        str_result:str = "/todos\n"
+        todo_count:int = 0 
+
+        async def scan_dir(directory_path:str,deep:int):
+            nonlocal str_result
+            nonlocal todo_count
+            if deep <= 0:
+                return
+            
+            if os.path.exists(directory_path) is False:
+                return 
+            
+            for entry in os.scandir(directory_path):
+                is_dir = entry.is_dir()
+                if not is_dir:
+                    continue
+
+                if entry.name.startswith("."):
+                    continue
+                
+                todo_count = todo_count +  1
+                str_result = str_result + f"{'  '*(4-deep)}{entry.name}\n"
+                await scan_dir(entry.path,deep-1)
+
+        await scan_dir(directory_path,deep)
+        return str_result,todo_count
+
+    async def get_todo_list(self,agent_id:str,path:str = None)->List[AgentTodo]:
+        logger.info("get_todo_list:%s,%s",agent_id,path)
+        if path:
+            directory_path = self.root_path + "/todos/" + path
+        else:
+            directory_path = self.root_path + "/todos"
+
+        result_list:List[AgentTodo] = []
+
+        async def scan_dir(directory_path:str,deep:int,parent:AgentTodo=None):
+            nonlocal result_list
+            if os.path.exists(directory_path) is False:
+                return 
+
+            for entry in os.scandir(directory_path):
+                is_dir = entry.is_dir()
+                if not is_dir:
+                    continue
+
+                if entry.name.startswith("."):
+                    continue
+                
+                todo = await self.get_todo_by_fullpath(entry.path)
+                if todo:
+                    if todo.worker:
+                        if todo.worker != agent_id:
+                            continue
+                        
+                    if parent:
+                        parent.sub_todos[todo.todo_id] = todo
+                    
+                    result_list.append(todo)
+                    todo.rank = int(todo.create_time)>>deep
+                    await scan_dir(entry.path,deep + 1,todo)
+            
+            return 
+
+        await scan_dir(directory_path,0) 
+        #sort by rank
+        result_list.sort(key=lambda x:(x.rank,x.title))
+        logger.info("get_todo_list return,todolist.length() is %d",len(result_list))
+        return result_list
+
+    async def get_todo_by_fullpath(self,path:str) -> AgentTodo:
+        logger.info("get_todo_by_fullpath:%s",path)
+
+        detail_path = path + "/detail"
+        try:
+            async with aiofiles.open(detail_path, mode='r', encoding="utf-8") as f:
+                content = await f.read(4096)
+                logger.debug("get_todo_by_fullpath:%s,content:%s",path,content)
+                todo_dict = json.loads(content)
+                result_todo =  AgentTodo.from_dict(todo_dict)
+                if result_todo:
+                    relative_path = os.path.relpath(path, self.root_path + "/todos/")
+                    if not relative_path.startswith('/'):
+                        relative_path = '/' + relative_path
+                    result_todo.todo_path = relative_path
+                    self.known_todo[result_todo.todo_id] = result_todo
+                else:
+                    logger.error("get_todo_by_path:%s,parse failed!",path)
+                
+                return result_todo
+        except Exception as e:
+            logger.error("get_todo_by_path:%s,failed:%s",path,e)
+            return None
+        
+    async def get_todo(self,id:str) -> AgentTodo:
+        return self.known_todo.get(id)
+
+    async def create_todo(self,parent_id:str,todo:AgentTodo) -> str:
+        try:
+            if parent_id:
+                if parent_id not in self.known_todo:
+                    logger.error("create_todo failed: parent_id not found!")
+                    return False
+                
+                parent_path = self.known_todo.get(parent_id).todo_path
+                todo_path = f"{parent_path}/{todo.title}"
+            else:
+                todo_path = todo.title
+
+            dir_path = f"{self.root_path}/todos/{todo_path}"
+    
+            os.makedirs(dir_path)
+            detail_path = f"{dir_path}/detail"
+            if todo.todo_path is None:
+                todo.todo_path = todo_path
+            logger.info("create_todo %s",detail_path)
+            async with aiofiles.open(detail_path, mode='w', encoding="utf-8") as f:
+                await f.write(json.dumps(todo.to_dict()))
+                self.known_todo[todo.todo_id] = todo
+        except Exception as e:
+            logger.error("create_todo failed:%s",e)
+            return str(e)
+        
+        return None
+
+    async def update_todo(self,todo_id:str,new_stat:str)->str:
+        try:
+            todo : AgentTodo = self.known_todo.get(todo_id)
+            if todo:
+                todo.state = new_stat
+                detail_path =  f"{self.root_path}/todos/{todo.todo_path}/detail"
+                async with aiofiles.open(detail_path, mode='w', encoding="utf-8") as f:
+                    await f.write(json.dumps(todo.to_dict()))
+                    return None
+            else:
+                return "todo not found."
+        except Exception as e:
+            return str(e)
+    
+    async def append_worklog(self,todo:AgentTodo,result:AgentTodoResult):
+        worklog = f"{self.root_path}/todos/{todo.todo_path}/.worklog"
+
+        async with aiofiles.open(worklog, mode='w+', encoding="utf-8") as f:
+            content = await f.read()
+            if len(content) > 0:
+                json_obj = json.loads(content)
+            else:
+                json_obj = {}
+            logs = json_obj.get("logs")
+            if logs is None:
+                logs = []
+            logs.append(result.to_dict())
+            json_obj["logs"] = logs
+            await f.write(json.dumps(json_obj))
+
+    async def set_wakeup_timer(self,todo_id:str,timestamp:int) -> str:
+        pass
+
+    # knowledge base system
+    def get_knowledge_base_ai_functions(self):
+        all_inner_function = []
+
+        all_inner_function.append(SimpleAIFunction("get_knowledge_catalog","get knowledge catalog in tree format",
+                                                                self.get_knowledege_catalog,
+                                                                {"path":f"catalog path,none is /","depth":"max depth of catalog tree,default is 4"}))
+        all_inner_function.append(SimpleAIFunction("get_knowledge","get knowledge metadata",
+                                                                self.get_knowledge,
+                                                                {"path":f"knowledge path"}))
+        all_inner_function.append(SimpleAIFunction("load_knowledge_content","load knowledge content",
+                                                                 self.load_knowledge_content,
+                                                                  {"path":f"knowledge path","pos":"start position of content","length":"length of content"}))
+        result_func = []
+        result_len = 0
+        for inner_func in all_inner_function:
+            func_name = inner_func.get_name()
+
+            this_func = {}
+            this_func["name"] = func_name
+            this_func["description"] = inner_func.get_description()
+            this_func["parameters"] = inner_func.get_parameters()
+            result_len += len(json.dumps(this_func)) / 4
+            result_func.append(this_func)
+
+        return result_func,result_len                                                        
+
+    async def get_knowledege_catalog(self,path:str=None,only_dir =True,max_depth:int=5)->str:
+        if path:
+            full_path = f"{self.root_path}/knowledge/{path}"
+        else:
+            full_path = f"{self.root_path}/knowledge"
+        
+        catlogs,file_count = await self.get_directory_structure(full_path,max_depth,only_dir)
+        return catlogs
+ 
+    async def get_directory_structure(self,root_dir, max_depth:int=4, only_dir=True, indent=1):
+        file_count = 0
+        structure_str = ''
+        if os.path.isdir(root_dir):
+            sub_files = []
+            with os.scandir(root_dir) as it:
+                for entry in it:
+                    if entry.is_dir():
+                        sub_structure, sub_count = await self.get_directory_structure(entry.path, max_depth, only_dir, indent + 1)
+                        if sub_structure:
+                            structure_str += sub_structure
+                        file_count += sub_count
+                    else:
+                        file_count += 1
+                        sub_files.append(entry.name)
+
+                if only_dir is False:
+                    for file_name in sub_files:
+                        structure_str = structure_str + '  ' * (indent+1) + file_name + '\n' 
+
+            dir_name = os.path.basename(root_dir)
+            dir_info = f"{dir_name} <count: {file_count}>"
+        
+
+            structure_str = '  ' * indent + dir_info + '\n' + structure_str
+
+        if indent - 1 >= max_depth:
+            return None, file_count
+        else:
+            return structure_str, file_count
+
+    # inner_function    
+    async def get_knowledge(self,path:str) -> str:
+        full_path = f"{self.root_path}/knowledge/{path}"
+        if os.islink(full_path):
+            org_path = os.readlink(full_path)
+            hash = self.kb_db.get_hash_by_doc_path(org_path)
+            if hash:
+                return self.kb_db.get_knowledge(org_path)
+        
+        return "not found"
+
+    async def load_knowledge_content(self,path:str,pos:int=0,length:int=None) -> str:
+        if path.endswith("pdf"):
+            logger.info("load_knowledge_content:pdf")
+            dir_path = os.path.dirname(path)
+            base_name = os.path.basename(path)
+            text_content_path = f"{dir_path}/.{base_name}.txt"
+            if os.path.exists(text_content_path) is False:
+                return None
+            async with aiofiles.open(path, mode='r', encoding=cur_encode) as f:
+                await f.seek(pos)
+                content = await f.read(length)
+                return content
+        else:
+            async with aiofiles.open(path,'rb') as f:
+                cur_encode = chardet.detect(await f.read())['encoding']
+
+            async with aiofiles.open(path, mode='r', encoding=cur_encode) as f:
+                await f.seek(pos)
+                content = await f.read(length)
+                return content
+
+        return "load content failed."
+
+    def _add_document_dir(self,path:str):
+        self.doc_dirs[path] = 0
+
+    def _start_scan_document(self):
+        if self._scan_thread is None:
+            self._scan_thread = threading.Thread(target=self._scan_document)
+            self._scan_thread.start()
+        if self._scan_dirthread is None:
+            self._scan_dirthread = threading.Thread(target=self._scan_dir)
+            self._scan_dirthread.start()
+
+    def _parse_pdf_bookmarks(self,bookmarks, parent:list):
+
+        for item in bookmarks:
+            if isinstance(item,list):
+                self._parse_pdf_bookmarks(item,parent)
+            else:
+                if item.title:
+                    new_item = {}
+                    new_item["page"] = item.page.idnum
+                    new_item["title"] = item.title 
+                    my_childs = []
+                    if item.childs:
+                        if len(item.childs) > 0:
+                            self._parse_pdf_bookmarks(item.childs, my_childs)
+                            new_item["childs"] = my_childs
+                    parent.append(new_item)
+                else:
+                    logger.warning("parse pdf bookmarks failed: item.title is None!")
+
+        return
+
+    def _parse_pdf(self,doc_path:str):
+        metadata = {}
+        with open(doc_path, 'rb') as file:
+            reader = PyPDF2.PdfReader(file)
+            try:
+                doc_info = reader.metadata
+                if doc_info:
+                    if doc_info.title:
+                        metadata["title"] = doc_info.title
+                    if doc_info.author:
+                        metadata["authors"] = doc_info.author
+            except Exception as e:
+                logger.warn("parse pdf metadata failed:%s",e)
+
+            dir_path = os.path.dirname(doc_path)
+            base_name = os.path.basename(doc_path)
+            text_content_path = f"{dir_path}/.{base_name}.txt"
+            full_text = ""
+
+            for page in reader.pages:
+                text = page.extract_text()
+                full_text += text
+            with open(text_content_path, 'w', encoding='utf-8') as f:
+                f.write(full_text)
+
+            try:
+                bookmarks = reader.outline
+                if bookmarks:
+                    catalogs = []
+                    self._parse_pdf_bookmarks(bookmarks,catalogs)
+                    metadata["catalogs"] = json.dumps(catalogs)
+            except Exception as e:
+                logger.warn("parse pdf bookmarks failed:%s",e)
+
+        return metadata
+
+    def _parse_txt(self,doc_path:str):
+        return {}
+
+    def _parse_md(self,doc_path:str):
+        metadata = {} 
+        cur_encode = "utf-8"
+        with open(doc_path,'rb') as f:
+            cur_encode = chardet.detect(f.read(1024))['encoding']
+
+        with open(doc_path, mode='r', encoding=cur_encode) as f:
+            content = f.read()
+            match = re.search(r'^# (.*)', content, re.MULTILINE)
+            if match:
+                metadata['title'] = match.group(1).strip()
+            md = Markdown(extensions=['toc'])
+            html_str = md.convert(content)
+            toc = md.toc
+            if toc:
+                metadata['catalogs'] = toc
+            
+        return metadata
+
+    def _parse_document(self,doc_path:str):
+        hash_result = None
+        title = os.path.basename(doc_path)
+        meta_data = {}
+
+        with open(doc_path, "rb") as f:
+            hash_md5 = hashlib.md5()
+            for chunk in iter(lambda: f.read(1024*1024), b""):
+                hash_md5.update(chunk)
+            hash_result = hash_md5.hexdigest()
+        try:
+            if doc_path.endswith(".md"):
+                meta_data = self._parse_md(doc_path)
+            elif doc_path.endswith(".pdf"):
+                meta_data = self._parse_pdf(doc_path)
+        except Exception as e:
+            logger.error("parse document %s failed:%s",doc_path,e)
+            traceback.print_exc()
+
+        if meta_data.get("title"):
+            title = meta_data["title"]
+        logger.info("parse document %s!",doc_path)
+        return hash_result,title,meta_data
+
+
+    def _support_file(self,file_name:str) -> bool:
+        if file_name.startswith("."):
+            return False
+        
+        if file_name.endswith(".pdf"):
+            return True
+        if file_name.endswith(".md"):
+            return True
+        if file_name.endswith(".txt"):
+            return True
+        return False
+
+    def _scan_dir(self):
+        while True:
+            time.sleep(10)
+            for directory in self.doc_dirs.keys():
+                now = time.time()
+                if now - self.doc_dirs[directory] > 60*15:
+                    self.doc_dirs[directory] = time.time()
+                else:
+                    continue
+
+                for root, dirs, files in os.walk(directory):
+                    for file in files:
+                        if self._support_file(file):
+                            full_path = os.path.join(root, file)
+                            full_path = os.path.normpath(full_path)
+                            if self.kb_db.is_doc_exist(full_path):
+                                continue
+                            
+                            file_stat = os.stat(full_path)
+                            if file_stat.st_size < 1:
+                                continue
+
+                            if file_stat.st_size < 1024*1024*8:
+                                #parse and insert
+                                hash,title,meta_data = self._parse_document(full_path)
+                                self.kb_db.add_doc(full_path,file_stat.st_size,file_stat.st_mtime,hash)
+                                self.kb_db.add_knowledge(hash,title,meta_data)
+                                
+                            else:
+                                self.kb_db.add_doc(full_path,file_stat.st_size,file_stat.st_mtime)
+
+    def _scan_document(self):
+       while True:
+        time.sleep(10)
+        parse_queue = self.kb_db.get_docs_without_hash()
+        for doc_path in parse_queue:
+                hash,title,meta_data = self._parse_document(doc_path)
+                self.kb_db.set_doc_hash(doc_path,hash)
+                self.kb_db.add_knowledge(hash,title,meta_data)
+        
+    
+
+
+# merge to standard workspace env, **ABANDON this!**
+class KnowledgeBaseFileSystemEnvironment(Environment):
+    def __init__(self, env_id: str) -> None:
+        super().__init__(env_id)
+        self.root_path = "."
+
+        operator_param = {
+            "path": "full path of target directory",
+        }
+        self.add_ai_function(SimpleAIFunction("list",
+                                        "list the files and sub directory in target directory,result is a json array",
+                                        self.list,operator_param))
+        
+        operator_param = {
+            "path": "full path of target file",
+        }
+        self.add_ai_function(SimpleAIFunction("cat",
+                                        "cat the file content in target path,result is a string",
+                                        self.cat,operator_param))
+    
+    def set_root_path(self,path:str):
+        self.root_path = path
+
+    
+    async def list(self,path:str) -> str:
+        directory_path = self.root_path + path
+        items = []
+
+        with await aiofiles.os.scandir(directory_path) as entries:
+            async for entry in entries:
+                item_type = "directory" if entry.is_dir() else "file"
+                items.append({"name": entry.name, "type": item_type})
+
+        return json.dumps(items)
+
+    async def cat(self,path:str) -> str:
+        file_path = self.root_path + path
+        cur_encode = "utf-8"
+        async with aiofiles.open(file_path,'rb') as f:
+            cur_encode = chardet.detect(await f.read())['encoding']
+
+        async with aiofiles.open(file_path, mode='r', encoding=cur_encode) as f:
+            content = await f.read(2048)
+        return content
+
+
+class ShellEnvironment(Environment):
+    def __init__(self, env_id: str) -> None:
+        super().__init__(env_id)
+
+        operator_param = {
+            "command": "command will execute",
+        }
+        self.add_ai_function(SimpleAIFunction("shell_exec",
+                                        "execute shell command in linux bash",
+                                        self.shell_exec,operator_param))
+        
+        #run_code_param = {
+        #    "pycode": "python code will execute",
+        #}
+        #self.add_ai_function(SimpleAIFunction("run_code",
+        #                                "execute python code",
+        #                                self.run_code,run_code_param))
+        
+
+    async def shell_exec(self,command:str) -> str:
+        import asyncio.subprocess
+        process = await asyncio.create_subprocess_shell(
+            command,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE
+        )
+        stdout, stderr = await process.communicate()
+        returncode = process.returncode
+        if returncode == 0:
+            return f"Execute success! stdout is:\n{stdout}\n"
+        else:
+            return f"Execute failed! stderr is:\n{stderr}\n"
+

src/aios/frame/bus.py

@@ -0,0 +1,149 @@
+from typing import Coroutine,Dict,Any
+import asyncio
+from asyncio import Queue
+import logging
+
+from ..proto.agent_msg import *
+from ..agent.agent_base import *
+
+logger = logging.getLogger(__name__)
+
+class AIBusHandler:
+    def __init__(self,handler:Coroutine,owner_bus,enable_defualt_proc=True) -> None:
+        self.handler = handler
+        self.working_task = None
+        self.results = {} # recv resps
+        self.queue:Queue = Queue()
+        self.enable_defualt_proc = enable_defualt_proc
+        self.owner_bus = owner_bus
+
+    async def handle_message(self,msg:AgentMsg) -> Any:
+        if self.handler is None:
+            return None
+
+        resp_msg = await self.handler(msg)
+        if self.enable_defualt_proc:
+            if resp_msg is not None:
+                if resp_msg.msg_type == AgentMsgType.TYPE_GROUPMSG:
+                    await self.owner_bus.post_message(resp_msg,resp_msg.target)
+                else:
+                    await self.owner_bus.post_message(resp_msg)
+
+        return resp_msg
+
+class AIBus:
+    _instance = None
+    @classmethod
+    def get_default_bus(cls):
+        if cls._instance is None:
+            cls._instance = AIBus()
+        return cls._instance
+
+    def __init__(self) -> None:
+        self.handlers:Dict[AIBusHandler] = {}
+        self.unhandle_handler:Coroutine = None
+
+
+    async def post_message(self,msg:AgentMsg,target_id = None,use_unhandle=True) -> bool:
+        if target_id is None:
+            target_id =msg.target
+
+        target_id = target_id.split(".")[0]
+
+        handler = self.handlers.get(target_id)
+        if handler:
+            if msg.rely_msg_id is not None:
+                handler.results[msg.rely_msg_id] = msg
+                return None
+
+            handler.queue.put_nowait(msg)
+            self.start_process(target_id)
+            return True
+
+        if use_unhandle:
+            if self.unhandle_handler is not None:
+                if await self.unhandle_handler(self,target_id):
+                    return await self.post_message(msg,target_id,False)
+
+        logger.warn(f"post message to {msg.target} failed!,target not found")
+        return False
+
+    async def resp_message(self,org_msg_id:str,resp:AgentMsg) -> None:
+        assert resp.rely_msg_id == org_msg_id
+        return await self.post_message(resp)
+
+    async def send_message(self,msg:AgentMsg,target_id = None, real_sender=None) -> AgentMsg:
+        if real_sender is None:
+            sender_id = msg.sender.split(".")[0]
+        else:
+            sender_id = real_sender.split(".")[0]
+            
+        sender_handler = self.handlers.get(sender_id) # sender already register on bus
+        if sender_handler is None:
+            logger.warn(f"sender {sender_id} not register on AI_BUS!")
+            return None
+
+        post_result = await self.post_message(msg,target_id)
+        if post_result is False:
+            return None
+
+        retry_times = 0
+        while True:
+            resp : AgentMsg = sender_handler.results.get(msg.msg_id)
+            if resp is not None:
+                msg.resp_msg = resp
+                msg.status = AgentMsgStatus.RESPONSED
+                del sender_handler.results[msg.msg_id]
+                return resp
+
+            await asyncio.sleep(0.2)
+            retry_times += 1
+            if retry_times > 5*240: # default timeout is 240 sec
+                msg.status = AgentMsgStatus.ERROR
+                return None
+
+        return None
+
+    def register_unhandle_message_handler(self,handler:Any) -> Queue:
+        self.unhandle_handler = handler
+
+    # means sub
+    def register_message_handler(self,handler_name:str,handler:Any) -> Queue:
+        handler_node =  AIBusHandler(handler,self)
+        if self.handlers.get(handler_name) is not None:
+            logger.warn(f"handler {handler_name} already register on AI_BUS!")
+                        
+        self.handlers[handler_name] = handler_node
+        return handler_node.queue
+
+    async def process_queue(self, handler:AIBusHandler):
+        while True:
+            # Wait for a message
+            message = await handler.queue.get()
+
+            try:
+                # Try to handle the message
+                await handler.handle_message(message)
+            except Exception as e:
+                # If an error occurs, put the message back into the queue
+                logger.error(f"handle message {message.msg_id} failed! {e}")
+                logger.exception(e)
+                raise e
+                #self.queues[name].put_nowait(message)
+
+        return
+
+    def start_process(self,target_name):
+        handler = self.handlers.get(target_name)
+        if handler is None:
+            logger.error(f"handler {target_name} not found!")
+            return
+
+        if handler.handler is None:
+            return
+
+        if handler.working_task is not None:
+            logger.warn(f"handler {target_name} is already working!")
+            return
+
+        handler.working_task = asyncio.create_task(self.process_queue(handler))

src/aios/frame/compute_kernel.py

@@ -0,0 +1,258 @@
+from abc import ABC, abstractmethod
+import random
+from typing import Optional
+import logging
+import asyncio
+import tiktoken
+from asyncio import Queue
+
+from ..proto.compute_task import *
+from ..knowledge import ObjectID
+from ..agent.agent_base import AgentPrompt
+from .compute_node import ComputeNode
+
+logger = logging.getLogger(__name__)
+
+# How to dispatch different computing tasks (some tasks may contain a large amount of state for correct execution)
+# to suitable computing nodes, achieving a balance of speed, cost, and power consumption,
+# is the CORE GOAL of the entire computing task schedule system (aios_kernel).
+
+class ComputeKernel:
+    _instance = None
+    @classmethod
+    def get_instance(cls):
+        if cls._instance is None:
+            cls._instance = ComputeKernel()
+        return cls._instance
+
+    def __init__(self) -> None:
+        self.is_start = False
+        self.task_queue = Queue()
+        self.is_start = False
+        self.compute_nodes = {}
+
+    def run(self, task: ComputeTask) -> None:
+        # check there is compute node can support this task
+        if self.is_task_support(task) is False:
+            logger.error(
+                f"task {task.display()} is not support by any compute node")
+            return
+        # add task to working_queue
+        self.task_queue.put_nowait(task)
+
+    async def start(self):
+        if self.is_start is True:
+            logger.warn("compute_kernel is already start")
+            return
+
+        self.is_start = True
+
+        async def _run_task_loop():
+            while True:
+                task = await self.task_queue.get()
+                logger.info(f"compute_kernel get task: {task.display()}")
+                c_node: ComputeNode = self._schedule(task)
+                if c_node:
+                    await c_node.push_task(task)
+
+            logger.warn("compute_kernel is stoped!")
+
+        asyncio.create_task(_run_task_loop())
+
+    def _schedule(self, task) -> ComputeNode:
+        # find all the node which supports this task
+        support_nodes = []
+        total_weights = 0
+
+        for node in self.compute_nodes.values():
+            if node.is_support(task) is True:
+                support_nodes.append({
+                    "pos": total_weights,
+                    "node": node
+                })
+                total_weights += node.weight()
+
+        if len(support_nodes) < 1:
+            logger.warning(f"task {task.display()} is not support by any compute node")
+            return None
+
+        # hit a random node with weight
+        hit_pos = random.randint(0, total_weights - 1)
+        for i in range(min(len(support_nodes) - 1, hit_pos), -1, -1):
+            if support_nodes[i]["pos"] <= hit_pos:
+                return support_nodes[i]["node"]
+
+        logger.warning(
+            f"task {task.display()} is not support by any compute node")
+        return None
+
+    def add_compute_node(self, node: ComputeNode):
+        if self.compute_nodes.get(node.node_id) is not None:
+            logger.warn(
+                f"compute_node {node.display()} already in compute_kernel")
+            return
+        self.compute_nodes[node.node_id] = node
+        logger.info(f"add compute_node {node.display()} to compute_kernel")
+
+    def disable_compute_node(self, node_id: str):
+        node = self.compute_nodes.get(node_id)
+        if node is None:
+            logger.warn(f"compute_node {node_id} not in compute_kernel")
+            return
+        node.enable = False
+
+    def is_task_support(self, task: ComputeTask) -> bool:
+        return True
+
+    @staticmethod
+    def llm_num_tokens_from_text(text:str,model:str) -> int:
+        try:
+            encoding = tiktoken.encoding_for_model(model)
+        except KeyError:
+            logger.debug("Warning: model not found. Using cl100k_base encoding.")
+            encoding = tiktoken.get_encoding("cl100k_base")
+
+        token_count = len(encoding.encode(text))
+        return token_count
+
+
+    # friendly interface for use:
+    def llm_completion(self, prompt: AgentPrompt, resp_mode:str="text",mode_name: Optional[str] = None, max_token: int = 0,inner_functions = None):
+        # craete a llm_work_task ,push on queue's end
+        # then task_schedule would run this task.(might schedule some work_task to another host)
+        task_req = ComputeTask()
+        task_req.set_llm_params(prompt,resp_mode,mode_name, max_token,inner_functions)
+        self.run(task_req)
+        return task_req
+
+    async def _wait_task(self,task_req:ComputeTask, timeout=60)->ComputeTaskResult:
+        async def check_timer():
+            check_times = 0
+            while True:
+                if task_req.state == ComputeTaskState.DONE:
+                    break
+
+                if task_req.state == ComputeTaskState.ERROR:
+                    break
+
+                if timeout is not None and check_times >= timeout*2:
+                    task_req.state = ComputeTaskState.ERROR
+                    break
+
+                await asyncio.sleep(0.5)
+                check_times += 1
+
+        await asyncio.create_task(check_timer())
+        if task_req.result:
+            return task_req.result
+        else:
+            time_out_result = ComputeTaskResult()
+            time_out_result.result_code = ComputeTaskResultCode.TIMEOUT
+            time_out_result.set_from_task(task_req)
+            task_req.result = time_out_result
+            return time_out_result
+
+
+    async def do_llm_completion(self, prompt: AgentPrompt,resp_mode:str="text", mode_name: Optional[str]=None, max_token:int=0, inner_functions=None, timeout=60) -> str:
+        task_req = self.llm_completion(prompt, resp_mode,mode_name, max_token,inner_functions)
+        return await self._wait_task(task_req, timeout)
+
+
+    def text_embedding(self,input:str,model_name:Optional[str] = None):
+        task_req = ComputeTask()
+        task_req.set_text_embedding_params(input,model_name)
+        self.run(task_req)
+        return task_req
+
+    async def do_text_embedding(self,input:str,model_name:Optional[str] = None) -> [float]:
+        task_req = self.text_embedding(input,model_name)
+        task_result = await self._wait_task(task_req)
+
+        if task_req.state == ComputeTaskState.DONE:
+            return task_result.result.get("content")
+        else:
+            logging.warning(f"do_text_embedding error: {task_req.error_str},input: {input}")
+        return None
+
+    def image_embedding(self,input:ObjectID,model_name:Optional[str] = None):
+        task_req = ComputeTask()
+        task_req.set_image_embedding_params(input,model_name)
+        self.run(task_req)
+        return task_req
+
+    async def do_image_embedding(self,input:ObjectID,model_name:Optional[str] = None) -> [float]:
+        task_req = self.image_embedding(input,model_name)
+        task_result = await self._wait_task(task_req)
+
+        if task_req.state == ComputeTaskState.DONE:
+            return task_result.result.get("content")
+
+        return None
+
+    async def do_text_to_speech(self,
+                       input:str,
+                       language_code:Optional[str] = None,
+                       gender: Optional[str] = None,
+                       age: Optional[str] = None,
+                       voice_name: Optional[str] = None,
+                       tone: Optional[str] = None,
+                       model_name: Optional[str] = None):
+        task_req = ComputeTask()
+        task_req.params["text"] = input
+        task_req.params["language_code"] = language_code
+        task_req.params["gender"] = gender
+        task_req.params["age"] = age
+        task_req.params["voice_name"] = voice_name
+        task_req.params["tone"] = tone
+        task_req.params["model_name"] = model_name
+        task_req.task_type = ComputeTaskType.TEXT_2_VOICE
+        self.run(task_req)
+
+        task_result = await self._wait_task(task_req)
+
+        if task_req.state == ComputeTaskState.DONE:
+            return task_result.result
+
+    async def do_speech_to_text(self,
+                                audio: str,
+                                model: str,
+                                prompt: Optional[str],
+                                response_format: Optional[str]):
+        task_req = ComputeTask()
+        task_req.params["file"] = audio
+        task_req.params["model_name"] = model
+        task_req.params["prompt"] = prompt
+        task_req.params["response_format"] = response_format
+        task_req.task_type = ComputeTaskType.VOICE_2_TEXT
+
+        self.run(task_req)
+
+        task_result = await self._wait_task(task_req)
+
+        if task_req.state == ComputeTaskState.DONE:
+            return task_result
+
+    def text_2_image(self, prompt:str, model_name:Optional[str] = None, negative_prompt = None):
+        task = ComputeTask()
+        task.set_text_2_image_params(prompt,model_name, negative_prompt)
+        self.run(task)
+        return task
+
+    async def do_text_2_image(self, prompt:str, model_name:Optional[str] = None, negative_prompt = None) -> ComputeTaskResult:
+        task = self.text_2_image(prompt,model_name, negative_prompt)
+        task = await self._wait_task(task)
+
+        return task.result
+        # if task_req.state == ComputeTaskState.DONE:
+        #     return None, task_result
+
+    def image_2_text(self, image_path: str, prompt:str, model_name:Optional[str] = None, negative_prompt = None):
+        task = ComputeTask()
+        task.set_image_2_text_params(image_path,prompt,model_name, negative_prompt)
+        self.run(task)
+        return task
+    async def do_image_2_text(self, image_path: str, prompt:str, model_name:Optional[str] = None, negative_prompt = None) -> ComputeTaskResult:
+        task = self.image_2_text(image_path,prompt, model_name, negative_prompt)
+        task = await self._wait_task(task)
+        return task.result
+

src/aios/frame/compute_node.py

@@ -0,0 +1,53 @@
+from abc import ABC, abstractmethod
+
+from ..proto.compute_task import ComputeTask, ComputeTaskType
+
+class ComputeNode(ABC):
+    def __init__(self) -> None:
+        self.node_id = "default"
+        self.enable = True
+
+    @abstractmethod
+    async def push_task(self, task: ComputeTask, proiority: int = 0):
+        pass
+
+    @abstractmethod
+    async def remove_task(self, task_id: str):
+        pass
+
+    @abstractmethod
+    def get_task_state(self, task_id: str):
+        pass
+
+    @abstractmethod
+    def display(self) -> str:
+        pass
+
+    @abstractmethod
+    def get_capacity(self):
+        pass
+
+    @abstractmethod
+    def is_support(self, task: ComputeTask) -> bool:
+        pass
+
+    @abstractmethod
+    def is_local(self) -> bool:
+        pass
+
+    # the hit weight when select this node in schedule
+    def weight(self) -> int:
+        return 1
+    
+    def is_trusted(self) -> bool:
+        return True
+
+    def get_fee_type(self) -> str:
+        return "free"
+
+class LocalComputeNode(ComputeNode):
+    def display(self) -> str:
+        return super().display()
+
+    def is_local(self) -> bool:
+        return True

src/aios/frame/contact.py

@@ -0,0 +1,85 @@
+from typing import List
+import logging
+from datetime import datetime
+
+from ..proto.agent_msg import AgentMsg
+from .tunnel import AgentTunnel
+
+
+logger = logging.getLogger(__name__)    
+class Contact:
+    def __init__(self, name, phone=None, email=None, telegram=None,added_by=None, tags=[], notes=""):
+        self.name = name
+        self.phone = phone
+        self.email = email
+        self.telegram = telegram
+        self.added_by = added_by
+        self.tags = tags
+        self.notes = notes
+        self.is_family_member = False
+        self.active_tunnels = {}
+
+    def to_dict(self):
+        return {
+            "name": self.name,
+            "phone": self.phone,
+            "email": self.email,
+            "telegram" : self.telegram,
+
+            "added_by": self.added_by,
+            "tags": self.tags,
+            "notes": self.notes,
+            "now" : datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        }
+    
+    async def _process_msg(self,msg:AgentMsg):
+        tunnel : AgentTunnel = self.get_active_tunnel(msg.sender)
+        if tunnel is not None:
+            await tunnel.post_message(msg)
+            return None
+        else:
+            tunnel = await self.create_default_tunnel(msg.sender)
+            if tunnel is not None:
+                self.active_tunnels[msg.sender] = tunnel
+                await tunnel.post_message(msg)
+                return None
+        
+        
+        logger.warn(f"contact {self.name} cann't get tunnel,post message failed!")
+
+    def get_active_tunnel(self,agent_id) -> AgentTunnel:
+        tunnel = self.active_tunnels.get(agent_id)
+        return tunnel
+    
+    def set_active_tunnel(self,agent_id,tunnel:AgentTunnel):
+        self.active_tunnels[agent_id] = tunnel
+    
+    async def create_default_tunnel(self,agent_id:str) -> AgentTunnel:
+        from .email_tunnel import EmailTunnel
+
+        result_tunnels = AgentTunnel.get_tunnel_by_agentid(agent_id)
+        for tunnel in result_tunnels:
+            if isinstance(tunnel,EmailTunnel):
+                return tunnel
+                    
+        return None
+
+    @classmethod
+    def from_dict(cls, data):
+        return Contact(data.get("name"), data.get("phone"), data.get("email"), data.get("telegram"),data.get("added_by"), data.get("tags"), data.get("notes"))
+
+class FamilyMember(Contact):
+    def __init__(self, name, relationship,phone=None, email=None,telegram=None):
+        super().__init__(name, phone, email, telegram)
+        self.name = name
+        self.relationship = relationship  
+        self.is_family_member = True
+
+    def to_dict(self):
+        result = super().to_dict()
+        result["relationship"] = self.relationship
+        return result
+
+    @classmethod
+    def from_dict(cls, data):
+        return FamilyMember(data.get("name"),data.get("relationship"),data.get("phone"), data.get("email"),data.get("telegram"))

src/aios/frame/contact_manager.py

@@ -0,0 +1,129 @@
+from typing import List
+import toml
+import time
+import logging
+
+from datetime import datetime
+from ..proto.agent_msg import AgentMsg
+from .tunnel import AgentTunnel
+from .contact import Contact,FamilyMember
+
+
+logger = logging.getLogger(__name__)    
+
+
+class ContactManager:
+    _instance = None
+    @classmethod
+    def get_instance(cls,filename=None) -> "ContactManager":
+        if cls._instance is None:
+            cls._instance = ContactManager(str(filename))
+        return cls._instance
+
+    def __init__(self, filename="contacts.toml"):
+        self.filename = filename
+        self.contacts = []
+        self.family_members = []    
+
+        self.is_auto_create_contact_from_telegram = True
+    
+    def load_data(self):
+        try:
+            with open(self.filename, "r") as f:
+                config =  toml.load(f)
+                return self.load_from_config(config)
+        except FileNotFoundError:
+            return {}
+        
+    def load_from_config(self,config_data:dict):
+        self.contacts = [Contact.from_dict(item) for item in config_data.get("contacts", [])]
+        self.family_members = [FamilyMember.from_dict(item) for item in config_data.get("family_members", [])]
+    
+    def save_data(self):
+        data = {
+            "contacts": [contact.to_dict() for contact in self.contacts],
+            "family_members": [member.to_dict() for member in self.family_members]
+        }
+        with open(self.filename, "w") as f:
+            toml.dump(data, f)
+
+    def set_contact(self, name:str, new_contact:Contact):
+        assert name == new_contact.name
+        for i, contact in enumerate(self.contacts):
+            if contact.name == name:
+                self.contacts[i] = new_contact
+                self.save_data()
+                return True
+        for i, member in enumerate(self.family_members):
+            if member.name == name:
+                self.family_members[i] = new_contact
+                self.save_data()
+                return True
+            
+        return False
+
+    def add_contact(self, name:str, new_contact:Contact):
+        assert name == new_contact.name
+        self.contacts.append(new_contact)
+        self.save_data()
+
+    def remove_contact(self, name:str):
+        self.contacts = [contact for contact in self.contacts if contact.name != name]
+        self.save_data()
+
+    def find_contact_by_name(self, name:str):
+        for contact in self.contacts:
+            if contact.name == name:
+                return contact
+            
+        for member in self.family_members:
+            if member.name == name:
+                return member
+        return None
+    
+    def find_contact_by_telegram(self, telegram:str):
+        for contact in self.contacts:
+            if contact.telegram == telegram:
+                return contact
+        for member in self.family_members:
+            if member.telegram == telegram:
+                return member            
+        return None
+
+    def find_contact_by_email(self, email:str):
+        for contact in self.contacts:
+            if contact.email == email:
+                return contact
+        for member in self.family_members:
+            if member.email == email:
+                return member   
+        return None
+
+    def find_contact_by_phone(self, phone:str):
+        for contact in self.contacts:
+            if contact.phone == phone:
+                return contact
+        for member in self.family_members:
+            if member.phone == phone:
+                return member   
+        return None
+
+
+    def add_family_member(self, name, new_member:FamilyMember):
+        assert name == new_member.name
+        self.family_members.append(new_member)
+        self.save_data()
+
+    def list_contacts(self):
+        return self.contacts
+
+    def list_family_members(self):
+        return self.family_members
+    
+    #def register_to_ai_bus(self, ai_bus:AIBus):
+    #    ai_bus.register_message_handler("contact_manager", self.process_msg)
+    
+
+    #async def process_msg(self,msg:AgentMsg):
+    #    # forword message to contact    
+    #    pass

src/aios/frame/queue_compute_node.py

@@ -0,0 +1,63 @@
+
+import asyncio
+from asyncio import Queue
+import logging
+from abc import abstractmethod
+
+from aios import ComputeTask, ComputeNode,ComputeTaskResult, ComputeTaskResultCode, ComputeTaskState, ComputeTaskType
+
+logger = logging.getLogger(__name__)
+
+class Queue_ComputeNode(ComputeNode):
+    def __init__(self):
+        super().__init__()
+        self.task_queue = Queue()
+        self.is_start = False
+
+    @abstractmethod
+    async def execute_task(self, task: ComputeTask)->ComputeTaskResult:
+        pass
+
+    async def push_task(self, task: ComputeTask, proiority: int = 0):
+        logger.info(f"{self.display()} push task: {task.display()}")
+        self.task_queue.put_nowait(task)
+
+    async def remove_task(self, task_id: str):
+        pass
+
+    async def _run_task(self, task: ComputeTask):
+        task.state = ComputeTaskState.RUNNING
+
+        result = ComputeTaskResult()
+        result.result_code = ComputeTaskResultCode.ERROR
+        result.set_from_task(task)
+        result.worker_id = self.node_id
+
+        real_result = await self.execute_task(task)
+
+        if real_result:
+            if real_result.result_code == ComputeTaskResultCode.OK:
+                task.state = ComputeTaskState.DONE
+            else:
+                task.state = ComputeTaskState.ERROR
+            return real_result
+        else:
+            task.state = ComputeTaskState.ERROR
+            return result
+
+    def start(self):
+        if self.is_start is True:
+            return
+        self.is_start = True
+
+        async def _run_task_loop():
+            while True:
+                task = await self.task_queue.get()
+                logger.info(f"openai_node get task: {task.display()}")
+                await self._run_task(task)
+
+        asyncio.create_task(_run_task_loop())
+
+
+    def get_task_state(self, task_id: str):
+        pass

src/aios/frame/tunnel.py

@@ -0,0 +1,102 @@
+from abc import ABC, abstractmethod
+import logging
+from typing import Coroutine
+
+from ..proto.agent_msg import AgentMsg
+from .bus import AIBus
+
+logger = logging.getLogger(__name__)
+
+class AgentTunnel(ABC):
+    _all_loader = {}
+    _all_tunnels = {}
+    @classmethod
+    def register_loader(cls,tunnel_type:str,loader:Coroutine) -> None:
+        cls._all_loader[tunnel_type] = loader
+
+
+    @classmethod
+    async def load_all_tunnels_from_config(cls,config:dict) -> None:
+        for tunnel_id,tunnel_config in config.items():
+            loader = cls._all_loader.get(tunnel_config["type"])
+            tid = tunnel_config.get("tunnel_id")
+            if tid is not None:
+                if tunnel_id != tid:
+                    logger.warning(f"load tunnel {tunnel_id} error,{tunnel_id} != {tid} in config!")
+                    continue
+            else:
+                tunnel_config["tunnel_id"] = tunnel_id
+
+            if loader is not None:
+                tunnel = await loader(tunnel_config)
+                if tunnel is not None:
+                    cls._all_tunnels[tunnel_id] = tunnel
+                    tunnel.connect_to(AIBus.get_default_bus(),tunnel.target_id)
+                    await tunnel.start()
+                else:
+                    logger.error(f"load tunnel {tunnel_id} failed") 
+            else:
+                logger.error(f"load tunnel {tunnel_id} failed,loader not found")
+
+    @classmethod
+    async def load_tunnel_from_config(cls,tunnel_config:dict):
+        loader = cls._all_loader.get(tunnel_config["type"])
+        if loader is not None:
+            tunnel = await loader(tunnel_config)
+            if tunnel is not None:
+                cls._all_tunnels[tunnel.tunnel_id] = tunnel
+                tunnel.connect_to(AIBus.get_default_bus(),tunnel.target_id)
+                await tunnel.start()
+                return True
+            else:
+                logger.error(f"load tunnel {tunnel_config['tunnel_id']} failed") 
+        else:
+            logger.error(f"load tunnel {tunnel_config['type']} failed,loader not found")
+
+        return False
+    
+    @classmethod
+    async def get_tunnel_by_agentid(cls,agent_id:str):
+        result = []
+        for tunnel in cls._all_tunnels.values():
+            if tunnel.target_id == agent_id:
+                result.append(tunnel)
+        return result
+    
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.tunnel_id = None
+        self.target_id = None
+        self.target_type = None
+        self.ai_bus = None
+        self.is_connected = False
+
+    def connect_to(self, ai_bus:AIBus,target_id: str) -> None:
+        """
+        Connect to the agent with the given id
+        """
+        if self.is_connected:
+            logger.warning(f"tunnel {self.tunnel_id} is already connected to {self.target_id}")
+            return
+        self.target_id = target_id
+        self.target_type = "agent"
+        self.ai_bus = ai_bus
+        self.is_connected = True
+
+    @abstractmethod
+    def post_message(self, msg: AgentMsg) -> None:
+        pass
+        
+
+    @abstractmethod
+    async def start(self) -> bool:
+        pass
+
+    @abstractmethod
+    async def close(self) -> None:
+        pass
+
+    @abstractmethod
+    async def _process_message(self, msg: AgentMsg) -> None:
+        pass

src/aios/knowledge/__init__.py

@@ -0,0 +1,6 @@
+from .object import *
+from .vector import *
+from .data import *
+from .store import KnowledgeStore
+from .core_object import *
+from .pipeline import *

src/aios/knowledge/core_object/__init__.py

@@ -0,0 +1,5 @@
+from .document_object import DocumentObject, DocumentObjectBuilder
+from .image_object import ImageObject, ImageObjectBuilder
+from .video_object import VideoObject, VideoObjectBuilder
+from .rich_text_object import RichTextObject, RichTextObjectBuilder
+from .email_object import EmailObject, EmailObjectBuilder

src/aios/knowledge/core_object/document_object.py

@@ -0,0 +1,60 @@
+from ..object import KnowledgeObject, ObjectRelationStore
+from ..data import ChunkList, ChunkListWriter
+from ..object import ObjectType
+
+# desc
+#   meta
+#   hash: "file-hash",
+#   tags: {}
+# body
+#   chunk_list: [chunk_id, chunk_id, ...]
+
+
+class DocumentObject(KnowledgeObject):
+    def __init__(self, meta: dict, tags: dict, chunk_list: ChunkList):
+        desc = dict()
+        body = dict()
+        desc["meta"] = meta
+        desc["tags"] = tags
+        desc["hash"] = chunk_list.hash.to_base58()
+        body["chunk_list"] = chunk_list.chunk_list
+
+        super().__init__(ObjectType.Document, desc, body)
+
+    def get_meta(self):
+        return self.desc["meta"]
+
+    def get_tags(self):
+        return self.desc["tags"]
+
+    def get_hash(self):
+        return self.desc["hash"]
+
+    def get_chunk_list(self):
+        return self.body["chunk_list"]
+
+
+class DocumentObjectBuilder:
+    def __init__(self, meta: dict, tags: dict, text: str):
+        self.meta = meta
+        self.tags = tags
+        self.text = text
+
+    def set_meta(self, meta: dict):
+        self.meta = meta
+        return self
+
+    def set_text(self, text: str):
+        self.text = text
+        return self
+
+    def build(self, store) -> DocumentObject:
+        chunk_list = store.get_chunk_list_writer().create_chunk_list_from_text(self.text)
+        doc = DocumentObject(self.meta, self.tags, chunk_list)
+        doc_id = doc.calculate_id()
+        
+        # Add relation to store
+        for chunk_id in chunk_list.chunk_list:
+            store.get_relation_store().add_relation(chunk_id, doc_id)
+            
+        return doc

src/aios/knowledge/core_object/email_object.py

@@ -0,0 +1,159 @@
+from .rich_text_object import RichTextObject, RichTextObjectBuilder
+from ..object import ObjectID, ObjectType, KnowledgeObject
+from .document_object import DocumentObjectBuilder
+from .image_object import ImageObjectBuilder
+from .video_object import VideoObjectBuilder
+import os
+import json
+import logging
+
+
+class EmailObject(KnowledgeObject):
+    def __init__(self, meta: dict, tags: dict, rich_text: RichTextObject):
+        desc = dict()
+        body = dict()
+        desc["meta"] = meta
+        desc["tags"] = tags
+
+        # FIXME rich text content store in desc or body? which one is better?
+        body["content"] = rich_text
+
+        super().__init__(ObjectType.Email, desc, body)
+
+    def get_meta(self) -> dict:
+        return self.desc["meta"]
+
+    def get_tags(self) -> dict:
+        return self.desc["tags"]
+
+    def get_rich_text(self) -> RichTextObject:
+        return self.body["content"]
+
+
+"""
+EmailObject folder structure:
+.
+├── email.txt
+└── meta.json
+    ├── image
+    │   ├── image1.jpg
+    │   ├── image2.jpg
+    │   └── ...
+    ├── video
+    │   ├── video1.mp4
+    │   ├── video2.mv
+    │   └── ...
+    └── audio
+        ├── audio1.m4a
+        ├── audio2.flac
+        └── ...
+EmailObjectBuilder will read the target folder and build the EmailObject
+Store meta.json to meta in EmailObject
+Store email.txt to DocumentObject and RichTextObject in EmailObject
+Store very image file in image folder to ImageObject and RichTextObject in EmailObject, etc
+"""
+
+
+class EmailObjectBuilder:
+    def __init__(self, tags: dict, folder: str):
+        self.tags = tags
+        self.folder = folder
+
+    def set_tags(self, tags: dict):
+        self.tags = tags
+        return self
+
+    def set_folder(self, folder: str):
+        self.folder = folder
+        return self
+
+    def build(self, store) -> EmailObject:
+        
+        # Just get the object store and relation store from global KnowledgeStore
+        store = store.get_object_store()
+        relation = store.get_relation_store()
+        
+        # Read meta.json
+        meta = {}
+        meta_file = os.path.join(self.folder, "meta.json")
+        if os.path.exists(meta_file):
+            logging.info(f"Will read meta.json {meta_file}")
+            with open(meta_file, "r", encoding="utf-8") as f:
+                meta = json.load(f)
+        else:
+            logging.info(f"Meta file missing! {meta_file}")
+
+        # Read email.txt
+        documents = {}
+        content_file = os.path.join(self.folder, "email.txt")
+        if os.path.exists(content_file):
+            logging.info(f"Will read email.txt {content_file}")
+
+            try:
+                with open(content_file, "r", encoding="utf-8") as f:
+                    text = f.read()
+
+                document = DocumentObjectBuilder({}, {}, text).build()
+                document_id = document.calculate_id()
+                store.put_object(document_id, document.encode())
+                documents = {"email.txt": document_id}
+            except Exception as e:
+                logging.error(f"Failed to read email.txt {content_file} {e}")
+        else:
+            logging.info(f"Content file missing! {content_file}")
+
+        # Process image files
+        images = {}
+        image_dir = os.path.join(self.folder, "image")
+        if os.path.exists(image_dir):
+            for image_file in os.listdir(image_dir):
+                image_path = os.path.join(image_dir, image_file)
+                logging.info(f"Will read image file {image_path}")
+
+                try:
+                    image = ImageObjectBuilder({}, {}, image_path).build()
+                    image_id = image.calculate_id()
+                    store.put_object(image_id, image.encode())
+                    images[image_file] = image_id
+                except Exception as e:
+                    logging.error(f"Failed to read image file {image_path} {e}")
+                    continue
+
+        # Process video files
+        videos = {}
+        video_dir = os.path.join(self.folder, "video")
+        if os.path.exists(video_dir):
+            for video_file in os.listdir(video_dir):
+                video_path = os.path.join(video_dir, video_file)
+                logging.info(f"Will read video file {video_path}")
+
+                try:
+                    video = VideoObjectBuilder({}, {}, video_path).build()
+                    video_id = video.calculate_id()
+                    store.put_object(video_id, video.encode())
+                    videos[video_file] = video_id
+                except Exception as e:
+                    logging.error(f"Failed to read video file {video_path} {e}")
+                    continue
+
+        # Create RichTextObject
+        rich_text = RichTextObject(images, videos, documents)
+        rich_text_id = rich_text.calculate_id()
+        
+        # build relations with rich_text
+        for image_id in images.values():
+            relation.add_relation(image_id, rich_text_id)
+        for video_id in videos.values():
+            relation.add_relation(video_id, rich_text_id)
+        for document_id in documents.values():
+            relation.add_relation(document_id, rich_text_id)
+            
+        # Create EmailObject
+        email_object = EmailObject(meta, {}, rich_text)
+        email_object_id = email_object.calculate_id()
+        store.put_object(email_object_id, email_object.encode())
+        
+        # build relations with email_object
+        relation.add_relation(rich_text_id, email_object_id)
+        
+        return email_object

src/aios/knowledge/core_object/image_object.py

@@ -0,0 +1,95 @@
+from ..object import KnowledgeObject
+from ..data import ChunkList, ChunkListWriter
+from ..object import ObjectType
+import os
+
+# desc
+#   meta
+#   tags
+#   hash: "file-hash",
+#   exif: {}
+# body
+#   chunk_list: [chunk_id, chunk_id, ...]
+
+
+class ImageObject(KnowledgeObject):
+    def __init__(self, meta: dict, tags: dict, exif: dict, file_size: int, chunk_list: ChunkList):
+        desc = dict()
+        body = dict()
+        desc["meta"] = meta
+        desc["exif"] = exif
+        desc["tags"] = tags
+        desc["hash"] = chunk_list.hash.to_base58()
+        desc["file_size"] = file_size
+        body["chunk_list"] = chunk_list.chunk_list
+
+        super().__init__(ObjectType.Image, desc, body)
+
+    def get_meta(self) -> dict:
+        return self.desc["meta"]
+
+    def get_exif(self) -> dict:
+        return self.desc["exif"]
+
+    def get_tags(self) -> dict:
+        return self.desc["tags"]
+    
+    def get_hash(self) -> str:
+        return self.desc["hash"]
+
+    def get_file_size(self) -> int:
+        return self.desc["file_size"]
+    
+    def get_chunk_list(self) -> ChunkList:
+        return self.body["chunk_list"]
+
+
+from PIL import Image
+from PIL.ExifTags import TAGS
+
+
+def get_exif_data(image_path: str):
+    with Image.open(image_path) as image:
+        exif_data = image._getexif()
+
+    if exif_data is not None:
+        return {
+            TAGS.get(key): exif_data[key]
+            for key in exif_data.keys()
+            if key in TAGS and isinstance(exif_data[key], str)
+        }
+    else:
+        return {}
+
+
+class ImageObjectBuilder:
+    def __init__(self, meta: dict, tags: dict, image_file: str):
+        self.meta = meta
+        self.tags = tags
+        self.image_file = image_file
+        self.restore_file = False
+
+    def set_meta(self, meta: dict):
+        self.meta = meta
+        return self
+    
+    def set_tags(self, tags: dict):
+        self.tags = tags
+        return self
+
+    def set_image_file(self, image_file: str):
+        self.image_file = image_file
+        return self
+
+    def set_restore_file(self, restore_file: bool):
+        self.restore_file = restore_file
+        return self
+
+    def build(self, store) -> ImageObject:
+        
+        file_size = os.path.getsize(self.image_file)
+        chunk_list = store.get_chunk_list_writer().create_chunk_list_from_file(
+            self.image_file, 1024 * 1024 * 4, self.restore_file
+        )
+        exif = get_exif_data(self.image_file)
+        return ImageObject(self.meta, self.tags, exif, file_size, chunk_list)

src/aios/knowledge/core_object/rich_text_object.py

@@ -0,0 +1,80 @@
+from ..object.object_id import ObjectType
+from ..object import KnowledgeObject
+from ..data import ChunkList, ChunkListWriter
+from ..object import ObjectType
+from .video_object import VideoObjectBuilder, VideoObject
+from .image_object import ImageObjectBuilder, ImageObject
+from .document_object import DocumentObjectBuilder, DocumentObject
+
+class RichTextObject(KnowledgeObject):
+    def __init__(self, images: dict = {}, videos: dict = {}, documents: dict = {}, rich_texts: dict = {}):
+        desc = dict()
+        desc["images"] = images
+        desc["videos"] = videos
+        desc["documents"] = documents
+        desc["rich_texts"] = rich_texts
+        
+        super().__init__(ObjectType.RichText, desc)
+        
+     
+    def add_image_with_key(self, key, image_object: ImageObject):
+        assert self.desc["images"][key] == None
+        self.desc["images"][key] = image_object
+           
+    def add_image(self, image_object: ImageObject):
+        self.desc["images"][image_object.object_id()] = image_object
+     
+    def get_image_with_key(self, key) -> ImageObject:
+        return self.desc["images"][key]
+
+    def get_images(self) -> dict:
+        return self.desc["images"]
+       
+    def add_video_with_key(self, key, video_object: VideoObject):
+        assert self.desc["videos"][key] == None
+        self.desc["videos"][key] = video_object
+        
+    def add_video(self, video_object: VideoObject):
+        self.desc["videos"][video_object.object_id()] = video_object
+        
+    def get_video_with_key(self, key) -> VideoObject:
+        return self.desc["videos"][key]
+    
+    def get_videos(self) -> dict:    
+        return self.desc["videos"]
+        
+        
+    def add_document_with_key(self, key, document_object: DocumentObject):
+        assert self.desc["documents"][key] == None
+        self.desc["documents"][key] = document_object
+        
+    def add_document(self, document_object: DocumentObject):
+        self.desc["documents"][document_object.object_id()] = document_object
+      
+    def get_document_with_key(self, key) -> DocumentObject:
+        return self.desc["documents"][key]
+    
+    def get_documents(self) -> dict:
+        return self.desc["documents"]
+      
+    def add_rich_text_with_key(self, key, rich_text_object):
+        assert self.desc["rich_texts"][key] == None
+        self.desc["rich_texts"][key] = rich_text_object
+        
+    def add_rich_text(self, rich_text_object):
+        self.desc["rich_texts"][rich_text_object.object_id()] = rich_text_object
+        
+    def get_rich_text_with_key(self, key):
+        return self.desc["rich_texts"][key]
+    
+    def get_rich_texts(self) -> dict:
+        return self.desc["rich_texts"]
+    
+    
+class RichTextObjectBuilder:
+    def __init__(self, folder: str):
+        self.folder = folder
+        
+    def build(self) -> RichTextObject:
+        # TODO
+        return RichTextObject()

src/aios/knowledge/core_object/video_object.py

@@ -0,0 +1,83 @@
+from ..object import KnowledgeObject
+from ..data import ChunkList, ChunkListWriter
+from ..object import ObjectType
+
+# desc
+#   meta
+#   tags
+#   hash: "file-hash",
+#   info: {}
+# body
+#   chunk_list: [chunk_id, chunk_id, ...]
+
+
+class VideoObject(KnowledgeObject):
+    def __init__(self, meta: dict, tags: dict, info: dict, chunk_list: ChunkList):
+        desc = dict()
+        body = dict()
+        desc["meta"] = meta
+        desc["tags"] = tags
+        desc["info"] = info
+        desc["hash"] = chunk_list.hash.to_base58()
+        body["chunk_list"] = chunk_list.chunk_list
+
+        super().__init__(ObjectType.Video, desc, body)
+
+    def get_meta(self):
+        return self.desc["meta"]
+
+    def get_tags(self):
+        return self.desc["tags"]
+
+    def get_info(self):
+        return self.desc["info"]
+
+    def get_hash(self):
+        return self.desc["hash"]
+
+    def get_chunk_list(self):
+        return self.body["chunk_list"]
+
+
+from moviepy.editor import VideoFileClip
+
+
+def get_video_info(video_path: str) -> dict:
+    clip = VideoFileClip(video_path)
+    return {
+        "duration": clip.duration,  # Duration in seconds
+        "fps": clip.fps,  # Frames per second
+        "nframes": clip.reader.nframes,  # Total number of frames
+        "size": clip.size,  # Size of the frames (width, height)
+    }
+
+
+class VideoObjectBuilder:
+    def __init__(self, meta: dict, tags: dict, video_file: str):
+        self.meta = meta
+        self.tags = tags
+        self.video_file = video_file
+        self.restore_file = False
+
+    def set_meta(self, meta: dict):
+        self.meta = meta
+        return self
+    
+    def set_tags(self, tags: dict):
+        self.tags = tags
+        return self
+
+    def set_video_file(self, video_file: str):
+        self.video_file = video_file
+        return self
+
+    def set_restore_file(self, restore_file: bool):
+        self.restore_file = restore_file
+        return self
+
+    def build(self, store) -> VideoObject:
+        chunk_list = store.get_chunk_list_writer().create_chunk_list_from_file(
+            self.video_file, 1024 * 1024 * 4, self.restore_file
+        )
+        info = get_video_info(self.video_file)
+        return VideoObject(self.meta, self.tags, info, chunk_list)

src/aios/knowledge/data/__init__.py

@@ -0,0 +1,6 @@
+from .chunk import ChunkID, PositionType, PositionFileRange
+from .tracker import ChunkTracker
+from .chunk_store import ChunkStore
+from .writer import ChunkListWriter
+from .chunk_list import ChunkList
+from .reader import ChunkReader, Chunk

src/aios/knowledge/data/chunk.py

@@ -0,0 +1,43 @@
+from enum import IntEnum
+from ..object import ObjectID
+
+ChunkID = ObjectID
+
+class PositionType(IntEnum):
+    Unknown = 1
+    Device = 2
+    File = 3
+    FileRange = 4
+    ChunkStore = 5
+
+
+class PositionFileRange:
+    def __init__(self, path: str, range_begin: int, range_end: int):
+        self.path = path
+        self.range_begin = range_begin
+        self.range_end = range_end
+
+    def encode(self):
+        return f"{self.range_begin}:{self.range_end}:{self.path}"
+
+    @staticmethod
+    def decode(value: str):
+        parts = value.split(":")
+        if len(parts) < 3:
+            raise ValueError("Invalid input string")
+
+        try:
+            range_begin = int(parts[0])
+            range_end = int(parts[1])
+        except ValueError as e:
+            raise ValueError("Invalid range_begin or range_end string") from e
+
+        path = ":".join(parts[2:])
+        return PositionFileRange(path, range_begin, range_end)
+
+    def __str__(self):
+        return self.encode()
+
+    @staticmethod
+    def from_string(value: str):
+        return PositionFileRange.decode(value)

src/aios/knowledge/data/chunk_list.py

@@ -0,0 +1,14 @@
+from ..object import HashValue
+from .chunk import ChunkID
+from typing import List
+
+class ChunkList:
+    def __init__(self, chunk_list: List[ChunkID], hash: HashValue):
+        self.chunk_list = chunk_list
+        self.hash = hash
+
+    def __str__(self):
+        return self.hash.to_base58()
+
+    def __repr__(self):
+        return f"chunk_list: {self.chunk_list}, hash: {self.hash}"

src/aios/knowledge/data/chunk_store.py

@@ -0,0 +1,28 @@
+import os
+import logging
+from ..object import FileBlobStorage
+from .chunk import ChunkID
+
+
+class ChunkStore:
+    def __init__(self, root_dir: str):
+        logging.info(f"will init chunk store, root_dir={root_dir}")
+
+        if not os.path.exists(root_dir):
+            os.makedirs(root_dir)
+            
+        self.root = root_dir
+        self.blob = FileBlobStorage(root_dir)
+
+    def put_chunk(self, chunk_id: ChunkID, contents: bytes):
+        self.blob.put(chunk_id, contents)
+
+    def get_chunk(self, chunk_id: ChunkID) -> bytes:
+        return self.blob.get(chunk_id)
+
+    def delete_chunk(self, chunk_id: ChunkID):
+        self.blob.delete(chunk_id)
+      
+    def get_chunk_file_path(self, chunk_id: ChunkID) -> str:
+        return self.blob.get_full_path(chunk_id, False)
+    

src/aios/knowledge/data/reader.py

@@ -0,0 +1,95 @@
+from .chunk import ChunkID, PositionType, PositionFileRange
+from .chunk_store import ChunkStore
+from .tracker import ChunkTracker
+from ..object import HashValue
+import logging
+from typing import List
+import hashlib
+
+class Chunk:
+    def __init__(self, file_path: str, range_start: int, size: int = -1):
+        self.file_path = file_path
+        self.range_start = range_start
+        self.size = size
+
+    def read(self) -> bytes:
+        with open(self.file_path, 'rb') as f:
+            f.seek(self.range_start)
+            return f.read(self.size)
+        
+        
+
+class ChunkReader:
+    def __init__(self, chunk_store: ChunkStore, chunk_tracker: ChunkTracker):
+        self.chunk_store = chunk_store
+        self.chunk_tracker = chunk_tracker
+
+    def get_chunk(self, chunk_id: ChunkID) -> Chunk:
+        positions = self.chunk_tracker.get_position(chunk_id)
+        logging.info(f"chunk positions: {chunk_id}, {positions}")
+        
+        if positions is None:
+            logging.warning(f"chunk not found: {chunk_id}")
+            return None 
+       
+        if len(positions) == 0:
+            logging.warning(f"chunk not found: {chunk_id}")
+            return None 
+
+        for pos in positions:
+            [position, position_type] = pos
+            logging.info(f"chunk position: {chunk_id}, {position}, {position_type}")
+            if position_type == PositionType.ChunkStore:
+                file_path = self.chunk_store.get_chunk_file_path(chunk_id)
+                return Chunk(file_path, 0, -1)
+            elif position_type == PositionType.File:
+                return Chunk(position, 0, -1)
+            elif position_type == PositionType.FileRange:
+                file_range = PositionFileRange.decode(position)
+                return Chunk(file_range.path, file_range.range_begin, file_range.range_end - file_range.range_begin)
+            else:
+                raise ValueError(f"invalid position type: {position_type}")
+
+        logging.error(f"chunk not found: {chunk_id}")
+        return None
+    
+    def get_chunk_list(self, chunk_list: List[ChunkID]) -> List[Chunk]:
+        return [self.get_chunk(chunk_id) for chunk_id in chunk_list]
+        
+    def read_chunk_list(self, chunk_ids: List[ChunkID]) -> bytes:
+        for chunk_id in chunk_ids:
+            chunk = self.get_chunk(chunk_id)
+            if chunk is None:
+                raise ValueError(f"chunk not found: {chunk_id}")
+            
+            yield chunk.read()
+    
+    def read_chunk_list_to_single_bytes(self, chunk_ids: List[ChunkID]) -> bytes:
+        chunks = []
+        for chunk in self.read_chunk_list(chunk_ids):
+            chunks.append(chunk)
+            
+        image_data = b''.join(chunks)
+        return image_data
+    
+    def read_text_chunk_list(self, chunk_ids: List[ChunkID]) -> str:
+        for chunk_id in chunk_ids:
+            chunk = self.get_chunk(chunk_id)
+            if chunk is None:
+                raise ValueError(f"text chunk not found: {chunk_id}")
+            
+            yield chunk.read().decode("utf-8")
+                
+    def calc_file_hash(self, file_path: str) -> HashValue:
+        hash_obj = hashlib.sha256()
+        with open(file_path, "rb") as file:
+            while True:
+                chunk = file.read(1024 * 1024)
+                if not chunk:
+                    break
+                hash_obj.update(chunk)
+        return HashValue(hash_obj.digest())
+
+    def calc_text_hash(self, text: str) -> HashValue:
+        hash_obj = hashlib.sha256()
+        hash_obj.update(text.encode("utf-8"))

src/aios/knowledge/data/tracker.py

@@ -0,0 +1,71 @@
+import sqlite3
+import time
+import logging
+import os
+from .chunk import ChunkID, PositionType, PositionFileRange
+from typing import List, Tuple
+
+class ChunkTracker:
+    def __init__(self, root_dir: str):
+        if not os.path.exists(root_dir):
+            os.makedirs(root_dir)
+        file = os.path.join(root_dir, "chunk_tracker.db")
+        logging.info(f"will init chunk tracker, db={file}")
+
+        self.conn = sqlite3.connect(file)
+        self.cursor = self.conn.cursor()
+        self.cursor.execute(
+            """
+            CREATE TABLE IF NOT EXISTS chunks (
+                id TEXT NOT NULL,
+                pos TEXT NOT NULL,
+                pos_type TINYINT NOT NULL,
+                insert_time UNSIGNED BIG INT NOT NULL,
+                update_time UNSIGNED BIG INT NOT NULL,
+                flags INTEGER DEFAULT 0,
+                PRIMARY KEY(id, pos, pos_type)
+            )
+        """
+        )
+        self.conn.commit()
+
+    def add_position(
+        self, chunk_id: ChunkID, position: str, position_type: PositionType
+    ):
+        logging.debug(f"add chunk position: {chunk_id}, {position}, {position_type}")
+
+        insert_time = update_time = int(time.time())
+        self.cursor.execute(
+            """
+            INSERT OR REPLACE INTO chunks (id, pos, pos_type, insert_time, update_time)
+            VALUES (?, ?, ?, ?, ?)
+        """,
+            (
+                str(chunk_id),
+                position,
+                position_type.value,
+                insert_time,
+                update_time,
+            ),
+        )
+        self.conn.commit()
+
+    def remove_position(self, chunk_id: ChunkID):
+        logging.info(f"remove chunk position: {chunk_id}")
+
+        self.cursor.execute(
+            """
+            DELETE FROM chunks WHERE id = ?
+        """,
+            (str(chunk_id),),
+        )
+        self.conn.commit()
+
+    def get_position(self, chunk_id: ChunkID) -> List[Tuple[str, PositionType]]:
+        self.cursor.execute(
+            """
+            SELECT pos, pos_type FROM chunks WHERE id = ?
+        """,
+            (str(chunk_id),),
+        )
+        return self.cursor.fetchmany()

src/aios/knowledge/data/writer.py

@@ -0,0 +1,214 @@
+import os
+import hashlib
+import re
+import tiktoken
+import logging
+from typing import Callable, Iterable, Optional, Tuple, List
+from .chunk_store import ChunkStore
+from .chunk import ChunkID, PositionFileRange, PositionType
+from ..object import HashValue
+from .tracker import ChunkTracker
+from .chunk_list import ChunkList
+
+def _join_docs(docs: List[str], separator: str) -> Optional[str]:
+    text = separator.join(docs)
+    text = text.strip()
+    if text == "":
+        return None
+    else:
+        return text
+
+def _merge_splits(
+        splits: Iterable[str], 
+        separator: str, 
+        chunk_size: int, 
+        chunk_overlap: int,  
+        length_function: Callable[[str], int]
+    ) -> List[str]:
+    # We now want to combine these smaller pieces into medium size
+    # chunks to send to the LLM.
+    separator_len = length_function(separator)
+
+    docs = []
+    current_doc: List[str] = []
+    total = 0
+    for d in splits:
+        _len = length_function(d)
+        if (
+            total + _len + (separator_len if len(current_doc) > 0 else 0)
+            > chunk_size
+        ):
+            if total > chunk_size:
+                logging.warning(
+                    f"Created a chunk of size {total}, "
+                    f"which is longer than the specified {self._chunk_size}"
+                )
+            if len(current_doc) > 0:
+                doc = _join_docs(current_doc, separator)
+                if doc is not None:
+                    docs.append(doc)
+                # Keep on popping if:
+                # - we have a larger chunk than in the chunk overlap
+                # - or if we still have any chunks and the length is long
+                while total > chunk_overlap or (
+                    total + _len + (separator_len if len(current_doc) > 0 else 0)
+                    > chunk_size
+                    and total > 0
+                ):
+                    total -= length_function(current_doc[0]) + (
+                        separator_len if len(current_doc) > 1 else 0
+                    )
+                    current_doc = current_doc[1:]
+        current_doc.append(d)
+        total += _len + (separator_len if len(current_doc) > 1 else 0)
+    doc = _join_docs(current_doc, separator)
+    if doc is not None:
+        docs.append(doc)
+    return docs
+
+
+def _split_text_with_regex(
+    text: str, separator: str, keep_separator: bool
+) -> List[str]:
+    # Now that we have the separator, split the text
+    if separator:
+        if keep_separator:
+            # The parentheses in the pattern keep the delimiters in the result.
+            _splits = re.split(f"({separator})", text)
+            splits = [_splits[i] + _splits[i + 1] for i in range(1, len(_splits), 2)]
+            if len(_splits) % 2 == 0:
+                splits += _splits[-1:]
+            splits = [_splits[0]] + splits
+        else:
+            splits = re.split(separator, text)
+    else:
+        splits = list(text)
+    return [s for s in splits if s != ""]
+
+
+def _split_text(
+        text: str, 
+        separators: List[str], 
+        chunk_size: int, 
+        chunk_overlap: int, 
+        length_function: Callable[[str], int]
+    ) -> List[str]:
+
+    """Split incoming text and return chunks."""
+    final_chunks = []
+    # Get appropriate separator to use
+    separator = separators[-1]
+    new_separators = []
+    for i, _s in enumerate(separators):
+        _separator = re.escape(_s)
+        if _s == "":
+            separator = _s
+            break
+        if re.search(_separator, text):
+            separator = _s
+            new_separators = separators[i + 1 :]
+            break
+
+    keep_separator = True
+    _separator = re.escape(separator)
+    splits = _split_text_with_regex(text, _separator, keep_separator)
+
+    # Now go merging things, recursively splitting longer texts.
+    _good_splits = []
+    _separator = "" if keep_separator else separator
+    for s in splits:
+        if length_function(s) < chunk_size:
+            _good_splits.append(s)
+        else:
+            if _good_splits:
+                merged_text = _merge_splits(_good_splits, _separator, chunk_size, chunk_overlap, length_function)
+                final_chunks.extend(merged_text)
+                _good_splits = []
+            if not new_separators:
+                final_chunks.append(s)
+            else:
+                other_info = _split_text(s, new_separators, chunk_size, chunk_overlap, length_function)
+                final_chunks.extend(other_info)
+    if _good_splits:
+        merged_text = _merge_splits(_good_splits, _separator, chunk_size, chunk_overlap, length_function)
+        final_chunks.extend(merged_text)
+    return final_chunks
+
+class ChunkListWriter:
+    def __init__(self, chunk_store: ChunkStore, chunk_tracker: ChunkTracker):
+        self.chunk_store = chunk_store
+        self.chunk_tracker = chunk_tracker
+
+    def create_chunk_list_from_file(
+        self, file_path: str, chunk_size: int, restore: bool
+    ) -> ChunkList:
+        assert (
+            chunk_size % (1024 * 1024) == 0
+        ), "chunk size should be an integral multiple of 1MB"
+        chunk_list = []
+        hash_obj = hashlib.sha256()
+
+        with open(file_path, "rb") as file:
+            while True:
+                chunk = file.read(chunk_size)
+                if not chunk:
+                    break
+                
+                chunk_len = len(chunk)
+                chunk_id = ChunkID.hash_data(chunk)
+                chunk_list.append(chunk_id)
+
+                hash_obj.update(chunk)
+
+                if restore:
+                    self.chunk_tracker.add_position(
+                        chunk_id, file_path, PositionType.ChunkStore
+                    )
+                    self.chunk_store.put_chunk(chunk_id, chunk)
+                else:
+                    pos = file.tell()
+                    file_range = PositionFileRange(
+                        file_path, pos - chunk_len, pos
+                    )
+                    self.chunk_tracker.add_position(
+                        chunk_id, str(file_range), PositionType.FileRange
+                    )
+
+        file_hash = HashValue(hash_obj.digest())
+        # print(f"calc file hash: {file_path}, {file_hash}")
+        
+        return ChunkList(chunk_list, file_hash)
+
+    def create_chunk_list_from_text(
+        self, 
+        text: str, 
+        chunk_size: int = 4000, 
+        chunk_overlap: int = 200, 
+        separators: str = ["\n\n", "\n", " ", ""]
+    ) -> ChunkList:
+        enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
+
+        def length_function(text: str) -> int:
+            return len(
+                enc.encode(
+                    text,
+                    allowed_special=set(),
+                    disallowed_special="all",
+                )
+            )
+        
+        text_list = _split_text(text, separators, chunk_size, chunk_overlap, length_function)
+        chunk_list = []
+        hash_obj = hashlib.sha256()
+
+        for text in text_list:
+            chunk_bytes = text.encode("utf-8")
+            hash_obj.update(chunk_bytes)
+
+            chunk_id = ChunkID.hash_data(chunk_bytes)
+            chunk_list.append(chunk_id)
+            self.chunk_tracker.add_position(chunk_id, "", PositionType.ChunkStore)
+            self.chunk_store.put_chunk(chunk_id, chunk_bytes)
+
+        hash = HashValue(hash_obj.digest())
+        return ChunkList(chunk_list, hash)

src/aios/knowledge/object/__init__.py

@@ -0,0 +1,6 @@
+from .object import KnowledgeObject
+from .blob import FileBlobStorage
+from .hash import HashValue, hash_data
+from .relation import ObjectRelationStore
+from .object_store import ObjectStore
+from .object_id import ObjectID, ObjectType

src/aios/knowledge/object/blob.py

@@ -0,0 +1,63 @@
+import os
+import shutil
+from .object import ObjectID
+import logging
+logger = logging.getLogger(__name__)
+
+
+class FileBlobStorage:
+    def __init__(self, root):
+        self.root = root
+
+    def get_full_path(self, object_id: ObjectID, auto_create: bool = True):
+        if os.name == "nt":  # Windows
+            hash_str = object_id.to_base36()
+            len = 3
+        else:
+            hash_str = str(object_id)
+            len = 2
+
+        tmp, first = hash_str[:-len], hash_str[-len:]
+        second = tmp[-len:]
+
+        if os.name == "nt":  # Windows
+            if second in ["con", "aux", "nul", "prn"]:
+                second = tmp[-(len + 1) :]
+            if first in ["con", "aux", "nul", "prn"]:
+                first = f"{first}_"
+
+        path = os.path.join(self.root, first, second)
+        if auto_create and not os.path.exists(path):
+            os.makedirs(path)
+
+        path = os.path.join(path, hash_str)
+
+        return path
+
+    def write_sync(self, path: str, contents: bytes):
+        with open(path, "wb") as f:
+            f.write(contents)
+
+    def put(self, object_id: ObjectID, contents: bytes):
+        full_path = self.get_full_path(object_id)
+        if os.path.exists(full_path):
+            logger.warning(f"will replace object: {object_id}")
+            
+        self.write_sync(full_path, contents)
+
+    def get(self, object_id: ObjectID) -> bytes:
+        full_path = self.get_full_path(object_id)
+        if not os.path.exists(full_path):
+            return None
+        
+        with open(full_path, "rb") as f:
+            return f.read()
+
+    def delete(self, object_id: ObjectID):
+        full_path = self.get_full_path(object_id)
+        if os.path.exists(full_path):
+            os.remove(full_path)
+
+    def exists(self, object_id: ObjectID) -> bool:
+        full_path = self.get_full_path(object_id)
+        return os.path.exists(full_path)

src/aios/knowledge/object/hash.py

@@ -0,0 +1,42 @@
+import hashlib
+import base58
+import base36
+
+class HashValue:
+    def __init__(self, value: bytes):
+        assert len(value) == 32, "HashValue must be 32 bytes long"
+        self.value = value
+
+    def __str__(self) -> str:
+        return self.to_base58()
+        
+    @staticmethod
+    def hash_data(data):
+        return hash_data(data)
+
+    def to_base58(self):
+        return base58.b58encode(self.value).decode()
+
+    @staticmethod
+    def from_base58(s):
+        return HashValue(base58.b58decode(s))
+
+    def to_base36(self):
+        # Convert the bytes to int before encoding
+        num = int.from_bytes(self.value, 'big')
+        return base36.dumps(num)
+
+    @staticmethod
+    def from_base36(s):
+        # Decode to int and then convert to bytes
+        num = base36.loads(s)
+        return HashValue(num.to_bytes((num.bit_length() + 7) // 8, 'big'))
+    
+    
+HASH_VALUE_LEN = 32
+
+
+def hash_data(data: bytes):
+    sha256 = hashlib.sha256()
+    sha256.update(data)
+    return HashValue(sha256.digest())

src/aios/knowledge/object/object.py

@@ -0,0 +1,78 @@
+# define a object type enum
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from enum import Enum
+
+from .object_id import ObjectID, ObjectType
+import hashlib
+import json
+import pickle
+from typing import Any
+
+
+class ObjectEnhancedJSONEncoder(json.JSONEncoder):
+    def default(self, o: Any) -> Any:
+        if isinstance(o, ObjectID):
+            return o.to_base58()
+
+        return super().default(o)
+
+
+class KnowledgeObject(ABC):
+    def __init__(self, object_type: ObjectType, desc: dict = {}, body: dict = {}):
+        self.desc = desc
+        self.body = body
+        self.object_type = object_type
+
+    def get_object_type(self) -> ObjectType:
+        return self.object_type
+
+    def object_id(self) -> ObjectID:
+        return self.calculate_id()
+
+    def set_desc_with_key_value(self, key, value):
+        self.desc[key] = value
+
+    def get_desc_with_key(self, key):
+        return self.desc.get(key)
+
+    def get_desc(self) -> dict:
+        return self.desc
+
+    def set_body_with_key_value(self, key, value):
+        self.body[key] = value
+
+    def get_body_with_key(self, key):
+        return self.body.get(key)
+
+    def get_body(self) -> dict:
+        return self.body
+    
+    def get_summary(self) -> str:
+        return self.desc.get("summary")
+    
+    # def get_articl_catelog(self) -> str:
+    #     assert self.object_type == ObjectType.Document
+    #     return self.desc.get("catelog")
+    
+    # def get_article_full_content(self) -> str:
+    #     assert self.object_type == ObjectType.Document
+    #     return self.body
+
+    def calculate_id(self):
+        # Convert the object_type and desc to string and compute the SHA256 hash
+        data = json.dumps(
+            {"object_type": self.object_type, "desc": self.desc},
+            cls=ObjectEnhancedJSONEncoder,
+        )
+        sha256 = hashlib.sha256()
+        sha256.update(data.encode())
+        hash_bytes = sha256.digest()
+        return ObjectID(bytes([self.object_type]) + hash_bytes[1:])
+
+    def encode(self) -> bytes:
+        return pickle.dumps(self)
+
+    # @staticmethod
+    # def decode(data: bytes) -> "ImageObject":
+    #     return pickle.loads(data)

src/aios/knowledge/object/object_id.py

@@ -0,0 +1,69 @@
+# define a object type enum
+from abc import ABC, abstractmethod
+from enum import IntEnum
+from .hash import HashValue
+import base58
+import base36
+
+
+class ObjectType(IntEnum):
+    Chunk = 7
+    Image = 101
+    Video = 102
+    Document = 103
+    RichText = 104
+    Email = 105
+    UserDef = 200
+
+    def is_user_def(self) -> bool:
+        return self.value >= 200
+    
+    def get_user_def_type_code(self):
+        return (self.value - 200) if self.is_user_def() else None
+    
+    @classmethod
+    def from_user_def_type_code(cls, value):
+        return value + 200
+
+
+# define a object ID class to identify a object
+class ObjectID:  # pylint: disable=too-few-public-methods
+    def __init__(self, value: bytes):
+        assert len(value) == 32, "ObjectID must be 32 bytes long"
+        self.value = value
+
+    def __str__(self):
+        return self.to_base58()
+    
+    def to_base58(self):
+        return base58.b58encode(self.value).decode()
+
+    @staticmethod
+    def from_base58(s):
+        return ObjectID(base58.b58decode(s))
+
+    def to_base36(self):
+        # Convert the bytes to int before encoding
+        num = int.from_bytes(self.value, "big")
+        return base36.dumps(num)
+
+    @staticmethod
+    def from_base36(s):
+        # Decode to int and then convert to bytes
+        num = base36.loads(s)
+        return ObjectID(num.to_bytes((num.bit_length() + 7) // 8, "big"))
+
+    @staticmethod
+    def new_chunk_id(chunk_hash: HashValue):
+        assert len(chunk_hash.value) == 32, "ObjectID must be 32 bytes long"
+        return ObjectID(bytes([ObjectType.Chunk]) + chunk_hash.value[1:])
+    
+    def get_object_type(self) -> ObjectType:
+        return ObjectType(self.value[0])
+    
+    @staticmethod
+    def hash_data(data: bytes):
+        return ObjectID.new_chunk_id(HashValue.hash_data(data))
+    
+    def __eq__(self, other) -> bool:
+        return self.value == other.value

src/aios/knowledge/object/object_store.py

@@ -0,0 +1,25 @@
+import os
+import logging
+from .blob import FileBlobStorage
+from .object_id import ObjectID
+
+
+class ObjectStore:
+    def __init__(self, root_dir: str):
+        logging.info(f"will init object blob store, root_dir={root_dir}")
+
+        blob_dir = os.path.join(root_dir, "blob")
+        if not os.path.exists(blob_dir):
+            logging.info(f"will create blob dir: {blob_dir}")
+            os.makedirs(blob_dir)
+        self.blob = FileBlobStorage(blob_dir)
+
+    def put_object(self, object_id: ObjectID, contents: bytes):
+        logging.info(f"will put object: {object_id}")
+        self.blob.put(object_id, contents)
+
+    def get_object(self, object_id: ObjectID) -> bytes:
+        return self.blob.get(object_id)
+
+    def delete_object(self, object_id: ObjectID):
+        self.blob.delete(object_id)

src/aios/knowledge/object/relation.py

@@ -0,0 +1,104 @@
+# define a relation store class
+from .object_id import ObjectID
+import sqlite3
+from typing import List, Tuple, Optional
+import logging
+import os
+from enum import IntEnum
+
+
+class ObjectRelationType(IntEnum):
+    Parent = 1
+
+
+class ObjectRelationStore:
+    def __init__(self, root_dir: str):
+        if not os.path.exists(root_dir):
+            os.makedirs(root_dir)
+        file = os.path.join(root_dir, "relation.db")
+        logging.info(f"will init object relation store, db={file}")
+
+        self.conn = sqlite3.connect(file)
+        self.cursor = self.conn.cursor()
+        self.cursor.execute(
+            """
+            CREATE TABLE IF NOT EXISTS relations (
+                object_id TEXT,
+                assoc_id TEXT,
+                relation_type TEXT,
+                PRIMARY KEY (object_id, assoc_id, relation_type)
+            )
+        """
+        )
+
+    def add_relation(
+        self,
+        object_id: ObjectID,
+        assoc_id: ObjectID,
+        relation_type: ObjectRelationType = ObjectRelationType.Parent,
+    ):
+        if relation_type == None:
+            relation_type = ObjectRelationType.Parent
+
+        self.cursor.execute(
+            """
+            INSERT OR IGNORE INTO relations (object_id, assoc_id, relation_type)
+            VALUES (?, ?, ?)
+        """,
+            (str(object_id), str(assoc_id), relation_type.value),
+        )
+        self.conn.commit()
+
+    def get_related_objects(
+        self, object_id: ObjectID, relation_type: Optional[ObjectRelationType] = None
+    ) -> List[ObjectID]:
+        if relation_type:
+            self.cursor.execute(
+                """
+                SELECT assoc_id FROM relations WHERE object_id = ? AND relation_type = ?
+            """,
+                (str(object_id), relation_type.value),
+            )
+        else:
+            self.cursor.execute(
+                """
+                SELECT assoc_id FROM relations WHERE object_id = ?
+            """,
+                (str(object_id),),
+            )
+        return [ObjectID.from_base58(row[0]) for row in self.cursor.fetchall()]
+
+    def get_related_root_objects(
+        self, object_id: ObjectID, relation_type: Optional[ObjectRelationType] = None
+    ) -> List[ObjectID]:
+        root_objects = []
+        related_objects = self.get_related_objects(object_id, relation_type)
+        history = []
+        history.append(object_id)
+
+        while related_objects:
+            for obj in related_objects:
+                next_related_objects = self.get_related_objects(obj, relation_type)
+                if not next_related_objects:
+                    if obj not in root_objects:
+                        root_objects.append(obj)
+                else:
+                    for related_object in next_related_objects:
+                        if obj not in history:
+                            related_objects.append(related_object)
+                        else:
+                            logging.warning(
+                                f"loop detected: {obj} <-> {related_object}"
+                            )
+            related_objects = next_related_objects
+
+        return root_objects
+
+    def delete_relation(self, object_id: ObjectID):
+        self.cursor.execute(
+            """
+            DELETE FROM relations WHERE object_id = ?
+        """,
+            (str(object_id),),
+        )
+        self.conn.commit()

src/aios/knowledge/pipeline.py

@@ -0,0 +1,131 @@
+import datetime
+import sqlite3
+import os
+import logging
+from . import ObjectID, KnowledgeStore
+from enum import Enum
+
+class KnowledgePipelineJournal:
+    def __init__(self, time: datetime.datetime, object_id: str, input: str, parser: str):
+        self.time = time
+        self.object_id = None if object_id is None else ObjectID.from_base58(object_id)
+        self.input = input
+        self.parser = parser
+    
+    def is_finish(self) -> bool:
+        return self.object_id is None
+    
+    def get_object_id(self) -> ObjectID:
+        return self.object_id
+
+    def get_input(self) -> str:
+        return self.input
+    
+    def get_parser(self) -> str:
+        return self.parser
+    
+    def __str__(self) -> str:
+        if self.is_finish():
+            return f"{self.time}: finished)"
+        else:
+            return f"{self.time}: object:{self.object_id} input:{self.input}, parser:{self.parser})"
+
+# init sqlite3 client
+class KnowledgePipelineJournalClient:
+    def __init__(self, pipeline_path: str = None):
+        if not os.path.exists(pipeline_path):
+            os.makedirs(pipeline_path)
+        self.journal_path = os.path.join(pipeline_path, "journal.db")
+    
+        conn = sqlite3.connect(self.journal_path)
+        conn.execute(
+            '''CREATE TABLE IF NOT EXISTS journal (
+                id INTEGER PRIMARY KEY AUTOINCREMENT, 
+                time DATETIME DEFAULT CURRENT_TIMESTAMP,  
+                object_id TEXT, 
+                input TEXT, 
+                parser TEXT)'''
+        )
+        conn.commit()
+
+    def insert(self, object_id: ObjectID, input: str, parser: str, timestamp: datetime.datetime = None):
+        timestamp = datetime.datetime.now() if timestamp is None else timestamp
+        conn = sqlite3.connect(self.journal_path)
+        conn.execute(
+            "INSERT INTO journal (time, object_id, input, parser) VALUES (?, ?, ?, ?)",
+            (timestamp, str(object_id), input, parser),
+        )
+        conn.commit()
+          
+    def latest_journals(self, topn) -> [KnowledgePipelineJournal]:
+        conn = sqlite3.connect(self.journal_path)
+        cursor = conn.cursor()
+        cursor.execute("SELECT * FROM journal ORDER BY id DESC LIMIT ?", (topn,))
+        return [KnowledgePipelineJournal(time, object_id, input, parser) for (_, time, object_id, input, parser) in cursor.fetchall()]
+
+class KnowledgePipelineEnvironment:
+    def __init__(self, pipeline_path: str):
+        self.knowledge_store = KnowledgeStore()
+        if not os.path.exists(pipeline_path):
+            os.makedirs(pipeline_path)
+        self.pipeline_path = pipeline_path
+        self.journal = KnowledgePipelineJournalClient(pipeline_path)
+        self.logger = logging.getLogger()
+
+    def get_journal(self) -> KnowledgePipelineJournalClient:
+        return self.journal
+    
+    def get_knowledge_store(self) -> KnowledgeStore:
+        return self.knowledge_store
+    
+    def get_logger(self) -> logging.Logger:
+        return self.logger
+
+class KnowledgePipelineState(Enum):
+    INIT = 0
+    RUNNING = 1
+    STOPPED = 2
+    FINISHED = 3
+
+class KnowledgePipeline:
+    def __init__(self, name: str, env: KnowledgePipelineEnvironment, input_init, input_params, parser_init, parser_params):
+        self.name = name
+        self.state = KnowledgePipelineState.INIT
+        self.input_init = input_init
+        self.input_params = input_params
+        self.parser_init = parser_init
+        self.parser_params = parser_params
+        self.env = env
+        self.input = None
+        self.parser = None
+
+    def get_name(self):
+        return self.name
+
+    def get_journal(self) -> KnowledgePipelineJournalClient:
+        return self.env.journal
+        
+    async def run(self):
+        if self.state == KnowledgePipelineState.INIT:
+            self.input = self.input_init(self.env, self.input_params)
+            self.parser = self.parser_init(self.env, self.parser_params)
+            self.state = KnowledgePipelineState.RUNNING
+        if self.state == KnowledgePipelineState.RUNNING:
+            async for input in self.input.next():
+                if input is None:
+                    self.state = KnowledgePipelineState.FINISHED
+                    self.env.journal.insert(None, "finished", "finished")
+                    return 
+                (object_id, input_journal) = input
+                if object_id is not None:
+                    parser_journal = await self.parser.parse(object_id)
+                    self.env.journal.insert(object_id, input_journal, parser_journal)
+                else:
+                    return
+        if self.state == KnowledgePipelineState.STOPPED:
+            return 
+        if self.state == KnowledgePipelineState.FINISHED:
+            return 
+
+
+    

src/aios/knowledge/store.py

@@ -0,0 +1,102 @@
+import os
+import json
+import logging
+
+from .object import ObjectStore, ObjectRelationStore, ObjectID, ObjectType, KnowledgeObject
+from .core_object import DocumentObject, ImageObject, VideoObject, RichTextObject, EmailObject
+from .data import ChunkStore, ChunkTracker, ChunkListWriter, ChunkReader
+from ..storage.storage import AIStorage
+
+# KnowledgeStore class, which aggregates ChunkStore, ChunkTracker, and ObjectStore, and is a global singleton that makes it easy to use these three built-in store examples
+class KnowledgeStore:
+    _instance = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            
+            knowledge_dir = AIStorage.get_instance().get_myai_dir() / "knowledge" / "objects"
+
+            if not os.path.exists(knowledge_dir):
+                os.makedirs(knowledge_dir)
+
+            cls._instance.__singleton_init__(knowledge_dir)
+
+        return cls._instance
+
+    def __singleton_init__(self, root_dir: str):
+        logging.info(f"will init knowledge store, root_dir={root_dir}")
+
+        self.root = root_dir
+
+        relation_store_dir = os.path.join(root_dir, "relation")
+        self.relation_store = ObjectRelationStore(relation_store_dir)
+
+        object_store_dir = os.path.join(root_dir, "object")
+        self.object_store = ObjectStore(object_store_dir)
+
+        chunk_store_dir = os.path.join(root_dir, "chunk")
+        self.chunk_store = ChunkStore(chunk_store_dir)
+        self.chunk_tracker = ChunkTracker(chunk_store_dir)
+        self.chunk_list_writer = ChunkListWriter(self.chunk_store, self.chunk_tracker)
+        self.chunk_reader = ChunkReader(self.chunk_store, self.chunk_tracker)
+
+    
+    def get_relation_store(self) -> ObjectRelationStore:
+        return self.relation_store
+
+    def get_object_store(self) -> ObjectStore:
+        return self.object_store
+
+    def get_chunk_store(self) -> ChunkStore:
+        return self.chunk_store
+
+    def get_chunk_tracker(self) -> ChunkTracker:
+        return self.chunk_tracker
+    
+    def get_chunk_list_writer(self) -> ChunkListWriter:
+        return self.chunk_list_writer
+    
+    def get_chunk_reader(self) -> ChunkReader:
+        return self.chunk_reader
+    
+    async def insert_object(self, object: KnowledgeObject):
+        self.object_store.put_object(object.calculate_id(), object.encode())
+    
+    def load_object(self, object_id: ObjectID) -> KnowledgeObject:
+        if object_id.get_object_type() == ObjectType.Document:
+            return DocumentObject.decode(self.object_store.get_object(object_id))
+        if object_id.get_object_type() == ObjectType.Image:
+            return ImageObject.decode(self.object_store.get_object(object_id))
+        if object_id.get_object_type() == ObjectType.Video:
+            return VideoObject.decode(self.object_store.get_object(object_id))
+        if object_id.get_object_type() == ObjectType.RichText:
+            return RichTextObject.decode(self.object_store.get_object(object_id))
+        if object_id.get_object_type() == ObjectType.Email:
+            return EmailObject.decode(self.object_store.get_object(object_id))
+        else:
+            pass
+    
+    def parse_object_in_message(self, message: str) -> KnowledgeObject:
+        # get message's first line 
+        logging.info(f"tg parse resp message: {message}")
+        lines = message.split("\n")
+        if len(lines) > 0:
+            message = lines[0]
+            try:
+                desc = json.loads(message)
+                if isinstance(desc, dict):
+                    object_id = desc["id"]
+                else:
+                    object_id = desc[0]["id"]
+            except Exception as e:
+                return None
+            
+            if object_id is not None:
+                return self.load_object(ObjectID.from_base58(object_id))
+            
+            
+    def bytes_from_object(self, object: KnowledgeObject) -> bytes:
+        if object.get_object_type() == ObjectType.Image:
+            image_object = object
+            return self.get_chunk_reader().read_chunk_list_to_single_bytes(image_object.get_chunk_list())

src/aios/knowledge/vector/__init__.py

@@ -0,0 +1,2 @@
+from .vector_base import VectorBase
+from .chroma_store import ChromaVectorStore

src/aios/knowledge/vector/chroma_store.py

@@ -0,0 +1,51 @@
+from .vector_base import VectorBase
+from ..object import ObjectID
+import chromadb
+import logging
+import os
+
+
+class ChromaVectorStore(VectorBase):
+    def __init__(self, root_dir, model_name: str) -> None:
+        super().__init__(model_name)
+
+        logging.info(
+            "will init chroma vector store, model={}".format(model_name)
+        )
+
+        directory = os.path.join(root_dir, "vector")
+        logging.info("will use vector store: {}".format(directory))
+
+        client = chromadb.PersistentClient(
+            path=directory, settings=chromadb.Settings(anonymized_telemetry=False)
+        )
+        # client = chromadb.Client()
+
+        collection_name = "coll_{}".format(model_name)
+        logging.info("will init chroma colletion: %s", collection_name)
+
+        collection = client.get_or_create_collection(collection_name)
+        self.collection = collection
+
+    async def insert(self, vector: [float], id: ObjectID):
+        logging.info(f"will insert vector: {len(vector)} id: {str(id)}")
+        logging.debug(f"vector is {vector}")
+        self.collection.add(
+            embeddings=vector,
+            ids=str(id),
+        )
+
+    async def query(self, vector: [float], top_k: int) -> [ObjectID]:
+        ret = self.collection.query(
+            query_embeddings=vector,
+            n_results=top_k,
+        )
+        logging.info(f"query result {ret}")
+        if len(ret['ids']) == 0:
+            return []
+        return list(map(ObjectID.from_base58, ret["ids"][0]))
+
+    async def delete(self, id: ObjectID):
+        self.collection.delete(
+            ids=id,
+        )

src/aios/knowledge/vector/vector_base.py

@@ -0,0 +1,16 @@
+# import the ObjectID class
+from ..object import ObjectID
+
+# define a vector base class
+class VectorBase:
+    def __init__(self, model_name) -> None:
+        self.model_name = model_name
+
+    async def insert(self, vector: [float], id: ObjectID):
+        pass
+
+    async def query(self, vector: [float], top_k: int) -> [ObjectID]:
+        pass
+    
+    async def delete(self, id: ObjectID):
+        pass

src/aios/net/__init__.py

@@ -0,0 +1,2 @@
+from .cid import ContentId
+from .ndn_client import NDN_Client

src/aios/net/cid.py

@@ -0,0 +1,12 @@
+
+
+class ContentId:
+    def __init__(self) -> None:
+        pass
+
+    def as_str(self) -> str:
+        pass
+
+    @staticmethod
+    def create_from_str(cid_str:str):
+        pass

src/aios/net/ndn_client.py

@@ -0,0 +1,117 @@
+import asyncio,aiofiles,aiohttp
+import logging
+from typing import Optional
+
+from .cid import ContentId 
+
+logger = logging.getLogger(__name__)
+
+NDN_GET_TASK_STATE_INIT = 0
+NDN_GET_TAKS_CONNECTING = 1
+NDN_GET_TASK_STATE_DOWNLOADING = 2
+NDN_GET_TASK_STATE_VERIFYING = 3
+NDN_GET_TASK_STATE_DONE = 4
+NDN_GET_TASK_STATE_ERROR = 5
+
+class NDN_GetTask:
+    def __init__(self) -> None:
+        self.cid:str = None
+        self.target_path:str = None
+        self.urls:[str] = None
+        self.options:Optional[dict] = None
+
+        self.working_task = None
+        self.state = NDN_GET_TASK_STATE_INIT
+        self.total_size = 0
+        self.recv_bytes = 0
+        self.write_bytes = 0
+        self.error_str = None
+        self.chunk_queue = None
+       
+        self.retry_count = 0
+        self.used_urls = []
+        self.hash_update = None
+        
+    
+    def select_url(self,index:int)->str:
+        return self.urls[0]
+    
+    def get_chunk_for_download(self)->bytes:
+        pass
+
+class NDN_Client:
+    def __init__(self):
+        self.cache_dir = ""
+        self.default_ndn_http_gateway = ""
+        self.all_task = {}
+        self.memory_chunk_size = 1024*1024*2
+        self.chunk_queue_size = 16
+
+    def load_config(self,config:dict):
+        if config.get("cache_dir"):
+            self.cache_dir = config.get("cache_dir")
+        if config.get("dndn_gateway"):
+            self.default_ndn_http_gateway = config.get("ndn_gateway")
+
+    def get_file(self,cid:ContentId,target_path:str,urls:{}=None,options:{}=None)->NDN_GetTask:
+        get_task = self.all_task.get(cid.as_str())
+        if get_task:
+            return get_task
+        else:
+            get_task = NDN_GetTask()
+            self.all_task[cid.as_str()] = get_task
+            
+            get_task.cid = cid
+            get_task.target_path = target_path
+            get_task.urls = urls
+            get_task.options = options
+            if get_task.urls is None:
+                get_task.urls = [f"{self.default_ndn_http_gateway}/{cid.as_str()}"]
+                logger.info(f"get_file {cid.as_str()} urls is None, use {get_task.urls[0]} as default")
+
+
+            async def get_file_async():
+                target_file = aiofiles.open(target, 'wb')
+                # if file exist, check hash first
+
+                http_session = aiohttp.ClientSession()
+                resp = http_session.get(get_task.select_url(0))
+                if resp.status != 200:
+                    get_task.error_str = f"get_file {cid.as_str()} failed,http status:{resp.status}"
+                    return
+                get_task.total_size = resp.content_length
+               
+                async def write_file_async():
+                    while True:
+                        chunk = await get_task.chunk_queue.pop()
+                        chunk_size = len(chunk)
+                        if not chunk or chunk_size == 0:
+                            break
+                        get_task.hash_update.update(chunk)
+                        await target_file.write(chunk)
+                        get_task.write_bytes += chunk_size
+                    
+                    #verify
+                    get_task.state = NDN_GET_TASK_STATE_VERIFYING
+                    await target_file.close()
+                    return
+
+                write_task = asyncio.create_task(write_file_async())
+                while True:
+                    await get_task.chunk_queue.pop()
+                    chunk = resp.content.read(self.memory_chunk_size) 
+                    chunk_size = len(chunk)
+                    if not chunk or chunk_size == 0:
+                        break
+
+                    get_task.recv_bytes += len(chunk)
+                    get_task.chunk_queue.push(chunk)
+
+                
+                get_task.state = NDN_GET_TASK_STATE_DONE
+                await write_task
+
+            get_task.working_task = asyncio.create_task(get_file_async())
+            return get_task
+
+    

src/aios/package_manager/README

@@ -0,0 +1 @@
+TODO

src/aios/package_manager/__init__.py

@@ -0,0 +1,3 @@
+from .env import PackageEnvManager,PackageEnv
+from .pkg import PackageInfo,PackageMediaInfo
+from .installer import PackageInstallTask

src/aios/package_manager/env.py

@@ -0,0 +1,158 @@
+
+import logging
+import toml
+import os
+
+from .pkg import PackageInfo,PackageMediaInfo
+from .media_reader import MediaReader
+
+logger = logging.getLogger(__name__)
+
+
+class PackageEnv:
+    def __init__(self,cfg_path:str) -> None:
+        self.pkg_dir : str = "./pkgs/"
+        self.pkg_obj_dir : str = "./.pkgs/"
+
+        self.locked_index : str = "./pkg.lock"
+        self.is_strict : bool = True
+        self.parent_envs : list[PackageEnv] = []
+        self.index_dbs = None
+
+        self.env_dir = None
+        self.cfg_path = cfg_path
+        self._load_pkg_cfg(cfg_path)
+        pass
+
+    def load_from_config(self,config:dict) -> bool:
+        if config.get("main") is not None:
+            self.pkg_dir = os.path.abspath(self.env_dir + "/" + config["main"])
+
+        if config.get("cache") is not None:
+            self.pkg_obj_dir = os.path.abspath(self.env_dir + "/ " + config["cache"])
+
+    def load(self,pkg_name:str,search_parent=True) -> PackageMediaInfo:
+        pkg_path = None
+        pkg_id,verion_str,cid = PackageInfo.parse_pkg_name(pkg_name)
+        
+        if cid is None:
+            if verion_str is None:
+                pkg_path = f"{self.pkg_dir}/{pkg_id}"
+            else:
+                #TODO fix bug about channel here
+                channel:str = self.get_pkg_channel_from_version(verion_str)
+                the_version:str = self.get_exact_version_from_installed(verion_str)
+                if the_version is None:
+                    logger.warn(f"load {pkg_name} failed: no match version from {verion_str}")
+                    return None
+                if channel is None:
+                    pkg_path = f"{self.pkg_dir}/{pkg_id}#{the_version}"
+                else:
+                    pkg_path = f"{self.pkg_dir}/{pkg_id}#{channel}#{the_version}"
+        else:
+            pkg_path = f"{self.pkg_obj_dir}/.{pkg_id}/{cid}"
+
+        media_info:PackageMediaInfo = self.try_load_pkg_media_info(pkg_path)
+        if media_info is None:
+            if search_parent is True and self.parent_envs is not None:
+                for parent_env in self.parent_envs:
+                    media_info = parent_env.load(pkg_id,False)
+                    if media_info is not None:
+                        return media_info
+            
+        if media_info is None:
+            logger.warn(f"pkg_load {pkg_id}, cid:{cid} error,not found ,search_parent={search_parent}")
+
+        return media_info
+    
+    def get_exact_version_from_installed(self,verion_str:str) -> str:
+        pass
+
+    def get_pkg_channel_from_version(self,pkg_version:str) -> str:
+        args = pkg_version.split("~")
+        if len(args) == 1:
+            return None
+        else: 
+            return args[0]
+        
+
+    def get_pkg_media_info(self,pkg_name:str)->PackageMediaInfo:
+        pass
+
+    def try_load_pkg_media_info(self,pkg_full_path:str) -> PackageMediaInfo:
+        the_result : PackageMediaInfo = None
+        logger.debug(f"try load pkng from:{pkg_full_path}")
+        if os.path.isdir(pkg_full_path):
+            the_result = PackageMediaInfo(pkg_full_path,"dir")
+        
+        return the_result
+    
+    def _create_media_loader(self,media_info:PackageMediaInfo) -> MediaReader:
+        match media_info.media_type:
+            case "dir":
+                from .media_reader import FolderMediaReader 
+                return FolderMediaReader(media_info.full_path)
+                
+        logger.error(f"create media loader for {media_info} failed!")
+        return None  
+
+    def get_installed_pkg_info(self,pkg_name:str) -> PackageInfo:
+        pass
+
+    def lookup(self,pkg_id:str,version_str:str) -> PackageInfo:
+        # to make sure pkg.cid is correct, we MUST verfiy eveything here 
+        pass
+
+    @classmethod    
+    def is_valied_media(pkg_full_path:str) -> bool:
+        pass
+    
+    def do_pkg_media_trans(self,pkg_info:PackageInfo,source_path:str,target_path:str) -> bool:
+        pass
+
+    def _load_pkg_cfg(self,cfg_path:str):
+        if cfg_path is None:
+            return
+        
+        cfg = None
+        if len(cfg_path) < 1:
+            return
+        try:
+            cfg = toml.load(cfg_path)
+            self.env_dir = os.path.abspath(os.path.dirname(cfg_path))
+            self.cfg_path = os.path.abspath(cfg_path)
+        except Exception as e:
+            logger.error(f"read pkg cfg from {cfg_path} failed! unexpected error occurred: {str(e)}")
+            return
+
+        return self.load_from_config(cfg)
+
+    
+   
+    def _preprocess_prefixs(self,prefixs):
+        pass
+
+class PackageEnvManager:
+    _instance = None
+    @classmethod
+    def get_instance(cls):
+        if cls._instance is None:
+            cls._instance = PackageEnvManager()
+        return cls._instance
+    
+    def __init__(self) -> None:
+        self._pkg_envs = {}
+        
+    def get_env(self,cfg_path:str) -> PackageEnv:
+        if cfg_path in self._pkg_envs:
+            return self._pkg_envs[cfg_path]
+        else:
+            pkg_env = PackageEnv(cfg_path)
+            self._pkg_envs[cfg_path] = pkg_env
+            return pkg_env
+
+    def get_user_env(self) -> PackageEnv:
+        pass
+
+    def get_system_env(self) -> PackageEnv:
+        pass

src/aios/package_manager/index_syncer.py

@@ -0,0 +1 @@
+

src/aios/package_manager/installer.py

@@ -0,0 +1,171 @@
+# installer download pkg by cid, than install it to target dir
+import logging
+import asyncio
+import aiohttp
+import aiofiles
+import os
+from typing import Tuple
+
+#from ndn_client import ContentId,NDN_Client
+from ..net import ContentId,NDN_Client
+from .pkg import PackageInfo,PackageMediaInfo
+from .env import PackageEnv
+
+logger = logging.getLogger(__name__)
+
+INSTALL_TASK_STATE_DONE = 0
+INSTALL_TASK_STATE_CHECK_DEPENDENCY = 1
+INSTALL_TASK_STATE_INSTALL_DEPENDENCY = 2
+INSTALL_TASK_STATE_DOWNLOADING = 3
+INSTALL_TASK_STATE_INSTALLING = 4
+INSTALL_TAKS_STATE_ERROR = 5
+
+class PackageInstallTask:
+    def __init__(self,owner:PackageEnv) -> None:
+        self.owner = owner
+        self.state = INSTALL_TASK_STATE_CHECK_DEPENDENCY
+
+        self.pkg_media_info = None
+        self.working_task = None
+        self.dependency_tasks = None
+        self.error_str = None
+
+class PackageInstaller:
+    def __init__(self,owner_env:PackageEnv) -> None:
+        self.all_tasks = {}
+        self.owner_env = owner_env
+    
+    def install(self,pkg_name:str,
+                install_from_dependency = False, can_upgrade = True,skip_depends = False,options = None)->Tuple[PackageInstallTask,str]:
+
+        the_pkg_info : PackageInfo = None
+        is_upgrade : bool = False
+        need_backup : bool = False
+        
+        pkg_id,version_str,cid = PackageInfo.parse_pkg_name(pkg_name)
+        media_info : PackageMediaInfo = self.owner_env.get_media_info(pkg_name) # must use index-db?
+        if media_info is not None:
+            if cid is not None:
+                if can_upgrade:
+                    is_upgrade = True
+                else:
+                    error_str = f"{pkg_name},{cid} already installed!"
+                    logger.error(error_str)
+                    return None,error_str
+            else:
+                the_pkg_info = self.owner_env.lookup(pkg_id,version_str,None)
+                if the_pkg_info is None:
+                    error_str = f"{pkg_name} old version exist in local but not found in index db!"
+                    logger.error(error_str)
+                    return None,error_str
+                else:
+                    is_upgrade = True
+                    need_backup = True
+
+        if the_pkg_info is None:
+            the_pkg_info = self.owner_env.lookup(pkg_id,version_str,cid)
+        
+        if the_pkg_info is None:
+            error_str = f"{pkg_name} ,cid:{cid} not found in index db"
+            logger.error(error_str)
+            return None,error_str
+        
+        result_task = self.all_tasks.get(the_pkg_info.cid)
+        if result_task is not None:
+            return result_task,"already installing"
+        
+        logger.info(f"start download&install {pkg_name},install_from_dependency={install_from_dependency},upgrade={is_upgrade},backup={need_backup},target_pkg_info={the_pkg_info}")
+        result_task = PackageInstallTask(self.owner_env)
+        self.all_tasks[the_pkg_info.cid] = result_task
+        async def download_and_install_pkg()->int:
+            # check dependency
+            if skip_depends is False:
+                result_task.dependency_tasks = {}
+                self.get_dependency_tasks(the_pkg_info,result_task.dependency_tasks)
+                result_task.state = INSTALL_TASK_STATE_INSTALL_DEPENDENCY
+                for depend_pkg_name in result_task.dependency_tasks:
+                    # check pkg in local?
+                    # install miss pkg
+                    pass
+            
+            result_task.state = INSTALL_TASK_STATE_DOWNLOADING
+            install_full_path = ""
+            target_full_path = ""
+            old_package_full_path = ""
+            is_download_directy = False
+
+            if the_pkg_info.target_media_type == the_pkg_info.source_media_type:
+                is_download_directy = True
+                if is_upgrade:
+                    target_full_path = ""
+                else:
+                    target_full_path = ""
+            else:
+                pass
+
+            urls = self.owner_env.get_pkg_urls(the_pkg_info)
+            #download
+            client = NDN_Client() # set watch
+            download_result = await client.get_file(the_pkg_info.cid,urls,target_full_path,options) 
+            if download_result !=0:
+                result_task.state = INSTALL_TAKS_STATE_ERROR
+                return result_task.state
+            
+            result_task.state = INSTALL_TASK_STATE_INSTALLING
+            if is_download_directy is False:
+                install_media_result = False
+                install_media_result = await self.owner_env.do_pkg_media_trans(the_pkg_info,target_full_path,install_full_path)
+                if install_media_result is False:
+                    result_task.state = INSTALL_TAKS_STATE_ERROR
+                    result_task.error_str = "install media error,from {target_full_path} to {install_full_path}"
+                    return result_task.state
+               
+            # last step,save install flag : install by manual or install by dependency
+            ## save cid dir
+            if is_upgrade:
+                os.rename(old_package_full_path, old_package_full_path + ".old" )
+                os.rename(target_full_path,install_full_path)
+            ## update/create version link
+            
+            ## update pkg state
+            ## remove old version
+    
+            result_task.state = INSTALL_TASK_STATE_DONE
+            return result_task.state
+                
+
+        result_task.working_task = asyncio.create_task(download_and_install_pkg())
+        return result_task,None
+
+        
+    def uninstall(self):
+        pass
+    
+    def get_dependency_tasks(self,pkg:PackageInfo,dependency_tasks):
+        pass
+
+    async def check_dependency(self,pkg:PackageInfo,task_list:{}) -> bool:
+        for depend_pkg_name in pkg.depends:
+            depend_task = task_list.get(depend_pkg_name)
+            if depend_task is not None:
+                logger.debug(f"{pkg.name}'s depend pkg {depend_pkg_name} already in task list")
+                continue
+            depend_task = PackageInstallTask(self.owner_env)
+            task_list[depend_pkg_name] = depend_task
+            
+            depend_pkg_info = self.owner_env.lookup(depend_pkg_name)
+            if depend_pkg_info is None:
+                logger.warn(f"{pkg.name}'s depend pkg {depend_pkg_name} not found in index db")
+                return False
+            
+            if await self.check_dependency(depend_pkg_info,task_list) is False:
+                return False
+    
+        return True
+        
+        
+
+        
+        
+
+

src/aios/package_manager/media_reader.py

@@ -0,0 +1,18 @@
+from abc import ABC, abstractmethod
+import aiofiles
+
+class MediaReader(ABC):
+    @abstractmethod
+    async def read(self, inner_path:str,mode:str):
+        pass
+
+
+class FolderMediaReader(MediaReader):
+    def __init__(self, root_dir:str) -> None:
+        self.root_dir = root_dir
+        pass
+
+    async def read(self, inner_path:str,mode:str):
+        full_path = self.root_dir + "/" + inner_path
+        result_file = await aiofiles.open(full_path, mode,encoding='utf-8')
+        return result_file

src/aios/package_manager/pkg.py

@@ -0,0 +1,41 @@
+from typing import Tuple
+
+
+class PackageInfo:
+    def __init__(self) -> None:
+        self.name = ""
+        self.cid = None
+        self.depends : list[str] = None
+        self.author = None
+        self.remote_urls = None
+        self.target_media_type = "dir"
+        self.source_media_type = "7z"
+
+    @staticmethod
+    def parse_pkg_name(pkg_name:str) -> Tuple[str, str, str]:
+        """parse pkg name like test-pkg#nightly~>0.2.31#sha1:323423423 to test-pkg,nightly#>0.2.31,sha1:323423423"""
+        args = pkg_name.split("#")
+        if len(args) == 1:
+            return args[0],None,None
+        elif len(args) == 2:  
+            return args[0],None,arg[2]
+        elif len(args) == 3:
+            return args[0],args[1],args[2]
+        else:
+            logger.error(f"parse pkg name {pkg_name} failed!")
+            return None,None,None
+
+    
+
+
+    @property
+    def cid(self) -> str:
+        return self.cid
+
+class PackageMediaInfo:
+    def __init__(self,full_path,media_type) -> None:
+        self.media_type = media_type
+        self.full_path = full_path
+
+
+

src/aios/proto/agent_msg.py

@@ -0,0 +1,167 @@
+import logging
+import uuid
+from enum import Enum
+import time
+
+logger = logging.getLogger(__name__)
+
+class AgentMsgType(Enum):
+    TYPE_MSG = 0
+    TYPE_GROUPMSG = 1
+    TYPE_INTERNAL_CALL = 10
+    TYPE_ACTION = 20
+    TYPE_EVENT = 30
+    TYPE_SYSTEM = 40
+
+
+class AgentMsgStatus(Enum):
+    RESPONSED = 0
+    INIT = 1
+    SENDING = 2
+    PROCESSING = 3
+    ERROR = 4
+    RECVED = 5
+    EXECUTED = 6
+
+# msg is a msg / msg resp
+# msg body可以有内容类型（MIME标签），text, image, voice, video, file,以及富文本(html)
+# msg is a inner function call with result
+# msg is a Action with result
+
+# qutoe Msg
+# forword msg
+# reply msg
+
+# 逻辑上的同一个Message在同一个session中看到的msgid相同
+#       在不同的session中看到的msgid不同
+
+
+
+class AgentMsg:
+    def __init__(self,msg_type=AgentMsgType.TYPE_MSG) -> None:
+        self.msg_id = "msg#" + uuid.uuid4().hex
+        self.msg_type:AgentMsgType = msg_type
+
+        self.prev_msg_id:str = None
+        self.quote_msg_id:str = None
+        self.rely_msg_id:str = None # if not none means this is a respone msg
+        self.session_id:str = None
+
+        #forword info
+
+
+        self.create_time = 0
+        self.done_time = 0
+        self.topic:str = None # topic is use to find session, not store in db
+
+        self.sender:str = None # obj_id.sub_objid@tunnel_id
+        self.target:str = None
+        self.mentions:[] = None #use in group chat only
+        #self.title:str = None
+        self.body:str = None
+        self.body_mime:str = None #//default is "text/plain",encode is utf8
+
+        #type is call / action
+        self.func_name = None
+        self.args = None
+        self.result_str = None
+
+        #type is event
+        self.event_name = None
+        self.event_args = None
+
+        self.status = AgentMsgStatus.INIT
+        self.inner_call_chain = []
+        self.resp_msg = None
+
+    @classmethod
+    def from_json(cls,json_obj:dict) -> 'AgentMsg':
+        msg = AgentMsg()
+
+        return msg
+
+    @classmethod
+    def create_internal_call_msg(self,func_name:str,args:dict,prev_msg_id:str,caller:str):
+        msg = AgentMsg(AgentMsgType.TYPE_INTERNAL_CALL)
+        msg.create_time = time.time()
+        msg.func_name = func_name
+        msg.args = args
+        msg.prev_msg_id = prev_msg_id
+        msg.sender = caller
+        return msg
+
+    def create_action_msg(self,action_name:str,args:dict,caller:str):
+        msg = AgentMsg(AgentMsgType.TYPE_ACTION)
+        msg.create_time = time.time()
+        msg.func_name = action_name
+        msg.args = args
+        msg.prev_msg_id = self.msg_id
+        msg.topic  = self.topic
+        msg.sender = caller
+        return msg
+
+    def create_error_resp(self,error_msg:str):
+        resp_msg = AgentMsg(AgentMsgType.TYPE_SYSTEM)
+        resp_msg.create_time = time.time()
+
+        resp_msg.rely_msg_id = self.msg_id
+        resp_msg.body = error_msg
+        resp_msg.topic  = self.topic
+        resp_msg.sender = self.target
+        resp_msg.target = self.sender
+
+        return resp_msg
+
+    def create_resp_msg(self,resp_body):
+        resp_msg = AgentMsg()
+        resp_msg.create_time = time.time()
+
+        resp_msg.rely_msg_id = self.msg_id
+        resp_msg.sender = self.target
+        resp_msg.target = self.sender
+        resp_msg.body = resp_body
+        resp_msg.topic = self.topic
+
+        return resp_msg
+
+    def create_group_resp_msg(self,sender_id,resp_body):
+        resp_msg = AgentMsg(AgentMsgType.TYPE_GROUPMSG)
+        resp_msg.create_time = time.time()
+
+        resp_msg.rely_msg_id = self.msg_id
+        resp_msg.target = self.target
+        resp_msg.sender = sender_id
+        resp_msg.body = resp_body
+        resp_msg.topic = self.topic
+
+        return resp_msg
+
+    def set(self,sender:str,target:str,body:str,topic:str=None) -> None:
+        self.sender = sender
+        self.target = target
+        self.body = body
+        self.create_time = time.time()
+        if topic:
+            self.topic = topic
+
+    def get_msg_id(self) -> str:
+        return self.msg_id
+
+    def get_sender(self) -> str:
+        return self.sender
+
+    def get_target(self) -> str:
+        return self.target
+
+    def get_prev_msg_id(self) -> str:
+        return self.prev_msg_id
+
+    def get_quote_msg_id(self) -> str:
+        return self.quote_msg_id
+
+    @classmethod
+    def parse_function_call(cls,func_string:str):
+        str_list = shlex.split(func_string)
+        func_name = str_list[0]
+        params = str_list[1:]
+        return func_name, params

src/aios/proto/compute_task.py

@@ -0,0 +1,142 @@
+
+from enum import Enum
+import uuid
+import time
+from typing import Union
+from ..knowledge import ObjectID
+from ..storage.storage import AIStorage
+
+class ComputeTaskResultCode(Enum):
+    OK = 0
+    TIMEOUT = 1
+    NO_WORKER = 2
+    ERROR = 3
+
+
+class ComputeTaskState(Enum):
+    DONE = 0
+    INIT = 1
+    RUNNING = 2
+    ERROR = 3
+    PENDING = 4
+
+class ComputeTaskType(Enum):
+    NONE = "None"
+    LLM_COMPLETION = "llm_completion"
+    TEXT_2_IMAGE = "text_2_image"
+    IMAGE_2_TEXT = "image_2_text"
+    IMAGE_2_IMAGE = "image_2_image"
+    VOICE_2_TEXT = "voice_2_text"
+    TEXT_2_VOICE = "text_2_voice"
+    TEXT_EMBEDDING ="text_embedding"
+    IMAGE_EMBEDDING ="image_embedding"
+
+
+class ComputeTask:
+    def __init__(self) -> None:
+        self.task_type = ComputeTaskType.NONE
+        self.create_time = None
+
+        self.task_id: str = None
+        self.callchain_id: str = None
+        self.params: dict = {}
+        self.refers: dict = None
+        self.pading_data: bytearray = None
+
+        self.state = ComputeTaskState.INIT
+        self.result = None
+        self.error_str = None
+
+    def set_llm_params(self, prompts, resp_mode,model_name, max_token_size, inner_functions = None, callchain_id=None):
+        self.task_type = ComputeTaskType.LLM_COMPLETION
+        self.create_time = time.time()
+        self.task_id = uuid.uuid4().hex
+        self.callchain_id = callchain_id
+        self.params["prompts"] = prompts.to_message_list()
+        self.params["resp_mode"] = resp_mode
+        if model_name is None:
+             model_name = AIStorage.get_instance().get_user_config().get_value("llm_model_name")
+        self.params["model_name"] = model_name
+        if max_token_size is None:
+            self.params["max_token_size"] = 4000
+        else:
+            self.params["max_token_size"] = max_token_size
+
+        if inner_functions is not None:
+            self.params["inner_functions"] = inner_functions
+
+    def set_text_embedding_params(self, input: str, model_name=None, callchain_id = None):
+        self.task_type = ComputeTaskType.TEXT_EMBEDDING
+        self.create_time = time.time()
+        self.task_id = uuid.uuid4().hex
+        self.callchain_id = callchain_id
+        if model_name is not None:
+            self.params["model_name"] = model_name
+        else:
+            self.params["model_name"] = "text-embedding-ada-002"
+        self.params["input"] = input
+        
+    def set_image_embedding_params(self, input = Union[ObjectID, bytes], model_name=None, callchain_id = None):
+        self.task_type = ComputeTaskType.IMAGE_EMBEDDING
+        self.create_time = time.time()
+        self.task_id = uuid.uuid4().hex
+        self.callchain_id = callchain_id
+        if model_name is not None:
+            self.params["model_name"] = model_name
+        else:
+            self.params["model_name"] = None
+        self.params["input"] = input
+    
+    def set_text_2_image_params(self, prompt: str, model_name, negative_prompt="", callchain_id=None):
+        self.task_type = ComputeTaskType.TEXT_2_IMAGE
+        self.create_time = time.time()
+        self.task_id = uuid.uuid4().hex
+        self.callchain_id = callchain_id
+        self.params["prompt"] = prompt
+        self.params["negative_prompt"] = negative_prompt
+        if model_name is not None:
+            self.params["model_name"] = model_name
+        else:
+            self.params["model_name"] = "v1-5-pruned-emaonly"
+
+    def set_image_2_text_params(self, image_path: str, prompt: str, model_name, negative_prompt="", callchain_id=None):
+        self.task_type = ComputeTaskType.IMAGE_2_TEXT
+        self.create_time = time.time()
+        self.task_id = uuid.uuid4().hex
+        self.callchain_id = callchain_id
+        self.params["image_path"] = image_path
+        if prompt == '':
+            self.params["prompt"] = "What's in this image?"
+        else:
+            self.params["prompt"] = prompt
+        self.params["negative_prompt"] = negative_prompt
+        if model_name is not None:
+            self.params["model_name"] = model_name
+        else:
+            self.params["model_name"] = "gpt-4-vision-preview"
+
+
+    def display(self) -> str:
+        return f"ComputeTask: {self.task_id} {self.task_type} {self.state}"
+
+
+class ComputeTaskResult:
+    def __init__(self) -> None:
+        self.create_time = None
+        self.task_id: str = None
+        self.callchain_id: str = None
+        self.worker_id: str = None
+        self.error_str : str = None 
+        self.result_code: int = 0
+        self.result_str: str = None # easy to use,can read from result
+
+        self.result : dict = {}
+
+        self.result_refers: dict = {}
+        self.pading_data: bytearray = None
+        
+
+    def set_from_task(self, task: ComputeTask):
+        self.task_id = task.task_id
+        self.callchain_id = task.callchain_id
+        task.result = self

src/aios/storage/storage.py

@@ -0,0 +1,246 @@
+from typing import Any
+from pathlib import Path
+import os
+import logging
+import toml
+import aiofiles
+
+logger = logging.getLogger(__name__)
+
+_file_dir = os.path.dirname(__file__)
+
+class ResourceLocation:
+    def __init__(self) -> None:
+        pass
+
+class FeatureItem:
+    def __init__(self) -> None:
+        pass
+
+class UserConfigItem:
+    def __init__(self,desc:str=None) -> None:
+        self.default_value = None 
+        self.is_optional = False
+        self.item_type = "str"
+        self.desc = desc
+        self.value = None
+        self.user_set = False
+
+    def clone(self):
+        new_config_item = UserConfigItem()
+        new_config_item.default_value = self.default_value
+        new_config_item.is_optional = self.is_optional
+        new_config_item.desc = self.desc
+        new_config_item.item_type = self.item_type
+        new_config_item.value = self.value
+        return new_config_item
+
+class UserConfig:
+    def __init__(self) -> None:
+        self.config_table = {}
+        self.user_config_path:str = None
+
+        self._init_default_value("llm_model_name","gpt-4-1106-preview")
+
+    def _init_default_value(self,key:str,value:Any) -> None:
+        if self.config_table.get(key) is not None:
+            logger.warning("user config key %s already exist, will be overrided",key)
+        
+        new_config_item = UserConfigItem()
+        new_config_item.default_value = value
+        new_config_item.is_optional = True
+        self.config_table[key] = new_config_item
+
+
+    def add_user_config(self,key:str,desc:str,is_optional:bool,default_value:Any=None,item_type="str") -> None:
+        if self.config_table.get(key) is not None:
+            logger.warning("user config key %s already exist, will be overrided",key)
+    
+        new_config_item = UserConfigItem()
+        new_config_item.default_value = default_value
+        new_config_item.is_optional = is_optional
+        new_config_item.desc = desc
+        new_config_item.item_type = item_type
+        self.config_table[key] = new_config_item
+        
+    async def load_value_from_file(self,file_path:str,is_user_config = False) -> None:
+        try: 
+            all_config = toml.load(file_path)
+            if all_config is not None:
+                for key,value in all_config.items():
+                    config_item = self.config_table.get(key)
+                    if config_item is None:
+                        logger.warning("user config key %s not exist",key)
+                        continue
+                    config_item.value = value
+                    config_item.user_set = is_user_config
+
+        except Exception as e:
+            logger.warn(f"load user config from {file_path} failed!")
+
+
+    async def save_to_user_config(self) -> None:
+        will_save_config = {}
+        for key,value in self.config_table.items():
+            if value.user_set:
+                will_save_config[key] = value.value
+        
+        if len(will_save_config) > 0:
+            try:
+                directory = os.path.dirname(self.user_config_path)
+                if not os.path.exists(directory):
+                    os.makedirs(directory)
+
+                async with aiofiles.open(self.user_config_path,"w") as f:
+                    toml_str = toml.dumps(will_save_config)
+                    await f.write(toml_str)
+            except Exception as e:
+                logger.error(f"save user config to {self.user_config_path} failed!")
+                return False
+        
+        return True
+
+    def get_config_item(self,key:str) -> Any:
+        config_item = self.config_table.get(key)
+        if config_item is None:
+            logger.warning(f"user config key {key} not exist")
+            return None
+
+        return config_item
+    
+    def get_value(self,key:str)->Any:
+        config_item = self.config_table.get(key)
+        if config_item is None:
+            logger.warning(f"user config key {key} not exist")
+            return None
+        
+        if config_item.value is None:
+            return config_item.default_value
+
+        return config_item.value
+
+    def set_value(self,key:str,value:Any) -> None:
+        config_item = self.config_table.get(key)
+        if config_item is None:
+            logger.warning("user config key %s not exist",key)
+            return 
+        
+        config_item.value = value
+        config_item.user_set = True
+        #TODO: save to file?
+
+        
+    def check_config(self) -> None:
+        check_result = {}
+        for key,config_item in self.config_table.items():
+            if config_item.value is None and not config_item.is_optional:
+                check_result[key] = config_item
+        
+        if len(check_result) > 0:
+            return check_result
+        else:
+            return None
+
+# storage sytem for current user
+class AIStorage:
+    _instance = None
+    @classmethod
+    def get_instance(cls):
+        if cls._instance is None:
+            cls._instance = AIStorage()
+        return cls._instance
+
+    def __init__(self) -> None:
+        self.is_dev_mode = False
+        self.user_config = UserConfig()
+        self.feature_init_results = {}
+
+    async def initial(self)->bool:
+        self.user_config.user_config_path = str(self.get_myai_dir() / "etc/system.cfg.toml")
+        await self.user_config.load_value_from_file(self.get_system_dir() + "/system.cfg.toml")
+        await self.user_config.load_value_from_file(self.user_config.user_config_path,True)
+
+    async def enable_feature(self,feature_name:str) -> None:
+        self.user_config.set_value(f"feature.{feature_name}","True")
+        await self.user_config.save_to_user_config()
+        
+
+    async def disable_feature(self,feature_name:str) -> None:
+        self.user_config.set_value(f"feature.{feature_name}","False")
+        await self.user_config.save_to_user_config()
+    
+    async def set_feature_init_result(self,feature_name:str,result:bool) -> None:
+        self.feature_init_results[feature_name] = result
+
+    async def is_feature_enable(self,feature_name:str) -> bool:
+        is_enable = self.user_config.get_value(f"feature.{feature_name}")
+        if is_enable is None:
+            return False
+        
+        init_result = self.feature_init_results.get(feature_name)
+        if init_result:
+            if init_result is False:
+                return False
+        
+        if is_enable == "True":
+            return True
+        return False
+
+    def get_user_config(self) -> UserConfig:
+        return self.user_config
+
+    def get_system_dir(self) -> str:
+        """ 
+        system dir is dir for aios system
+        /opt/aios
+        """
+        if self.is_dev_mode:
+            return os.path.abspath(_file_dir + "/../../")
+        else:
+            return "/opt/aios/"
+        
+    
+    def get_system_app_dir(self)->str:
+        """
+        system app dir is the dir for aios build-in app 
+        /opt/aios/app
+        """
+        if self.is_dev_mode:
+            return os.path.abspath(_file_dir + "/../../../rootfs/")
+        else:
+            return "/opt/aios/app/"
+    
+    def get_myai_dir(self) -> str:
+        """
+        my ai dir is the dir for user to store their ai app and data
+        ~/myai/
+        """
+        return Path.home() / "myai"
+        
+    def get_db(self,app_name:str)->ResourceLocation:
+        pass
+
+    def open_file(self,file_path:str,options:dict):
+        pass
+
+    def get_named_object(self,name:str) -> Any:
+        pass
+
+    def put_named_object(self,name:str,obj:Any) -> None:
+        pass
+    
+    async def try_create_file_with_default_value(self,path:str,default_value:str):
+        if os.path.exists(path):
+            return None
+        
+        try:
+            directory = os.path.dirname(path)
+            if not os.path.exists(directory):
+                os.makedirs(directory)
+            async with aiofiles.open(path,"w") as f:
+                await f.write(default_value)
+
+        except Exception as e:
+            logger.error(f"open or create file {path} failed! {str(e)}")
+
+