Complete self_thinking llm process and Agent Memory

src/aios/__init__.py

@@ -19,7 +19,7 @@ from .frame.compute_kernel import ComputeKernel,ComputeTask,ComputeTaskResult,Co
 from .frame.compute_node import ComputeNode,LocalComputeNode
 from .frame.bus import AIBus
 from .frame.tunnel import AgentTunnel
-from .frame.contact_manager import ContactManager,Contact,FamilyMember
+from .frame.contact_manager import ContactManager,Contact
 from .frame.queue_compute_node import Queue_ComputeNode
 
 from .environment.environment import BaseEnvironment,SimpleEnvironment,CompositeEnvironment

src/aios/agent/agent.py

@@ -24,21 +24,16 @@ from .llm_do_task import *
 from .chatsession import *
 
 from ..environment.workspace_env import WorkspaceEnvironment, TodoListType
-from ..frame.contact_manager import ContactManager,Contact,FamilyMember
-from ..frame.compute_kernel import ComputeKernel
-from ..frame.bus import AIBus
 from ..environment.environment import *
-from ..environment.workspace_env import WorkspaceEnvironment
 from ..storage.storage import AIStorage
 from ..knowledge import *
-from ..utils import video_utils, image_utils
-from ..proto.compute_task import ComputeTaskResult,ComputeTaskResultCode,LLMPrompt,LLMResult
+from ..proto.compute_task import LLMPrompt,LLMResult
 
 logger = logging.getLogger(__name__)
 
 class AIAgentTemplete:
     def __init__(self) -> None:
-        self.llm_model_name:str = "gpt-4-0613"
+        self.llm_model_name:str = "gpt-4-turbo-preview"
         self.max_token_size:int = 0
         self.template_id:str = None
         self.introduce:str = None
@@ -99,7 +94,8 @@ class AIAgent(BaseAIAgent):
         }
         self.todo_prompts = todo_prompts
 
-        self.memory_db = None
+        self.base_dir = None
+        #self.memory_db = None
         self.unread_msg = Queue() # msg from other agent
         self.owenr_bus = None
 
@@ -109,7 +105,9 @@ class AIAgent(BaseAIAgent):
         self.behaviors:Dict[str,BaseLLMProcess] = {}
         
     async def initial(self,params:Dict = None):
-        self.memory = AgentMemory(self.agent_id,self.memory_db)
+        self.base_dir = f"{AIStorage.get_instance().get_myai_dir()}/agent_data/{self.agent_id}"
+        memory_base_dir = f"{self.base_dir}/memory"
+        self.memory = AgentMemory(self.agent_id,memory_base_dir)
         self.prviate_workspace = AgentWorkspace(self.agent_id) 
         init_params = {}
         init_params["memory"] = self.memory
@@ -241,6 +239,34 @@ class AIAgent(BaseAIAgent):
         return await self.llm_process_msg(msg)
 
 
+    async def llm_self_think(self):
+        llm_process : BaseLLMProcess = self.behaviors.get("self_thinking")
+        if llm_process:
+            logger.info(f"agent {self.agent_id} self thinking start!")
+
+            context_info = await self._get_context_info()
+            known_session_list = AIChatSession.list_session(self.agent_id,self.memory.memory_db)
+            known_experience_list = await self.memory.list_experience()
+            record_list = await self.memory.load_records(await self.memory.get_last_think_time())
+
+            input_parms = {
+                "record_list":record_list,
+                "known_session_list":known_session_list,
+                "known_experience_list":known_experience_list,
+                "context_info":context_info
+            }
+
+            llm_result : LLMResult = await llm_process.process(input_parms)
+            if llm_result.state == LLMResultStates.ERROR:
+                logger.error(f"llm process self thinking error:{llm_result.compute_error_str}")
+            elif llm_result.state == LLMResultStates.IGNORE:
+                logger.info(f"llm process self thinking  ignore!")
+            else:
+                logger.info(f"llm process self thinking  ok!,think is:{llm_result.resp}")
+                self.memory.set_last_think_time(time.time())
+            self.agent_energy -= 2  
+            return
+
     async def llm_triage_tasklist(self):
         llm_process : BaseLLMProcess = self.behaviors.get("triage_tasks")
         if llm_process:
@@ -361,21 +387,8 @@ class AIAgent(BaseAIAgent):
             self.agent_energy -= 1
 
 
-    async def do_self_think(self):
-        session_id_list = AIChatSession.list_session(self.agent_id,self.memory_db)
-        for session_id in session_id_list:
-            if self.agent_energy <= 0:
-                break
-            used_energy = await self.think_chatsession(session_id)
-            self.agent_energy -= used_energy
-        return
-
-    def need_self_think(self) -> bool:
-        return False
-
     async def _self_imporve(self):
-        if self.need_self_think():
-            await self.do_self_think()
+        await self.llm_self_think()
 
     def wake_up(self) -> None:
         if self.agent_task is None:
@@ -446,13 +459,16 @@ class AIAgent(BaseAIAgent):
                     
                 await self._self_imporve()
                 
+               
+                
             except Exception as e:
                 tb_str = traceback.format_exc()
                 logger.error(f"agent {self.agent_id} on timer error:{e},{tb_str}")
-                continue
-
+            
             # Because the LLM itself is very slow, the accuracy of the system processing task is in minutes.
             await asyncio.sleep(30) 
+                
+
 
 
 

src/aios/agent/agent_memory.py

@@ -1,12 +1,18 @@
 # pylint:disable=E0402
 from datetime import datetime,timedelta
 import json
+import os
 import threading
 from typing import Dict, List
 import sqlite3
 
+import aiofiles
+
+from ..storage.storage import AIStorage
 from ..frame.compute_kernel import ComputeKernel
-from ..proto.ai_function import SimpleAIAction
+from ..frame.contact_manager import ContactManager
+from ..frame.contact import Contact
+from ..proto.ai_function import ParameterDefine, SimpleAIAction, SimpleAIFunction
 from ..proto.agent_msg import AgentMsg, AgentMsgType
 from ..proto.agent_task import AgentWorkLog
 
@@ -17,12 +23,34 @@ import logging
 
 logger = logging.getLogger(__name__)
 
+
+#class ObjectSummary:
+#    def __init__(self) -> None:
+#        self.summary : str = None
+#        self.object_name : str = None
+#        self.priority : int = 5
+        # [info_source, info]
+#        self.infos : Dict[str,str] = {}
+
+
+
 class AgentMemory:
-    def __init__(self,agent_id:str,db_path:str) -> None:
+    def __init__(self,agent_id:str,base_dir:str) -> None:
+        self.agent_memory_base_dir = base_dir
         self.agent_id:str= agent_id
-        self.memory_db:str = db_path
+
+        AIStorage.get_instance().ensure_directory_exists(self.agent_memory_base_dir)
+        AIStorage.get_instance().ensure_directory_exists(f"{self.agent_memory_base_dir}/experience")
+        AIStorage.get_instance().ensure_directory_exists(f"{self.agent_memory_base_dir}/contacts")
+        AIStorage.get_instance().ensure_directory_exists(f"{self.agent_memory_base_dir}/relations")
+        AIStorage.get_instance().ensure_directory_exists(f"{self.agent_memory_base_dir}/summary")
+
+        self.memory_db:str = f"{self.agent_memory_base_dir}/memory.db"
         self.model_name:str = "gp4-1106-preview"
         self.threshold_hours = 72
+        self.last_think_time : float = 0.0
+
+        self.load_memory_meta()
 
 
     def _get_conn(self):
@@ -54,6 +82,15 @@ class AgentMemory:
             session_topic = msg.get_sender() + "#" + msg.topic
             chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.memory_db)
         return chatsession
+    
+    # return last record time
+    async def load_records(self,starttime,tokenlimit=8000)->float:
+        # 专用思路：做聊天记录/工作经验的整理
+        # 通用思路：没有具体的目的，让Agent根据提示词自己工作（可能效果很差也可能很好）
+        # 先实现通用思路
+        msg_records = AIChatSession.load_message_records_by_agentid(self.agent_id,starttime,32,self.memory_db)
+        work_records = self.load_worklogs(self.agent_id,token_limit=tokenlimit)
+        pass
 
     async def load_chatlogs(self,msg:AgentMsg,n:int=6,m:int=64,token_limit=800)->str:
         chatsession = self.get_session_from_msg(msg)
@@ -176,23 +213,295 @@ class AgentMemory:
         conn.commit()
         #conn.close()
 
+    def memory_meta_to_dict(self) -> Dict:
+        return {
+            "last_think_time" : self.last_think_time
+        }
+    
+    def load_meta(self,Dict):
+        self.last_think_time = Dict.get("last_think_time",0.0)
+    
+    def load_memory_meta(self):
+        meta_file_path = f"{self.agent_memory_base_dir}/meta.json"
+        try:
+            with open(meta_file_path, mode='r') as file:
+                meta = json.load(file)
+                self.load_meta(meta)
+
+        except Exception as e:
+            logger.error(f"load memory meta failed: {e}")
+            self.last_think_time = 0.0
+
+    def save_memory_meta(self):
+        meta_file_path = f"{self.agent_memory_base_dir}/meta.json"
+        try:
+            with open(meta_file_path, mode='w') as file:
+                meta = self.memory_meta_to_dict()
+                json.dump(meta,file)
+        except Exception as e:
+            logger.error(f"save memory meta failed: {e}")
+
+    async def get_last_think_time(self)->float:
+        return self.last_think_time
+
+    async def set_last_think_time(self,last_time:float):
+        self.last_think_time = last_time    
+        self.save_memory_meta()
+
     async def get_contact_summary(self,contact_id:str) -> str:
         if contact_id is None:
-            return None
+            return "Contact id is None"
         
-        # TODO : fix this by system config.
-        if contact_id == "lzc":
-            return "lzc is your master. Male, 40 years old, Mother tongue is Chinese, senior software engineer."
-        return None
+        result = {}
+        contact_info:Contact = ContactManager.get_instance().find_contact_by_name(contact_id)
+        if contact_info:
+            result["name"] = contact_info.name
+            result["relation"] = contact_info.relationship
+            result["notes"]  = contact_info.notes
+
+        summary_path = f"{self.agent_memory_base_dir}/contacts/{contact_id}.summary"
+        try:
+            async with aiofiles.open(summary_path, mode='r') as file:
+                result["summary"] =  await file.read()
+                
+        except Exception as e:
+            logger.error(f"read contact summary failed: {e}")
+            
+        return json.dumps(result,ensure_ascii=False)
     
-    async def update_contact_summary(self,contact_id:str,summary:str) -> str:
-        return "OK"
+    async def update_contact_summary(self,contact_id:str,summary:str):
+        summary_path = f"{self.agent_memory_base_dir}/contacts/{contact_id}.summary"
+        try:
+            async with aiofiles.open(summary_path, mode='w') as file:
+                await file.write(summary)
+                return "OK"
+        except Exception as e:
+            logger.error(f"write contact summary failed: {e}")
+            return "write contact summary failed: {e}"
     
-    async def get_sth_summary(self,sth_id:str) -> str:
-        return None
+    async def get_summary(self,object_name:str) -> str:
+        summary_path = f"{self.agent_memory_base_dir}/{object_name}.summary"
+        try:
+            async with aiofiles.open(summary_path, mode='r') as file:
+                return await file.read()
+        except Exception as e:
+            logger.error(f"read summary failed: {e}")
+            return f"read summary failed: {e}"
+        
+    async def update_summary(self,object_name:str,summary:str) -> str:
+        summary_path = f"{self.agent_memory_base_dir}/{object_name}.summary"
+        try:
+            async with aiofiles.open(summary_path, mode='w') as file:
+                await file.write(summary)
+                return "OK"
+        except Exception as e:
+            logger.error(f"write summary failed: {e}")
+            return f"write summary failed: {e}"
+    
+    async def list_summary_object_names(self) -> List[str]:
+        # list dir
+        try:
+            contents = os.listdir(self.agent_memory_base_dir)
+            return [x for x in contents if x.endswith(".summary")]
+        except Exception as e:
+            logger.error(f"list summary object names failed: {e}")
+            return []
+        
+    # means object1 feel object2 is ... 
+    async def get_relation_summary(self,object_name1:str,object_name2:str) -> str:
+        summary_path = f"{self.agent_memory_base_dir}/relations/{object_name1}.relation.{object_name2}.summary"
+        try:
+            async with aiofiles.open(summary_path, mode='r') as file:
+                await file.read()
+        except FileNotFoundError:
+            return "no summary"
+        except Exception as e:
+            logger.error(f"read relation summary failed: {e}")
+            return f"read relation summary failed: {e}"
+        
+    
+    async def update_relation_summary(self,object_name1:str,object_name2:str,summary:Dict):
+        summary_path = f"{self.agent_memory_base_dir}/relations/{object_name1}.relation.{object_name2}.summary"
+        try:
+            async with aiofiles.open(summary_path, mode='w') as file:
+                await file.write(json.dumps(summary))
+                return "OK"
+        except Exception as e:
+            logger.error(f"write relation summary failed: {e}")
+            return "write relation summary failed: {e}"
+    
+    async def get_experience(self,topic_name:str) -> str:
+        experience_path = f"{self.agent_memory_base_dir}/experience/{topic_name}.experience"
+        try:
+            async with aiofiles.open(experience_path, mode='r') as file:
+                await file.read()
+        except FileNotFoundError:
+            return "no experience"
+        except Exception as e:
+            logger.error(f"read experience failed: {e}")
+            return f"read experience failed: {e}"
+
+    
+    async def set_experience(self,topic_name:str,summary:str) -> str:
+        experience_path = f"{self.agent_memory_base_dir}/experience/{topic_name}.experience"
+        try:
+            async with aiofiles.open(experience_path, mode='w') as file:
+                await file.write(summary)
+                return "OK"
+        except Exception as e:
+            logger.error(f"write experience failed: {e}")
+            return "write experience failed: {e}"
+    
+    async def list_experience(self) -> List[str]:
+        dir_path = f"{self.agent_memory_base_dir}/experience"
+        try:
+            contents = os.listdir(dir_path)
+            return [x for x in contents if x.endswith(".experience")]
+        except Exception as e:
+            logger.error(f"list experience failed: {e}")
+            return []
+
+    @staticmethod
+    def register_ai_functions():
+        async def get_contact_summary(parameters):
+            agent_memory:AgentMemory = parameters.get("_agent_memory")
+            contact_name = parameters.get("contact_name")
+            return await agent_memory.get_contact_summary(contact_name)
+        parameters = ParameterDefine.create_parameters({
+            "contact_name": {"type": "string", "description": "contact name"}
+        })
+        get_contact_summary_func = SimpleAIFunction("agent.memory.get_contact_summary",
+                                                    "get contact summary",
+                                                    get_contact_summary,
+                                                    parameters)
+        GlobaToolsLibrary.register_tool_function(get_contact_summary_func)
+
+        async def update_contact_summary(parameters):
+            agent_memory:AgentMemory = parameters.get("_agent_memory")
+            contact_name = parameters.get("contact_name")
+            summary = parameters.get("summary")
+            return await agent_memory.update_contact_summary(contact_name,summary)
+        parameters = ParameterDefine.create_parameters({
+            "contact_name": {"type": "string", "description": "contact name"},
+            "summary": {"type": "string", "description": "new summary"}
+        })
+        update_contact_summary_func = SimpleAIFunction("agent.memory.update_contact_summary",
+                                                    "update contact summary",
+                                                    update_contact_summary,
+                                                    parameters)
+        GlobaToolsLibrary.register_tool_function(update_contact_summary_func)
+
+        async def get_summary(parameters):
+            agent_memory:AgentMemory = parameters.get("_agent_memory")
+            object_name = parameters.get("object_name")
+            return await agent_memory.get_summary(object_name)
+        parameters = ParameterDefine.create_parameters({
+            "object_name": {"type": "string", "description": "object name"}
+        })
+        get_summary_func = SimpleAIFunction("agent.memory.get_summary",
+                                                    "get summary of sth",
+                                                    get_summary,
+                                                    parameters)
+        GlobaToolsLibrary.register_tool_function(get_summary_func)
+
+        async def update_summary(parameters):
+            agent_memory:AgentMemory = parameters.get("_agent_memory")
+            object_name = parameters.get("object_name")
+            summary = parameters.get("summary")
+            return await agent_memory.update_summary(object_name,summary)
+        parameters = ParameterDefine.create_parameters({
+            "object_name": {"type": "string", "description": "object name"},
+            "summary": {"type": "string", "description": "new summary"}
+        })
+        update_summary_func = SimpleAIFunction("agent.memory.update_summary",
+                                                    "update summary of sth",
+                                                    update_summary,
+                                                    parameters)
+        GlobaToolsLibrary.register_tool_function(update_summary_func)
+
+        async def list_summary_object_names(parameters):
+            agent_memory:AgentMemory = parameters.get("_agent_memory")
+            return await agent_memory.list_summary_object_names()
+        parameters = ParameterDefine.create_parameters({})
+        list_summary_object_names_func = SimpleAIFunction("agent.memory.list_summary",
+                                                    "list summary object names",
+                                                    list_summary_object_names,
+                                                    parameters)
+        GlobaToolsLibrary.register_tool_function(list_summary_object_names_func)
+
+        async def get_relation_summary(parameters):
+            agent_memory:AgentMemory = parameters.get("_agent_memory")
+            object_name1 = parameters.get("object1_name")
+            object_name2 = parameters.get("object2_name")
+            return await agent_memory.get_relation_summary(object_name1,object_name2)
+        parameters = ParameterDefine.create_parameters({
+            "object1_name": {"type": "string", "description": "object name1"},
+            "object2_name": {"type": "string", "description": "object name2"}
+        })
+        get_relation_summary_func = SimpleAIFunction("agent.memory.get_relation_summary",
+                                                    "object1 feel object2 is ...",
+                                                    get_relation_summary,
+                                                    parameters)
+        GlobaToolsLibrary.register_tool_function(get_relation_summary_func)
+
+        async def update_relation_summary(parameters):
+            agent_memory:AgentMemory = parameters.get("_agent_memory")
+            object_name1 = parameters.get("object1_name")
+            object_name2 = parameters.get("object2_name")
+            summary = parameters.get("summary")
+            return await agent_memory.update_relation_summary(object_name1,object_name2,summary)
+        parameters = ParameterDefine.create_parameters({
+            "object1_name": {"type": "string", "description": "object name1"},
+            "object2_name": {"type": "string", "description": "object name2"},
+            "summary": {"type": "string", "description": "new summary"}
+        })
+        update_relation_summary_func = SimpleAIFunction("agent.memory.update_relation_summary",
+                                                    "object1 feel object2 is ...",
+                                                    update_relation_summary,
+                                                    parameters)
+        GlobaToolsLibrary.register_tool_function(update_relation_summary_func)
+
+        async def get_experience(parameters):
+            agent_memory:AgentMemory = parameters.get("_agent_memory")
+            topic_name = parameters.get("topic_name")
+            return await agent_memory.get_experience(topic_name)
+        parameters = ParameterDefine.create_parameters({
+            "topic_name": {"type": "string", "description": "topic name"}
+        })
+        get_experience_func = SimpleAIFunction("agent.memory.get_experience",
+                                                    "get experience",
+                                                    get_experience,
+                                                    parameters)
+        GlobaToolsLibrary.register_tool_function(get_experience_func)
+
+        async def set_experience(parameters):
+            agent_memory:AgentMemory = parameters.get("_agent_memory")
+            topic_name = parameters.get("topic_name")
+            summary = parameters.get("summary")
+            return await agent_memory.set_experience(topic_name,summary)
+        parameters = ParameterDefine.create_parameters({
+            "topic_name": {"type": "string", "description": "topic name"},
+            "summary": {"type": "string", "description": "new summary"}
+        })
+        set_experience_func = SimpleAIFunction("agent.memory.set_experience",
+                                                    "set experience",
+                                                    set_experience,
+                                                    parameters)
+        GlobaToolsLibrary.register_tool_function(set_experience_func)
+
+        async def list_experience(parameters):
+            agent_memory:AgentMemory = parameters.get("_agent_memory")
+            return await agent_memory.list_experience()
+        parameters = ParameterDefine.create_parameters({})
+        list_experience_func = SimpleAIFunction("agent.memory.list_experience",
+                                                    "list exist experience topics",
+                                                    list_experience,
+                                                    parameters)
+        GlobaToolsLibrary.register_tool_function(list_experience_func)
+
+        
+
     
-    async def update_sth_summary(self,sth_id:str,summary:str) -> str:
-        return None
     
     
 

src/aios/agent/chatsession.py

@@ -211,6 +211,23 @@ class ChatSessionDB:
             logging.error("Error occurred while getting messages: %s", e)
             return -1, None  # return -1 and None if an error occurs
         
+    def load_message_by_agentid(self,agent_id,limit,start_time="1970-01-01 00:00:00"):
+        try:
+            conn = self._get_conn()
+            cursor = conn.cursor()
+            cursor.execute("""
+                SELECT MessageID, SessionID, MsgType, PrevMsgID, SenderID, ReceiverID, Timestamp, Topic,Mentions,ContentMIME,Content,ActionName,ActionParams,ActionResult,DoneTime,Status FROM Messages
+                WHERE SenderID = ? or ReceiverID =? AND Timestamp > ?
+                ORDER BY Timestamp 
+                LIMIT ? 
+            """, (agent_id, agent_id, start_time,limit))
+            results = cursor.fetchall()
+            #self.close()
+            return results  # return 0 and the result if successful
+        except Error as e:
+            logging.error("Error occurred while getting messages: %s", e)
+            return -1, None
+        
     # read message from  now->beign    
     def get_messages(self, session_id, limit, offset):
         """ retrieve messages of a session with pagination """
@@ -287,6 +304,38 @@ class AIChatSession:
     #        cls._dbs[db_path] = db
     #    db.get_chatsession_by_id(session_id)
     #    #result = AIChatSession()
+    @classmethod
+    # start_time is a string like "2021-01-01 00:00:00"
+    def load_message_records_by_agentid(cls,agent_id:str,start_time:str,limit:int,db_path:str)->List[AgentMsg]:
+        db = cls._dbs.get(db_path)
+        if db is None:
+            db = ChatSessionDB(db_path)
+            cls._dbs[db_path] = db
+        msgs = db.load_message_by_agentid(agent_id,start_time,limit)
+        result = []
+        for msg in msgs:
+            agent_msg = AgentMsg()
+            agent_msg.msg_id = msg[0]
+            agent_msg.session_id = msg[1]
+            agent_msg.msg_type = AgentMsgType(msg[2])
+            agent_msg.prev_msg_id = msg[3]
+            agent_msg.sender = msg[4]
+            agent_msg.target = msg[5]
+            agent_msg.create_time = msg[6]
+            agent_msg.topic = msg[7]
+            if msg[8] is not None:
+                agent_msg.mentions = json.loads(msg[8])
+            agent_msg.body_mime = msg[9]
+            agent_msg.body = msg[10]
+            agent_msg.func_name = msg[11]
+            if msg[12] is not None:
+                agent_msg.args = json.loads(msg[12])
+            agent_msg.result_str = msg[13]
+            agent_msg.done_time = msg[14]
+            agent_msg.status = AgentMsgStatus(msg[15])
+
+            result.append(agent_msg)
+        return result
 
     @classmethod
     def get_session(cls,owner_id:str,session_topic:str,db_path:str,auto_create = True) -> 'AIChatSession':

src/aios/agent/llm_process.py

@@ -42,6 +42,9 @@ class BaseLLMProcess(ABC):
 
         self.llm_context:LLMProcessContext = None
 
+    def get_llm_model_name(self) -> str:
+        return self.model_name
+
     @abstractmethod
     async def prepare_prompt(self,input:Dict) -> LLMPrompt:
         pass
@@ -123,10 +126,11 @@ class BaseLLMProcess(ABC):
         else:
             inner_functions = None
        
+
         task_result: ComputeTaskResult = await (ComputeKernel.get_instance().do_llm_completion(
             prompt,
             resp_mode=resp_mode,
-            mode_name=self.model_name,
+            mode_name=self.get_llm_model_name(),
             max_token=max_result_token,
             inner_functions=inner_functions, #NOTICE: inner_function in prompt can be a subset of get_inner_function
             timeout=self.timeout))
@@ -166,7 +170,7 @@ class BaseLLMProcess(ABC):
         task_result: ComputeTaskResult = await (ComputeKernel.get_instance().do_llm_completion(
                 prompt,
                 resp_mode=resp_mode,
-                mode_name=self.model_name,
+                mode_name=self.get_llm_model_name(),
                 max_token=max_result_token,
                 inner_functions=prompt.inner_functions, #NOTICE: inner_function in prompt can be a subset of get_inner_function
                 timeout=self.timeout))
@@ -309,6 +313,9 @@ class LLMAgentBaseProcess(BaseLLMProcess):
 class AgentMessageProcess(LLMAgentBaseProcess):
     def __init__(self) -> None:
         super().__init__()
+        self.mutil_model = None
+        self.enable_media2text = False
+        self.is_mutil_model = False
 
     async def load_default_config(self) -> bool:
         return True
@@ -319,8 +326,18 @@ class AgentMessageProcess(LLMAgentBaseProcess):
 
         if await super().load_from_config(config) is False:
             return False
-         
 
+        self.enable_media2text = config.get('enable_media2text', 'false').lower() in ('true', '1', 't', 'y', 'yes')
+
+        if config.get("mutil_model"):
+            self.mutil_model = config.get("mutil_model")
+         
+    def get_llm_model_name(self) -> str:
+        if self.is_mutil_model:
+            return self.mutil_model
+        else:
+            return self.model_name
+    
     def check_and_to_base64(self, image_path: str) -> str:
         if image_utils.is_file(image_path):
             return image_utils.to_base64(image_path, (1024, 1024))
@@ -329,14 +346,24 @@ class AgentMessageProcess(LLMAgentBaseProcess):
              
     async def get_prompt_from_msg(self,msg:AgentMsg) -> LLMPrompt:
         msg_prompt = LLMPrompt()
-        if msg.is_image_msg():
-            image_prompt, images = msg.get_image_body()
-            if image_prompt is None:
-                msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images]}]
+        self.is_mutil_model = False
+        if msg.is_image_msg():  
+            if self.enable_media2text:
+                logger.error(f"enable_media2text is not supported yet")
             else:
-                content = [{"type": "text", "text": image_prompt}]
-                content.extend([{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images])
-                msg_prompt.messages = [{"role": "user", "content": content}]
+                image_prompt, images = msg.get_image_body()
+                if image_prompt is None:
+                    msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images]}]
+                else:
+                    content = [{"type": "text", "text": image_prompt}]
+                    content.extend([{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images])
+                    msg_prompt.messages = [{"role": "user", "content": content}]
+                
+                if self.mutil_model:
+                    self.is_mutil_model = True
+                else:
+                    logger.warning(f"mutil_model is not set!")
+                
         elif msg.is_video_msg():
             video_prompt, video = msg.get_video_body()
             frames = video_utils.extract_frames(video, (1024, 1024))
@@ -459,25 +486,7 @@ class AgentMessageProcess(LLMAgentBaseProcess):
 
         return True
 
-class AgentSelfLearning(BaseLLMProcess):
-    def __init__(self) -> None:
-        super().__init__()
-
-    async def load_from_config(self, config: dict) -> Coroutine[Any, Any, bool]:
-        if await super().load_from_config(config) is False:
-            return False
-
-    async def prepare_prompt(self) -> LLMPrompt:
-        prompt = LLMPrompt()
-        pass  
-
-    async def get_inner_function_for_exec(self,func_name:str) -> AIFunction:
-        pass
-
-    async def post_llm_process(self,actions:List[ActionNode]) -> bool:
-        pass
-
-class AgentSelfThinking(BaseLLMProcess):
+class AgentSelfThinking(LLMAgentBaseProcess):
     def __init__(self) -> None:
         super().__init__()
 
@@ -553,6 +562,68 @@ class AgentSelfThinking(BaseLLMProcess):
                 chatsession.update_think_progress(next_pos,new_summary)
         return
 
+    async def prepare_prompt(self,input:Dict) -> LLMPrompt:
+        prompt = LLMPrompt()
+
+        record_list = input.get("record_list")
+        context_info = input.get("context_info")
+        
+        if record_list is None:
+            logger.error(f"AgentSelfThinking prepare_prompt failed! input  not found")
+            return None
+        
+        prompt.append_user_message(json.dumps(record_list,ensure_ascii=False))
+        system_prompt_dict = self.prepare_role_system_prompt(context_info)
+
+        # Known_info is the SESSION summary of the existence, the current task work record summary,
+        known_info = {}
+        have_known_info = False
+        known_session_list = input.get("known_session_list")
+        known_task_list = input.get("known_task_list")
+        known_contact_list = input.get("known_contact_list")
+        known_experience_list = input.get("known_experience_list")
+        if known_session_list:
+            known_info["known_session_list"] = known_session_list
+            have_known_info = True
+        if known_task_list:
+            known_info["known_task_list"] = known_task_list
+            have_known_info = True
+        if known_contact_list:
+            known_info["known_contact_list"] = known_contact_list
+            have_known_info = True
+        if known_experience_list:
+            known_info["known_experience_list"] = known_experience_list
+            have_known_info = True
+        
+        if have_known_info:
+            system_prompt_dict["known_info"] = known_info
+
+        prompt.inner_functions =LLMProcessContext.aifunctions_to_inner_functions(self.llm_context.get_all_ai_functions())
+        prompt.append_system_message(json.dumps(system_prompt_dict,ensure_ascii=False))
+
+    async def post_llm_process(self,actions:List[ActionNode],input:Dict,llm_result:LLMResult) -> bool:
+        action_params = {}
+        action_params["_input"] = input
+        action_params["_memory"] = self.memory
+        action_params["_workspace"] = self.workspace
+        action_params["_llm_result"] = llm_result
+        action_params["_agentid"] = self.memory.agent_id
+        action_params["_start_at"] = datetime.now()
+        try:
+            if await self._execute_actions(actions,action_params) is False:
+                result_str = "execute action failed!"
+        except Exception as e:
+            logger.error(f"execute action failed! {e}")
+            result_str = "execute action failed!,error:" + str(e)
+
+class AgentSelfLearning(BaseLLMProcess):
+    def __init__(self) -> None:
+        super().__init__()
+
+    async def load_from_config(self, config: dict) -> Coroutine[Any, Any, bool]:
+        if await super().load_from_config(config) is False:
+            return False
+
     async def prepare_prompt(self) -> LLMPrompt:
         prompt = LLMPrompt()
         pass  

src/aios/agent/workspace.py

@@ -21,7 +21,7 @@ logger = logging.getLogger(__name__)
 class LocalAgentTaskManger(AgentTaskManager):
     def __init__(self, owner_id):
         super().__init__() 
-        self.root_path = f"{AIStorage.get_instance().get_myai_dir()}/workspaces/{owner_id}_workspace"
+        self.root_path = f"{AIStorage.get_instance().get_myai_dir()}/agent_data/{owner_id}/workspace/"
         #self.root_path = os.path.join(workspace, list_type)
         if not os.path.exists(self.root_path):
             os.makedirs(self.root_path)

src/aios/environment/workflow_env.py

@@ -13,7 +13,7 @@ from ..agent.llm_context import GlobaToolsLibrary
 from ..proto.compute_task import *
 from ..proto.ai_function import *
 from ..frame.compute_kernel import ComputeKernel
-from ..frame.contact_manager import ContactManager,Contact,FamilyMember
+from ..frame.contact_manager import ContactManager,Contact
 from ..storage.storage import AIStorage
 
 from .environment import SimpleEnvironment, CompositeEnvironment

src/aios/frame/compute_kernel.py

@@ -107,7 +107,7 @@ class ComputeKernel:
     @staticmethod
     def llm_num_tokens_from_text(text:str,model:str) -> int:
         if model is None:
-            model = "gpt4"
+            model = "gpt-4-turbo-preview"
 
         try:
             encoding = tiktoken.encoding_for_model(model)

src/aios/frame/contact.py

@@ -18,6 +18,7 @@ class Contact:
         self.notes = notes
         self.is_family_member = False
         self.active_tunnels = {}
+        self.relationship = "friends"
 
     def to_dict(self):
         return {
@@ -25,11 +26,13 @@ class Contact:
             "phone": self.phone,
             "email": self.email,
             "telegram" : self.telegram,
+            "is_family_member": self.is_family_member,
 
             "added_by": self.added_by,
             "tags": self.tags,
             "notes": self.notes,
-            "now" : datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+            "now" : datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
+            "relationship" : self.relationship
         }
     
     async def _process_msg(self,msg:AgentMsg):
@@ -55,6 +58,7 @@ class Contact:
         self.active_tunnels[agent_id] = tunnel
     
     async def create_default_tunnel(self,agent_id:str) -> AgentTunnel:
+        #TODO:fix this
         from .email_tunnel import EmailTunnel
 
         result_tunnels = AgentTunnel.get_tunnel_by_agentid(agent_id)
@@ -66,20 +70,7 @@ class Contact:
 
     @classmethod
     def from_dict(cls, data):
-        return Contact(data.get("name"), data.get("phone"), data.get("email"), data.get("telegram"),data.get("added_by"), data.get("tags"), data.get("notes"))
-
-class FamilyMember(Contact):
-    def __init__(self, name, relationship,phone=None, email=None,telegram=None):
-        super().__init__(name, phone, email, telegram)
-        self.name = name
-        self.relationship = relationship  
-        self.is_family_member = True
-
-    def to_dict(self):
-        result = super().to_dict()
-        result["relationship"] = self.relationship
+        result = Contact(data.get("name"), data.get("phone"), data.get("email"), data.get("telegram"),data.get("added_by"), data.get("tags"), data.get("notes"))
+        if data.get("relationship") is not None:
+            result.relationship = data.get("relationship")
         return result
-
-    @classmethod
-    def from_dict(cls, data):
-        return FamilyMember(data.get("name"),data.get("relationship"),data.get("phone"), data.get("email"),data.get("telegram"))

src/aios/frame/contact_manager.py

@@ -8,7 +8,7 @@ from ..proto.agent_msg import AgentMsg
 from ..proto.ai_function import ParameterDefine, SimpleAIFunction
 from ..agent.llm_context import GlobaToolsLibrary
 from .tunnel import AgentTunnel
-from .contact import Contact,FamilyMember
+from .contact import Contact
 
 
 logger = logging.getLogger(__name__)    
@@ -25,11 +25,12 @@ class ContactManager:
     def register_global_functions(self):
         gl = GlobaToolsLibrary.get_instance()
 
-        get_parameters = ParameterDefine.create_parameters({"name":"name"})
+        get_parameters = ParameterDefine.create_parameters({"name":"contact name name"})
         gl.register_tool_function(SimpleAIFunction("system.contacts.get",
                                         "get contact info",
                                         self._get_contact,get_parameters))
 
+        # todo: use json to save contact info
         update_parameters = ParameterDefine.create_parameters({"name":"name","contact_info":"A json to descrpit contact"})
         gl.register_tool_function(SimpleAIFunction("system.contacts.set",
                                         "set contact info",
@@ -42,7 +43,6 @@ class ContactManager:
     def __init__(self, filename="contacts.toml"):
         self.filename = filename
         self.contacts = []
-        self.family_members = []    
 
         self.is_auto_create_contact_from_telegram = True
     
@@ -56,12 +56,10 @@ class ContactManager:
         
     def load_from_config(self,config_data:dict):
         self.contacts = [Contact.from_dict(item) for item in config_data.get("contacts", [])]
-        self.family_members = [FamilyMember.from_dict(item) for item in config_data.get("family_members", [])]
-    
+
     def save_data(self):
         data = {
             "contacts": [contact.to_dict() for contact in self.contacts],
-            "family_members": [member.to_dict() for member in self.family_members]
         }
         with open(self.filename, "w") as f:
             toml.dump(data, f)
@@ -95,54 +93,28 @@ class ContactManager:
             if contact.name == name:
                 return contact
             
-        for member in self.family_members:
-            if member.name == name:
-                return member
         return None
     
     def find_contact_by_telegram(self, telegram:str):
         for contact in self.contacts:
             if contact.telegram == telegram:
                 return contact
-        for member in self.family_members:
-            if member.telegram == telegram:
-                return member            
+         
         return None
 
     def find_contact_by_email(self, email:str):
         for contact in self.contacts:
             if contact.email == email:
                 return contact
-        for member in self.family_members:
-            if member.email == email:
-                return member   
+
         return None
 
     def find_contact_by_phone(self, phone:str):
         for contact in self.contacts:
             if contact.phone == phone:
                 return contact
-        for member in self.family_members:
-            if member.phone == phone:
-                return member   
+
         return None
 
-
-    def add_family_member(self, name, new_member:FamilyMember):
-        assert name == new_member.name
-        self.family_members.append(new_member)
-        self.save_data()
-
     def list_contacts(self):
         return self.contacts
-
-    def list_family_members(self):
-        return self.family_members
-    
-    #def register_to_ai_bus(self, ai_bus:AIBus):
-    #    ai_bus.register_message_handler("contact_manager", self.process_msg)
-    
-
-    #async def process_msg(self,msg:AgentMsg):
-    #    # forword message to contact    
-    #    pass

src/aios/knowledge/store.py

@@ -15,7 +15,7 @@ class KnowledgeStore:
         if cls._instance is None:
             cls._instance = super().__new__(cls)
             
-            knowledge_dir = AIStorage.get_instance().get_myai_dir() / "knowledge" / "objects"
+            knowledge_dir = f"{AIStorage.get_instance().get_myai_dir()}/knowledge/objects"
 
             if not os.path.exists(knowledge_dir):
                 os.makedirs(knowledge_dir)

src/aios/storage/storage.py

@@ -40,7 +40,7 @@ class UserConfig:
         self.config_table = {}
         self.user_config_path:str = None
 
-        self._init_default_value("llm_model_name","gpt-4-1106-preview")
+        self._init_default_value("llm_model_name","gpt-4-turbo-preview")
 
     def _init_default_value(self,key:str,value:Any) -> None:
         if self.config_table.get(key) is not None:
@@ -92,7 +92,7 @@ class UserConfig:
                     os.makedirs(directory)
 
                 async with aiofiles.open(self.user_config_path,"w") as f:
-                    toml_str = toml.dumps(will_save_config,ensure_ascii=False)
+                    toml_str = toml.dumps(will_save_config)
                     await f.write(toml_str)
             except Exception as e:
                 logger.error(f"save user config to {self.user_config_path} failed!")
@@ -156,7 +156,7 @@ class AIStorage:
         self.feature_init_results = {}
 
     async def initial(self)->bool:
-        self.user_config.user_config_path = str(self.get_myai_dir() / "etc/system.cfg.toml")
+        self.user_config.user_config_path = str(self.get_myai_dir() + "/etc/system.cfg.toml")
         await self.user_config.load_value_from_file(self.get_system_dir() + "/system.cfg.toml")
         await self.user_config.load_value_from_file(self.user_config.user_config_path,True)
 
@@ -215,7 +215,7 @@ class AIStorage:
         my ai dir is the dir for user to store their ai app and data
         ~/myai/
         """
-        return Path.home() / "myai"
+        return f"{Path.home()}/myai"
     
     def get_download_dir(self) -> str:
         """
@@ -249,3 +249,10 @@ class AIStorage:
 
         except Exception as e:
             logger.error(f"open or create file {path} failed! {str(e)}")
+
+    @staticmethod
+    def ensure_directory_exists(directory_path):
+        if not os.path.exists(directory_path):
+            os.makedirs(directory_path)
+            logger.info(f"Directory created: {directory_path}")
+

src/component/agent_manager/agent_manager.py

@@ -72,8 +72,6 @@ class AgentManager:
             logger.warn(f"load agent {agent_id} from media failed!")
             return None
         
-        the_agent.memory_db = f"{self.agent_memory_base_dir}/{agent_id}/{agent_id}_memory.db"
-        os.makedirs(os.path.dirname(the_agent.memory_db),exist_ok=True)
         if await the_agent.initial():
             return the_agent
         else:

src/component/common_environment/local_document.py

@@ -453,7 +453,7 @@ class ParseLocalDocument:
         return f"# Known information:\n## Current directory structure:\n{kb_tree}\n## Knowlege Metadata:\n{json.dumps(known_obj,ensure_ascii=False)}\n"
 
     def _token_len(self, text: str) -> int:
-        return CustomAIAgent("", "gpt-4-1106-preview", self.token_limit).token_len(text=text)
+        return CustomAIAgent("", "gpt-4-turbo-preview", self.token_limit).token_len(text=text)
 
 
     async def _learn_by_agent(self, meta:dict) -> dict:

src/component/openai_node/dall_e_compute_node.py

@@ -64,7 +64,7 @@ class DallEComputeNode(ComputeNode):
             self.output_dir = "./"
             self.output_dir = os.path.abspath(self.output_dir)
 
-        self.start()
+        await self.start()
 
         return True
 

src/component/openai_node/open_ai_node.py

@@ -209,7 +209,7 @@ class OpenAI_ComputeNode(ComputeNode):
                     if max_token_size > 4096:
                         result_token = 4096
                     else:
-                        result_token = max_token_size
+                        result_token = -1
                 else:
                     result_token = NOT_GIVEN
 

src/component/tg_tunnel.py

@@ -12,7 +12,7 @@ from telegram import Bot
 from telegram.ext import Updater
 from telegram.error import Forbidden, NetworkError
 
-from aios import ObjectType, KnowledgeStore,AgentTunnel,AIStorage,ContactManager,Contact,FamilyMember,AgentMsg,AgentMsgType
+from aios import ObjectType, KnowledgeStore,AgentTunnel,AIStorage,ContactManager,Contact,AgentMsg,AgentMsgType
 
 logger = logging.getLogger(__name__)
 

src/service/aios_shell/aios_shell.py

@@ -76,8 +76,9 @@ class AIOS_Shell:
 
         user_config = AIStorage.get_instance().get_user_config()
         user_config.add_user_config("username","username is your full name when using AIOS",False,None)
-        user_config.add_user_config("telegram","Your telgram username",False,None)
-        user_config.add_user_config("email","Your email",False,None)
+        user_config.add_user_config("user_telegram","Your telgram username",False,None)
+        user_config.add_user_config("user_email","Your email",False,None)
+        user_config.add_user_config("user_notes","Introduce yourself to your Agent!",False,None)
 
         user_config.add_user_config("feature.llama","enable Local-llama feature",True,"False")
         user_config.add_user_config("feature.aigc","enable AIGC feature",True,"False")
@@ -126,15 +127,16 @@ class AIOS_Shell:
 
     async def initial(self) -> bool:
         cm = ContactManager.get_instance()
-        owenr = cm.find_contact_by_name(self.username)
-        if owenr is None:
-            owenr = Contact(self.username)
-            owenr.added_by = self.username
-            owenr.is_family_member = True
-            owenr.email = AIStorage.get_instance().get_user_config().get_value("email")
-            owenr.telegram = AIStorage.get_instance().get_user_config().get_value("telegram")
+        owner = cm.find_contact_by_name(self.username)
+        if owner is None:
+            owner = Contact(self.username)
+            owner.added_by = self.username
+            owner.relationship = "Principal"
+            owner.email = AIStorage.get_instance().get_user_config().get_value("user_email")
+            owner.telegram = AIStorage.get_instance().get_user_config().get_value("user_telegram")
+            owner.notes = AIStorage.get_instance().get_user_config().get_value("user_notes")
 
-            cm.add_family_member(self.username,owenr)
+            cm.add_contact(self.username,owner)
 
         # cal_env = CalenderEnvironment("calender")
         # await cal_env.start()
@@ -247,7 +249,7 @@ class AIOS_Shell:
             if tunnel_config is not None:
                 await AgentTunnel.load_all_tunnels_from_config(tunnel_config)
         except Exception as e:
-            logger.warning(f"load tunnels config from {tunnels_config_path} failed!")
+            logger.warning(f"load tunnels config from {tunnels_config_path} failed! {e}")
 
 
         return True
@@ -385,7 +387,7 @@ class AIOS_Shell:
 
         contact_note = await try_get_input(f"Input {contact_name}'s note (optional):")
         if contact_note is not None:
-            contact.note = contact_note
+            contact.notes = contact_note
 
         contact.added_by = self.username
         if is_update:
@@ -712,9 +714,9 @@ async def get_user_config_from_input(check_result:dict) -> bool:
                 continue
             else:
                 True
-
-        if len(user_input) > 0:
-            AIStorage.get_instance().get_user_config().set_value(key,user_input)
+        if user_input:
+            if len(user_input) > 0:
+                AIStorage.get_instance().get_user_config().set_value(key,user_input)
 
     await AIStorage.get_instance().get_user_config().save_to_user_config()
     return True