Use LLMProcess implement Agent.OnMessage

src/aios/agent/agent.py

@@ -18,6 +18,7 @@ from ..proto.agent_task import *
 from ..proto.compute_task import *
 
 from .agent_base import *
+from .llm_process import *
 from .chatsession import *
 from ..environment.workspace_env import WorkspaceEnvironment, TodoListType
 
@@ -64,6 +65,8 @@ logger = logging.getLogger(__name__)
 # 我给你一段内容，尝试为期建立目录。目录的标题不能超过16个字，
 # 目录要指向正文的位置（用字符偏移即可），整个目录的文本长度不能超过256个字节。并用json表达这个目录
 # """
+
+
 class AIAgentTemplete:
     def __init__(self) -> None:
         self.llm_model_name:str = "gpt-4-0613"
@@ -73,6 +76,7 @@ class AIAgentTemplete:
         self.author:str = None
         self.prompt:LLMPrompt = None
 
+
     def load_from_config(self,config:dict) -> bool:
         if config.get("llm_model_name") is not None:
             self.llm_model_name = config["llm_model_name"]
@@ -87,9 +91,6 @@ class AIAgentTemplete:
                 return False
 
 
-        return True
-
-
 class AIAgent(BaseAIAgent):
     def __init__(self) -> None:
         self.role_prompt:LLMPrompt = None
@@ -103,7 +104,6 @@ class AIAgent(BaseAIAgent):
         self.enable_thread = False
         self.can_do_unassigned_task = True
 
-
         self.agent_id:str = None
         self.template_id:str = None
         self.fullname:str = None
@@ -135,7 +135,24 @@ class AIAgent(BaseAIAgent):
         self.owenr_bus = None
         self.enable_function_list = None
 
-    def load_from_config(self,config:dict) -> bool:
+        self.llm_process:Dict[str,BaseLLMProcess] = {}
+        
+
+    async def initial(self,params:Dict = None):
+        self.memory = AgentMemory(self.agent_id,self.chat_db)
+
+        init_params = {}
+        init_params["memory"] = self.memory
+        for process_name in self.llm_process.keys():
+            init_result = await self.llm_process[process_name].initial(init_params)
+            if init_result is False:
+                logger.error(f"llm process {process_name} initial failed! initial return False")
+                return False
+        
+        self.wake_up()
+        return True
+
+    async def load_from_config(self,config:dict) -> bool:
         if config.get("instance_id") is None:
             logger.error("agent instance_id is None!")
             return False
@@ -203,8 +220,23 @@ class AIAgent(BaseAIAgent):
             self.enable_timestamp = bool(config["enable_timestamp"])
         if config.get("history_len"):
             self.history_len = int(config.get("history_len"))
+ 
+        #load all LLMProcess
+        self.llm_process = {}
+        LLMProcess = config.get("LLMProcess")
+        for process_config_name in LLMProcess.keys():
+            process_config = LLMProcess[process_config_name]
+            real_config = {}
+            real_config.update(config)
+            real_config.update(process_config)
+            load_result = await LLMProcessLoader.get_instance().load_from_config(real_config)
+            if load_result:
+                self.llm_process[process_config_name] = load_result
+            else:
+                logger.error(f"load LLMProcess {process_config_name} failed!")
+                return False
 
-        self.wake_up()
+       
 
         return True
 
@@ -284,52 +316,14 @@ class AIAgent(BaseAIAgent):
             return image_utils.to_base64(image_path, (1024, 1024))
         else:
             return image_path
-
-    async def _process_msg(self,msg:AgentMsg,workspace = None) -> AgentMsg:
-        msg_prompt = LLMPrompt()
+        
+    async def llm_process_msg(self,msg:AgentMsg) -> AgentMsg:
+        need_process:bool = True
         if msg.msg_type == AgentMsgType.TYPE_GROUPMSG:
             need_process = False
-            if msg.is_image_msg():
-                image_prompt, images = msg.get_image_body()
-                if image_prompt is None:
-                    content = [[{"type": "text", "text": f"{msg.sender}'s message"}]]
-                    content.extend([{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images])
-                    msg_prompt.messages = [{"role": "user", "content": content}]
-                else:
-                    content = [{"type": "text", "text": f"{msg.sender}:{image_prompt}"}]
-                    content.extend([{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images])
-                    msg_prompt.messages = [{"role": "user", "content": content}]
-            elif msg.is_video_msg():
-                video_prompt, video = msg.get_video_body()
-                frames = video_utils.extract_frames(video, (1024, 1024))
-                if video_prompt is None:
-                    content = [{"type": "text", "text": f"{msg.sender}'s message"}]
-                    content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
-                    msg_prompt.messages = [{"role": "user", "content": content}]
-                else:
-                    content = [{"type": "text", "text": f"{msg.sender}:{video_prompt}"}]
-                    content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
-                    msg_prompt.messages = [{"role": "user", "content": content}]
-            elif msg.is_audio_msg():
-                prompt, audio_file = msg.get_audio_body()
-                resp = await ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text")
-                if resp.result_code != ComputeTaskResultCode.OK:
-                    error_resp = msg.create_error_resp(resp.error_str)
-                    return error_resp
-                else:
-                    if prompt is None or prompt == "":
-                        msg.body_mime = "text/plain"
-                        msg.body = resp.result_str
-                        msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{resp.result_str}"}]
-                    else:
-                        msg.body_mime = "text/plain"
-                        msg.body = f"{msg.sender} prompt:{prompt}\nasr response:{resp.result_str}"
-                        msg_prompt.messages = [{"role": "user", "content": msg.body}]
-            else:
-                msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{msg.body}"}]
+           
             session_topic = msg.target + "#" + msg.topic
             chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.chat_db)
-
             if msg.mentions is not None:
                 if self.agent_id in msg.mentions:
                     need_process = True
@@ -339,6 +333,39 @@ class AIAgent(BaseAIAgent):
                 chatsession.append(msg)
                 resp_msg = msg.create_group_resp_msg(self.agent_id,"")
                 return resp_msg
+        
+        input_parms = {
+            "msg":msg
+        }
+        msg_process = self.llm_process.get("message")
+        llm_result : LLMResult = await msg_process.process(input_parms)
+        if llm_result.state == LLMResultStates.ERROR:
+            error_resp = msg.create_error_resp(llm_result.error_str)
+            return error_resp
+        elif llm_result.state == LLMResultStates.IGNORE:
+            return None
+        else: # OK
+            resp_msg = llm_result.raw_result.get("resp_msg")
+            return resp_msg
+
+    async def _process_msg(self,msg:AgentMsg,workspace = None) -> AgentMsg:
+        msg.context_info = {}
+        msg.context_info["location"] = "SanJose"
+        msg.context_info["now"] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        msg.context_info["weather"] = "Partly Cloudy, 60°F"
+        return await self.llm_process_msg(msg)
+        msg_prompt = LLMPrompt()
+        need_process = True
+        if msg.msg_type == AgentMsgType.TYPE_GROUPMSG:
+            need_process = False
+           
+            session_topic = msg.target + "#" + msg.topic
+            chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.chat_db)
+
+            if msg.mentions is not None:
+                if self.agent_id in msg.mentions:
+                    need_process = True
+                    logger.info(f"agent {self.agent_id} recv a group chat message from {msg.sender},but is not mentioned,ignore!")
         else:
             if msg.is_image_msg():
                 image_prompt, images = msg.get_image_body()
@@ -358,20 +385,14 @@ class AIAgent(BaseAIAgent):
                     content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
                     msg_prompt.messages = [{"role": "user", "content": content}]
             elif msg.is_audio_msg():
-                prompt, audio_file = msg.get_audio_body()
+                audio_file = msg.body
                 resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text"))
                 if resp.result_code != ComputeTaskResultCode.OK:
                     error_resp = msg.create_error_resp(resp.error_str)
                     return error_resp
                 else:
-                    if prompt is None or prompt == "":
-                        msg.body_mime = "text/plain"
-                        msg.body = resp.result_str
-                        msg_prompt.messages = [{"role":"user","content":resp.result_str}]
-                    else:
-                        msg.body_mime = "text/plain"
-                        msg.body = f"user prompt:{prompt}\nasr response:{resp.result_str}"
-                        msg_prompt.messages = [{"role": "user", "content": msg.body}]
+                    msg.body = resp.result_str
+                    msg_prompt.messages = [{"role":"user","content":resp.result_str}]
             else:
                 msg_prompt.messages = [{"role":"user","content":msg.body}]
             session_topic = msg.get_sender() + "#" + msg.topic

src/aios/agent/agent_base.py

@@ -22,6 +22,7 @@ logger = logging.getLogger(__name__)
 
 
 
+
 class BaseAIAgent(abc.ABC):
     @abstractmethod
     def get_id(self) -> str:

src/aios/agent/agent_memory.py

@@ -0,0 +1,104 @@
+from ast import Dict
+from datetime import timedelta
+from typing import List
+
+from ..frame.compute_kernel import ComputeKernel
+from ..proto.ai_function import SimpleAIOperation
+
+from .chatsession import *
+
+class AgentMemory:
+    def __init__(self,agent_id:str,db_path:str) -> None:
+        self.agent_id:str= agent_id
+        self.chat_db:str = db_path
+        self.model_name:str = "gp4-1106-preview"
+        self.threshold_hours = 72
+
+        self.actions = {}
+        self.init_actions()
+
+    def init_actions(self) -> Dict:
+        chatlog_append_op = SimpleAIOperation("chatlog_append","Append request & reply message to chatlog. No params",self.action_chatlog_append)
+        self.actions[chatlog_append_op.get_name()] = chatlog_append_op
+
+    def get_session_from_msg(self,msg:AgentMsg) -> AIChatSession:
+        if msg.msg_type == AgentMsgType.TYPE_GROUPMSG:
+            session_topic = msg.target + "#" + msg.topic
+            chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.chat_db)
+        else:
+            session_topic = msg.get_sender() + "#" + msg.topic
+            chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.chat_db)
+        return chatsession
+
+    async def load_chatlogs(self,msg:AgentMsg,n:int=6,m:int=64,token_limit=800)->str:
+        chatsession = self.get_session_from_msg(msg)
+        # 必定加载n条(n>=2),期望加载m条
+        # m条里的信息逐步添加，知道距离现在的时间未72小时以上，且消耗了足够的Token
+
+        messages_n = chatsession.read_history(n) # read
+        if len(messages_n) >= n:
+            messages_m = chatsession.read_history(m,n)
+        else:
+            messages_m = []
+
+        histroy_str = ""
+        read_count = 0
+        for msg in messages_n:
+            dt = datetime.datetime.fromtimestamp(float(msg.create_time))
+            formatted_time = dt.strftime('%y-%m-%d %H:%M:%S')
+            record_str = f"{msg.sender},[{formatted_time}]\n{msg.body}\n"
+            token_limit -= ComputeKernel.llm_num_tokens_from_text(record_str,self.model_name)
+            if token_limit <= 32:
+                break
+            read_count += 1
+            histroy_str = record_str + histroy_str
+
+        if len(messages_n) > 2:
+            if read_count < 3:
+                logging.warning(f"read history {read_count} < 3, will not load more")
+
+        now = datetime.datetime.now()
+        for msg in messages_m:
+            dt = datetime.datetime.fromtimestamp(float(msg.create_time))
+            time_diff = now - dt
+            if time_diff > timedelta(hours=self.threshold_hours):
+                break
+
+            formatted_time = dt.strftime('%y-%m-%d %H:%M:%S')
+            record_str = f"{msg.sender},[{formatted_time}]\n{msg.body}\n"
+            token_limit -= ComputeKernel.llm_num_tokens_from_text(record_str,self.model_name)
+            if token_limit <= 32:
+                break
+            read_count += 1
+            histroy_str = record_str + histroy_str
+
+        return histroy_str 
+    
+    async def action_chatlog_append(self,params:Dict) -> str:
+        # 使用params可以得到: LLM Process的输入，LLM Result,基于LLM Result构造的参数，当前actionItem
+        input_msg:AgentMsg = params.get("input").get("msg")
+        llm_result = params.get("llm_result")
+        chatsession = self.get_session_from_msg(input_msg)
+        resp_msg = params.get("resp_msg")
+        if resp_msg:
+            tags =  llm_result.raw_result.get("tags")
+            chatsession.append(input_msg,tags)
+            chatsession.append(resp_msg,tags)
+    
+        return "OK"
+    
+    async def get_contact_summary(self,contact_id:str) -> str:
+        if contact_id is None:
+            return None
+        
+        if contact_id == "lzc":
+            return "lzc is your master. Male, 40 years old, Mother tongue is Chinese, senior software engineer."
+        return None
+    
+    def get_actions(self) -> Dict:
+        return self.actions
+    
+    async def get_log_summary(self,msg:AgentMsg) -> str:
+        return None
+    
+    

src/aios/agent/chatsession.py

@@ -6,6 +6,7 @@ import threading
 import datetime
 import uuid
 import json
+from typing import List
 
 from ..proto.agent_msg import AgentMsgType, AgentMsg, AgentMsgStatus
 
@@ -83,7 +84,8 @@ class ChatSessionDB:
                     ActionResult TEXT,
                     DoneTime TEXT,     
                          
-                    Status INTEGER
+                    Status INTEGER,
+                    Tags TEXT
                 );
             """)
             conn.commit()
@@ -104,7 +106,7 @@ class ChatSessionDB:
             logging.error("Error occurred while inserting session: %s", e)
             return -1  # return -1 if an error occurs
 
-    def insert_message(self, msg:AgentMsg):
+    def insert_message(self, msg:AgentMsg,tags:List[str] = None):
         """ insert a new message into the Messages table """
         try:
             action_name = None
@@ -128,13 +130,15 @@ class ChatSessionDB:
                 case AgentMsgType.TYPE_EVENT:
                     action_name = msg.event_name
                     action_params = json.dumps(msg.event_args)
+            if tags is None:
+                tags = []
 
-
+            str_tags = ','.join(tags)
             conn = self._get_conn()
             conn.execute("""
-                INSERT INTO Messages (MessageID, SessionID, MsgType, PrevMsgID, SenderID, ReceiverID, Timestamp, Topic,Mentions,ContentMIME,Content,ActionName,ActionParams,ActionResult,DoneTime,Status)
-                VALUES (?, ?, ?, ?, ?, ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?)
-            """, (msg.msg_id, msg.session_id, msg.msg_type.value, msg.prev_msg_id, msg.sender, msg.target, msg.create_time, msg.topic,mentions,msg.body_mime,msg.body,action_name,action_params,action_result,msg.done_time,msg.status.value))
+                INSERT INTO Messages (MessageID, SessionID, MsgType, PrevMsgID, SenderID, ReceiverID, Timestamp, Topic,Mentions,ContentMIME,Content,ActionName,ActionParams,ActionResult,DoneTime,Status,Tags)
+                VALUES (?, ?, ?, ?, ?, ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?,?)
+            """, (msg.msg_id, msg.session_id, msg.msg_type.value, msg.prev_msg_id, msg.sender, msg.target, msg.create_time, msg.topic,mentions,msg.body_mime,msg.body,action_name,action_params,action_result,msg.done_time,msg.status.value,str_tags))
             conn.commit()
 
             if msg.inner_call_chain:
@@ -385,9 +389,9 @@ class AIChatSession:
             result.append(agent_msg)
         return result
 
-    def append(self,msg:AgentMsg) -> None:
+    def append(self,msg:AgentMsg,tags:List[str] = None) -> None:
         msg.session_id = self.session_id
-        self.db.insert_message(msg)
+        self.db.insert_message(msg,tags)
 
 
     def update_think_progress(self,progress:int,new_summary:str) -> None:

src/aios/agent/llm_process.py

@@ -3,34 +3,90 @@ from abc import ABC,abstractmethod
 import copy
 import json
 import shlex
-from typing import Any, Callable, Optional,Dict,Awaitable,List
+from typing import Any, Callable, Coroutine, Optional,Dict,Awaitable,List
 from enum import Enum
 
+from aios.agent.chatsession import AIChatSession
+
+from ..utils import video_utils
+
 from ..proto.compute_task import *
 from ..proto.ai_function import *
+
+from .agent_base import *
+from .agent_memory import *
+
 from ..frame.compute_kernel import *
+from ..environment.environment import *
+from ..environment.workspace_env import *
 
 import logging
 logger = logging.getLogger(__name__)
 
 MIN_PREDICT_TOKEN_LEN = 32
 
-
-class BaseLLMProcess:
+class LLMProcessContext:
     def __init__(self) -> None:
+        pass
+
+class BaseLLMProcess(ABC):
+    def __init__(self) -> None:
+        self.behavior:str = None #行为名字
+        self.goal:str = None #目标
+        self.input_example:str= None #输入样例
+        self.result_example:str = None #llm_result样例
+        
         self.enable_json_resp = False
         self.model_name = "gpt-4"
-        self.max_token = 2000 # include input prompt
+        self.max_token = 1000 # result_token
+        self.max_prompt_token = 1000 # not include input prompt
         self.timeout = 1800 # 30 min
 
+        self.envs : Dict[str,BaseEnvironment] = []
+        self.env : CompositeEnvironment = None
+
     @abstractmethod
-    async def prepare_prompt(self) -> LLMPrompt:
+    async def prepare_prompt(self,input:Dict) -> LLMPrompt:
         pass
 
     @abstractmethod
     async def get_inner_function(self,func_name:str) -> AIFunction:
         pass
 
+    @abstractmethod
+    async def exec_actions(self,actions:List[ActionItem],input:Dict,llm_result:LLMResult) -> bool:
+        pass
+
+    @abstractmethod
+    async def load_from_config(self,config:dict) -> bool:
+        #self.behavior = config.get("behavior")
+        #self.goal = config.get("goal")
+        self.input_example = config.get("input_example")
+        self.result_example = config.get("result_example")
+
+        if config.get("model_name"):
+            self.model_name = config.get("model_name")
+        if config.get("enable_json_resp"):
+            self.enable_json_resp = config.get("enable_json_resp") == "true"
+        if config.get("max_token"):
+            self.max_token = config.get("max_token")
+        if config.get("timeout"):
+            self.timeout = config.get("timeout")
+        
+
+        return True
+    
+    @abstractmethod
+    async def initial(self,params:Dict = None) -> bool:
+        pass
+
+    def append_envs(self,envs:Dict[str,BaseEnvironment]):
+        self.envs.update(envs)
+        self.env = CompositeEnvironment(self.envs)
+    
+    def _format_content_by_env_value(self,content:str,env)->str:
+        return content.format_map(env)
+
     async def _execute_inner_func(self,inner_func_call_node,prompt: LLMPrompt,stack_limit = 5) -> ComputeTaskResult:
         arguments = None
         try:
@@ -55,7 +111,7 @@ class BaseLLMProcess:
         else:
             resp_mode = "text"
 
-        max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt)
+        max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.model_name)
         if max_result_token < MIN_PREDICT_TOKEN_LEN:
             task_result = ComputeTaskResult()
             task_result.result_code = ComputeTaskResultCode.ERROR
@@ -67,7 +123,7 @@ class BaseLLMProcess:
             resp_mode=resp_mode,
             mode_name=self.model_name,
             max_token=max_result_token,
-            inner_functions=prompt.inner_functions,
+            inner_functions=prompt.inner_functions, #NOTICE: inner_function in prompt can be a subset of get_inner_function
             timeout=self.timeout))
 
         if task_result.result_code != ComputeTaskResultCode.OK:
@@ -94,23 +150,23 @@ class BaseLLMProcess:
         else:
             return task_result
 
-    async def process(self) -> LLMResult:
+    async def process(self,input:Dict) -> LLMResult:
         if self.enable_json_resp:
             resp_mode = "json"
         else:
             resp_mode = "text"
 
-        prompt = await self.prepare_prompt()
-        max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt)
+        prompt = await self.prepare_prompt(input)
+        max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.model_name)
         if max_result_token < MIN_PREDICT_TOKEN_LEN:
             return LLMResult.from_error_str(f"prompt too long,can not predict")
-
+        
         task_result: ComputeTaskResult = await (ComputeKernel.get_instance().do_llm_completion(
                 prompt,
                 resp_mode=resp_mode,
                 mode_name=self.model_name,
                 max_token=max_result_token,
-                inner_functions=prompt.inner_functions,
+                inner_functions=prompt.inner_functions, #NOTICE: inner_function in prompt can be a subset of get_inner_function
                 timeout=self.timeout))
         
         if task_result.result_code != ComputeTaskResultCode.OK:
@@ -136,12 +192,385 @@ class BaseLLMProcess:
         else:
             llm_result = LLMResult.from_str(task_result.result_str)
 
-        # execute op_list in LLM Result?
+        # use action to save history?
+        if llm_result.action_list or len(llm_result.action_list) > 0:
+            await self.exec_actions(llm_result.action_list,input,llm_result)
 
         return llm_result
     
-#class LLMProcess
+class LLMAgentMessageProcess(BaseLLMProcess):
+    def __init__(self) -> None:
+        super().__init__()
+
+        self.role_description:str = None
+        self.process_description:str = None
+        self.reply_format:str = None
+        self.context : str = None
+
+        self.known_info_tips :str = None
+        self.tools_tips:str = None
+
+        self.enable_inner_functions : Dict[str,bool] = None
+        self.enable_actions : Dict[str,AIOperation] = None
+        self.actions_desc : Dict[str,Dict] = None
+        self.workspace : WorkspaceEnvironment = None
+
+        self.memory : AgentMemory = None
+        self.enable_kb = False
+        self.kb = None
+
+    def init_actions(self):
+        self.enable_actions = {}
+        self.actions_desc = {}
+        self.enable_actions.update(self.memory.get_actions())
+        if self.workspace:
+            self.enable_actions.update(self.workspace.get_actions())
+        if self.enable_kb:
+            self.enable_actions.update(self.kb.get_actions())
+
+        for name,op in self.enable_actions.items():
+            self.actions_desc[name] = op.get_description()
+        
+    async def initial(self,params:Dict = None) -> bool:
+        self.memory = params.get("memory")
+        if self.memory is None:
+            logger.error(f"LLMAgeMessageProcess initial failed! memory not found")
+            return False
+        
+        self.init_actions()
+        return True
+
+    async def load_default_config(self) -> bool:
+        return True
+        
+        
+    async def load_from_config(self, config: dict,is_load_default=True) -> Coroutine[Any, Any, bool]:
+        if is_load_default:
+            await self.load_default_config()
+
+        if await super().load_from_config(config) is False:
+            return False
+        
+        self.role_description = config.get("role_desc")
+        if self.role_description is None:
+            logger.error(f"role_description not found in config")
+            return False
+        
+        if config.get("process_description"):
+            self.process_description = config.get("process_description")
+        
+        if config.get("reply_format"):
+            self.reply_format = config.get("reply_format")
+
+        if config.get("context"):
+            self.context = config.get("context")
+
+        if config.get("known_info_tips"):
+            self.known_info_tips = config.get("known_info_tips")
+
+        if config.get("tools_tips"):
+            self.tools_tips = config.get("tools_tips")  
+
+        if config.get("enable_kb"):
+            self.enable_kb = config.get("enable_kb") == "true"
+        
+        if config.get("enable_function"):
+            self.enable_inner_functions = config.get("enable_function")
+        
+        if config.get("enable_actions"):
+            self.enable_actions = config.get("enable_actions")
+
+        
+
+    async def get_prompt_from_msg(self,msg:AgentMsg) -> LLMPrompt:
+        msg_prompt = LLMPrompt()
+        if msg.is_image_msg():
+            image_prompt, images = msg.get_image_body()
+            if image_prompt is None:
+                msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images]}]
+            else:
+                content = [{"type": "text", "text": image_prompt}]
+                content.extend([{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images])
+                msg_prompt.messages = [{"role": "user", "content": content}]
+        elif msg.is_video_msg():
+            video_prompt, video = msg.get_video_body()
+            frames = video_utils.extract_frames(video, (1024, 1024))
+            if video_prompt is None:
+                msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": frame}} for frame in frames]}]
+            else:
+                content = [{"type": "text", "text": video_prompt}]
+                content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
+                msg_prompt.messages = [{"role": "user", "content": content}]
+        elif msg.is_audio_msg():
+            audio_file = msg.body
+            resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text"))
+            if resp.result_code != ComputeTaskResultCode.OK:
+                error_resp = msg.create_error_resp(resp.error_str)
+                return error_resp
+            else:
+                msg.body = resp.result_str
+                msg_prompt.messages = [{"role":"user","content":resp.result_str}]
+        else:
+            msg_prompt.messages = [{"role":"user","content":msg.body}]
+
+        return msg_prompt
+    
+    async def get_action_desc(self) -> Dict:
+        result = {}
+        for name,op in self.enable_actions.items():
+            result[name] = op.get_description()
+        return result
+
+    async def sender_info(self,msg:AgentMsg)->str:
+        sender_id = msg.sender
+        #TODO Is sender an agent?
+        return await self.memory.get_contact_summary(sender_id)
+
+    async def load_chatlogs(self,msg:AgentMsg)->str:
+        ## like
+        #sender,[2023-11-1 12:00:00]
+        #content
+        return await self.memory.load_chatlogs(msg)
+
+    async def get_log_summary(self,msg:AgentMsg)->str:
+        return await self.memory.get_log_summary(msg)
+        
+
+    async def get_extend_known_info(self,msg:AgentMsg,prompt:LLMPrompt)->str:
+        return None
+
+    async def prepare_prompt(self,input:Dict) -> LLMPrompt:
+        prompt = LLMPrompt()
+        # User Prompt 
+        ## Input Msg
+        msg : AgentMsg = input.get("msg")
+        if msg is None:
+            logger.error(f"LLMAgeMessageProcess prepare_prompt failed! input msg not found")
+            return None
+        msg_prompt = await self.get_prompt_from_msg(msg)
+        if msg_prompt is None:
+            logger.error(f"LLMAgeMessageProcess prepare_prompt failed! get_prompt_from_msg return None")
+            return None
+        prompt.append(msg_prompt)
+
+        system_prompt_dict = {}
+
+        # System Prompt
+        ## LLM的身份说明
+        system_prompt_dict["role_description"] = self.role_description
+        #prompt.append_system_message(self.role_description)
+
+        ## 处理信息的流程说明
+        system_prompt_dict["process_rule"] = self.process_description
+        #prompt.append_system_message(self.process_description)
+        ### 回复的格式
+        system_prompt_dict["reply_format"] = self.reply_format
+        #prompt.append_system_message(self.reply_format)
+        ### 修改chatlog的action
+        ### 修改todo/task的action
+        ### workspace提供的额外的action
+        system_prompt_dict["support_actions"] = await self.get_action_desc()
+        #prompt.append_system_message(await self.get_action_desc())
+
+        ## Context （文本替换）,是否应该覆盖全部消息
+        context = self._format_content_by_env_value(self.context,msg.context_info)
+        system_prompt_dict["context"] = context
+        #prompt.append_system_message(context)
+               
+        ## 已知信息  
+        known_info = {}
+        #prompt.append_system_message(self.known_info_tips)
+        ### 信息发送者资料
+        known_info["sender_info"] = await self.sender_info(msg)
+        #prompt.append_system_message(await self.sender_info(self,msg))
+        ### 近期的聊天记录
+        chat_record = await self.load_chatlogs(msg)
+        if chat_record:
+            if len(chat_record) > 4:
+                known_info["chat_record"] = chat_record
+        #prompt.append_system_message(await self.load_chatlogs(self,msg))
+        ### 交流总结
+        summary = await self.get_log_summary(msg)
+        if summary:
+            if len(summary) > 4:
+                known_info["summary"] = summary
+        #prompt.append_system_message(await self.get_log_summary(self,msg))
+        system_prompt_dict["known_info"] = known_info
+        
+        ## 可以使用的tools(inner function)的解释，注意不定义该tips,则不会导入任何workspace中的tools
+        if self.tools_tips:
+            system_prompt_dict["tools_tips"] = self.tools_tips
+            #prompt.append_system_message(self.tools_tips)
+            prompt.inner_functions.extend(self.get_inner_function_desc_from_env())
+
+        ## 给予查询KB的权限    
+        if self.enable_kb:        
+            prompt.inner_functions.extend(self.get_inner_function_desc_from_kb())
+
+        prompt.append_system_message(json.dumps(system_prompt_dict))
+        ## 扩展已知信息 (这可能是一个LLM过程)
+        prompt.append_system_message(await self.get_extend_known_info(msg,prompt))
+
+        return prompt
+    
+
+    async def get_inner_function(self,func_name:str) -> AIFunction:
+        return None
+
+    async def exec_actions(self,actions:List[ActionItem],input:Dict,llm_result:LLMResult) -> bool:
+        msg = input.get("msg")
+        if msg.msg_type == AgentMsgType.TYPE_GROUPMSG:
+            resp_msg = msg.create_group_resp_msg(self.memory.agent_id,llm_result.resp)
+        else:
+            resp_msg = msg.create_resp_msg(llm_result.resp)
+        
+        llm_result.raw_result["resp_msg"] = resp_msg
+
+        for action_item in actions:
+            op : AIOperation = self.enable_actions.get(action_item.name)
+            if op:
+                if action_item.parms is None:
+                    action_item.parms = {}
+
+                action_item.parms["input"] = input
+                action_item.parms["resp_msg"] = resp_msg  
+                action_item.parms["llm_result"] = llm_result
+                action_item.parms["start_at"] = datetime.now()
+                action_item.parms["result"] = await op.execute(action_item.parms)
+                action_item.parms["end_at"] = datetime.now()
+            else:
+                logger.warn(f"action {action_item.name} not found")
+                return False
             
+        return True
+
+        
+
+class ReviewTaskProcess(BaseLLMProcess):
+    def __init__(self) -> None:
+        super().__init__()
+
+    async def load_from_config(self, config: dict) -> Coroutine[Any, Any, bool]:
+        if await super().load_from_config(config) is False:
+            return False
+
+    async def prepare_prompt(self) -> LLMPrompt:
+        prompt = LLMPrompt()
+        pass  
+
+    async def get_inner_function(self,func_name:str) -> AIFunction:
+        pass
+
+    async def exec_actions(self,actions:List[ActionItem]) -> bool:
+        pass
+
+class DoTodoProcess(BaseLLMProcess):
+    def __init__(self) -> None:
+        super().__init__()
+
+    async def load_from_config(self, config: dict) -> Coroutine[Any, Any, bool]:
+        if await super().load_from_config(config) is False:
+            return False
+
+    async def prepare_prompt(self) -> LLMPrompt:
+        prompt = LLMPrompt()
+        pass  
+
+    async def get_inner_function(self,func_name:str) -> AIFunction:
+        pass
+
+    async def exec_actions(self,actions:List[ActionItem]) -> bool:
+        pass
+
+
+class CheckTodoProcess(BaseLLMProcess):
+    def __init__(self) -> None:
+        super().__init__()
+
+    async def load_from_config(self, config: dict) -> Coroutine[Any, Any, bool]:
+        if await super().load_from_config(config) is False:
+            return False
+
+    async def prepare_prompt(self) -> LLMPrompt:
+        prompt = LLMPrompt()
+        pass  
+
+    async def get_inner_function(self,func_name:str) -> AIFunction:
+        pass
+
+    async def exec_actions(self,actions:List[ActionItem]) -> bool:
+        pass
+
+class SelfLearningProcess(BaseLLMProcess):
+    def __init__(self) -> None:
+        super().__init__()
+
+    async def load_from_config(self, config: dict) -> Coroutine[Any, Any, bool]:
+        if await super().load_from_config(config) is False:
+            return False
+
+    async def prepare_prompt(self) -> LLMPrompt:
+        prompt = LLMPrompt()
+        pass  
+
+    async def get_inner_function(self,func_name:str) -> AIFunction:
+        pass
+
+    async def exec_actions(self,actions:List[ActionItem]) -> bool:
+        pass
+
+class SelfThinkingProcess(BaseLLMProcess):
+    def __init__(self) -> None:
+        super().__init__()
+
+    async def load_from_config(self, config: dict) -> Coroutine[Any, Any, bool]:
+        if await super().load_from_config(config) is False:
+            return False
+
+    async def prepare_prompt(self) -> LLMPrompt:
+        prompt = LLMPrompt()
+        pass  
+
+    async def get_inner_function(self,func_name:str) -> AIFunction:
+        pass
+
+    async def exec_actions(self,actions:List[ActionItem]) -> bool:
+        pass
+    
+class LLMProcessLoader:
+    def __init__(self) -> None:
+        self.loaders : Dict[str,Callable[[dict],Awaitable[BaseLLMProcess]]] = {}
+        return
+    
+    @classmethod
+    def get_instance(cls)->"LLMProcessLoader":
+        if not hasattr(cls,"_instance"):
+            cls._instance = LLMProcessLoader()
+        return cls._instance
+    
+    def register_loader(self, typename:str,loader:Callable[[dict],Awaitable[BaseLLMProcess]]):
+        self.loaders[typename] = loader
+    
+    async def load_from_config(self,config:dict) -> BaseLLMProcess:
+        llm_type_name = config.get("type")
+        if llm_type_name:
+            loader = self.loaders.get(llm_type_name)
+            if loader:
+                return await loader(config)
+
+            selected_type = globals().get(llm_type_name)   
+            if selected_type:
+                result : BaseLLMProcess = selected_type()
+                load_result = await result.load_from_config(config)
+                if load_result is False:
+                    logger.warn(f"load LLMProcess {llm_type_name} from config failed! load_from_config return False")
+                    return None
+                else:
+                    return result
+
+
+        logger.warn(f"load LLMProcess {llm_type_name} from config failed! type not found")
+        return None
 
 
 

src/aios/environment/environment.py

@@ -101,4 +101,10 @@ class CompositeEnvironment(SimpleEnvironment):
             self.functions[func.get_name()] = func
         operations = env.get_all_ai_operations()
         for op in operations:
-            self.operations[op.get_name()] = op
+            self.operations[op.get_name()] = op
+
+    def get_value(self,key:str) -> Optional[str]:
+        for env in self.envs:
+            val = env.get_value(key)
+            if val is not None:
+                return val

src/aios/proto/agent_msg.py

@@ -75,6 +75,11 @@ class AgentMsg:
         self.inner_call_chain = []
         self.resp_msg = None
 
+        self.action_list = []
+
+        #context info
+        self.context_info:dict= {}
+
     @classmethod
     def from_json(cls,json_obj:dict) -> 'AgentMsg':
         msg = AgentMsg()

src/aios/proto/ai_function.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import Dict,Coroutine,Callable
+from typing import Dict,Coroutine,Callable,List
 
 class ParameterDefine:
     def __init__(self) -> None:
@@ -74,10 +74,11 @@ class AIFunction:
     #    pass
 
 class ActionItem:
-    def __init__(self,name,args) -> None:
-        self.name = name
-        self.args = args
-        self.body = None
+    def __init__(self,name:str,args:List[str]) -> None:
+        self.name:str= name
+        self.args:List[str]= args
+        self.body:str = None
+        self.parms : Dict = None
 
     def append_body(self,body:str) -> None:
         if self.body is None:
@@ -88,6 +89,17 @@ class ActionItem:
     def dumps(self) -> str:
         pass
 
+    @classmethod
+    def from_json(cls,json_obj:dict) -> 'ActionItem':
+        args = json_obj.get("args",[])
+        r = ActionItem(json_obj["name"],args)
+        if json_obj.get("body"):
+            r.body = json_obj["body"]
+        r.parms = json_obj
+
+        return r
+    
+
 # call chain is a combination of ai_function,group of ai_function.
 class CallChain:
     def __init__(self) -> None:

src/aios/proto/compute_task.py

@@ -6,7 +6,8 @@ import shlex
 import uuid
 import time
 from typing import List, Union
-from ..proto.ai_function import *
+from .ai_function import *
+from .agent_msg import *
 from ..knowledge import ObjectID
 from ..storage.storage import AIStorage
 
@@ -40,20 +41,63 @@ class ComputeTaskType(Enum):
     TEXT_EMBEDDING ="text_embedding"
     IMAGE_EMBEDDING ="image_embedding"
 
+# class Function(TypedDict, total=False):
+#     name: Required[str]
+#     """The name of the function to be called.
+
+#     Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length
+#     of 64.
+#     """
+
+#     parameters: Required[shared_params.FunctionParameters]
+#     """The parameters the functions accepts, described as a JSON Schema object.
+
+#     See the [guide](https://platform.openai.com/docs/guides/gpt/function-calling)
+#     for examples, and the
+#     [JSON Schema reference](https://json-schema.org/understanding-json-schema/) for
+#     documentation about the format.
+
+#     To describe a function that accepts no parameters, provide the value
+#     `{"type": "object", "properties": {}}`.
+#     """
+
+#     description: str
+#     """
+#     A description of what the function does, used by the model to choose when and
+#     how to call the function.
+#     """
+
 class LLMPrompt:
     def __init__(self,prompt_str = None) -> None:
-        self.messages = []
+        self.messages : List[Dict] = []
         if prompt_str:
             self.messages.append({"role":"user","content":prompt_str})
-        self.system_message = None
+        self.system_message : Dict = None
+        self.inner_functions : List[Dict] = []
+
+    def append_system_message(self,content:str):
+        if content is None:
+            return
+        
+        if self.system_message is None:
+            self.system_message = {"role":"system","content":content}
+        else:
+            self.system_message["content"] += content
+    
+    def append_user_message(self,content:str):
+        if content is None:
+            return
+        
+        self.messages.append({"role":"user","content":content})
 
     def as_str(self)->str:
         result_str = ""
         if self.system_message:
-            result_str += self.system_message.get("role") + ":" + self.system_message.get("content") + "\n"
+            result_str = json.dumps(self.system_message)
         if self.messages:
-            for msg in self.messages:
-                result_str += msg.get("role") + ":" + msg.get("content") + "\n"
+            result_str += json.dumps(self.messages)
+        if self.inner_functions:
+            result_str += json.dumps(self.inner_functions)
 
         return result_str
 
@@ -63,10 +107,18 @@ class LLMPrompt:
             result.append(self.system_message)
         result.extend(self.messages)
         return result
+    
+    
 
     def append(self,prompt:'LLMPrompt'):
         if prompt is None:
             return
+        
+        if prompt.inner_functions:
+            if self.inner_functions is None:
+                self.inner_functions = copy.deepcopy(prompt.inner_functions)
+            else:
+                self.inner_functions.extend(prompt.inner_functions)
 
         if prompt.system_message is not None:
             if self.system_message is None:
@@ -76,11 +128,11 @@ class LLMPrompt:
 
         self.messages.extend(prompt.messages)
 
-    def load_from_config(self,config:list) -> bool:
+    def load_from_config(self,config:List[Dict]) -> bool:
         if isinstance(config,list) is not True:
             logger.error("prompt is not list!")
             return False
-        self.messages = []
+        self.messages : List[Dict] = []
         for msg in config:
             if msg.get("content"):
                 if msg.get("role") == "system":
@@ -126,11 +178,16 @@ class LLMResult:
         if llm_json_str == "**IGNORE**":
             r.state = LLMResultStates.IGNORE
             return r
+        
+        r.state = LLMResultStates.OK
 
         llm_json = json.loads(llm_json_str)
         r.resp = llm_json.get("resp")
         r.raw_result = llm_json
-        r.action_list = llm_json.get("actions")
+        action_list = llm_json.get("actions")
+        for action in action_list:
+            action_item = ActionItem.from_json(action)
+            r.action_list.append(action_item)
 
         return r