Merge pull request #121 from wugren/MVP

Support multimodal input

rootfs/agents/Jarvis/agent.toml

@@ -16,6 +16,8 @@ Only clearly specifying the task you completed can be completed independently.
 type="AgentMessageProcess"
 # TODO: 是否应该自动记录 inner function和action的执行细节
 mutil_model="gpt-4-vision-preview"
+asr_model="openai-whisper"
+tts_model="tts-1"
 
 process_description="""
 1. Based on your role and the existing information, please think and then make a brief and efficient reply.
@@ -30,7 +32,7 @@ The Response must be directly parsed by `python json.loads`. Here is an example:
 {
     think:'$think step-by-step to be sure you have the right reply.'
     resp: '$What you want to reply',
-    actions: [{ 
+    actions: [{
     name: '$action_name',
     $param_name: '$parm' #Optional, fill in only if the action has parameters.
     }]
@@ -63,7 +65,7 @@ The Response must be directly parsed by `python json.loads`. Here is an example:
 {
     think:'$think step-by-step to be sure you can triage tasks well.'
     resp : '$determine, summary what you do',
-    actions: [{ 
+    actions: [{
     name: '$action_name',
     $param_name: '$parm' #Optional, fill in only if the action has parameters.
     }]
@@ -89,7 +91,7 @@ The Response must be directly parsed by `python json.loads`. Here is an example:
 {
     think:'$thinking step by step to ensure the accurate and efficient processing task.',
     resp:'$determine, summary what you do'
-    actions: [{ 
+    actions: [{
     name: '$action_name',
     $param_name: '$parm' #Optional, fill in only if the action has parameters.
     }]
@@ -114,7 +116,7 @@ The Response must be directly parsed by `python json.loads`. Here is an example:
 {
     think:'$think step-by-step to be sure you have the right result.',
     resp : '$determine, summary what you will do',
-    actions: [{ 
+    actions: [{
     name: '$action_name',
     $param_name: '$parm' #Optional, fill in only if the action has parameters.
     }]
@@ -124,11 +126,11 @@ The Response must be directly parsed by `python json.loads`. Here is an example:
 llm_context.actions.enable = ["agent.workspace.cancel_task","agent.workspace.update_task"]
 
 context="Your Principal now in {location}, time: {now}, weather: {weather}."
-[behavior.do] 
+[behavior.do]
 # do TODO
 type="AgentDo"
 process_description="""
-The input is a TODO comes from a Task. 
+The input is a TODO comes from a Task.
 1. Your task is to combine your role definition, tools on hand, known information, and complete a certain Todo.After completing the Todo, you will get a tip of $ 200.
 2. 8000 word limit for short term memory. Your short term memory is short, so immediately save important information to files.
 3. In the process of completing Todo, you should think first and then execute. During the execution, you can use functions to access the results of the front steps.
@@ -141,10 +143,10 @@ The Response must be directly parsed by `python json.loads`. Here is an example:
 {
     think:'$think step by step, how to complete the todo',
     resp: '$simport report about what you do',
-    actions: [{ 
+    actions: [{
     name: '$action1_name',
     $param_name: '$parm' #Optional, fill in only if the action has parameters.
-    }, ... 
+    }, ...
     ]
 }
 """
@@ -168,7 +170,7 @@ The Response must be directly parsed by `python json.loads`. Here is an example:
     resp:'$think step by step, how to check the todo',
     name: '$action1_name',
     $param_name: '$parm' #Optional, fill in only if the action has parameters.
-    }, ... 
+    }, ...
     ]
 }
 """
@@ -197,7 +199,7 @@ The Response must be directly parsed by `python json.loads`. Here is an example:
     resp:'$Summary in one sentence',
     name: '$action1_name',
     $param_name: '$parm' #Optional, fill in only if the action has parameters.
-    }, ... 
+    }, ...
     ]
 }
 """

src/aios/agent/agent.py

@@ -84,7 +84,7 @@ class AIAgent(BaseAIAgent):
         todo_prompts = {}
         todo_prompts[TodoListType.TO_WORK] = {
             "do": None,
-            "check": None, 
+            "check": None,
             "review": None,
         }
         todo_prompts[TodoListType.TO_LEARN] = {
@@ -103,12 +103,12 @@ class AIAgent(BaseAIAgent):
         self.prviate_workspace : AgentWorkspace = None
 
         self.behaviors:Dict[str,BaseLLMProcess] = {}
-        
+
     async def initial(self,params:Dict = None):
         self.base_dir = f"{AIStorage.get_instance().get_myai_dir()}/agent_data/{self.agent_id}"
         memory_base_dir = f"{self.base_dir}/memory"
         self.memory = AgentMemory(self.agent_id,memory_base_dir)
-        self.prviate_workspace = AgentWorkspace(self.agent_id) 
+        self.prviate_workspace = AgentWorkspace(self.agent_id)
         init_params = {}
         init_params["memory"] = self.memory
         init_params["workspace"] = self.prviate_workspace
@@ -117,7 +117,7 @@ class AIAgent(BaseAIAgent):
             if init_result is False:
                 logger.error(f"llm process {process_name} initial failed! initial return False")
                 return False
-        
+
         self.wake_up()
         return True
 
@@ -151,7 +151,7 @@ class AIAgent(BaseAIAgent):
             self.enable_timestamp = bool(config["enable_timestamp"])
         if config.get("history_len"):
             self.history_len = int(config.get("history_len"))
- 
+
         #load all LLMProcess
         self.behaviors = {}
         behaviors = config.get("behavior")
@@ -201,12 +201,12 @@ class AIAgent(BaseAIAgent):
         context_info["owner"] = AIStorage.get_instance().get_user_config().get_value("username")
 
         return context_info
-    
+
     async def llm_process_msg(self,msg:AgentMsg) -> AgentMsg:
         need_process:bool = True
         if msg.msg_type == AgentMsgType.TYPE_GROUPMSG:
             need_process = False
-           
+
             session_topic = msg.target + "#" + msg.topic
             chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.memory.memory_db)
             if msg.mentions is not None:
@@ -218,7 +218,7 @@ class AIAgent(BaseAIAgent):
                 chatsession.append(msg)
                 resp_msg = msg.create_group_resp_msg(self.agent_id,"")
                 return resp_msg
-        
+
         context_info = await self._get_context_info()
         input_parms = {
             "msg":msg,
@@ -232,8 +232,11 @@ class AIAgent(BaseAIAgent):
         elif llm_result.state == LLMResultStates.IGNORE:
             return None
         else: # OK
-            resp_msg = llm_result.raw_result.get("_resp_msg")
-            return resp_msg
+            if llm_result.raw_result is not None:
+                resp_msg = llm_result.raw_result.get("_resp_msg")
+                return resp_msg
+            else:
+                return msg.create_resp_msg(llm_result.resp)
 
     async def _process_msg(self,msg:AgentMsg,workspace = None) -> AgentMsg:
         return await self.llm_process_msg(msg)
@@ -264,7 +267,7 @@ class AIAgent(BaseAIAgent):
             else:
                 logger.info(f"llm process self thinking  ok!,think is:{llm_result.resp}")
                 self.memory.set_last_think_time(time.time())
-            self.agent_energy -= 2  
+            self.agent_energy -= 2
             return
 
     async def llm_triage_tasklist(self):
@@ -273,7 +276,7 @@ class AIAgent(BaseAIAgent):
             if self.prviate_workspace:
                 filter = {}
                 filter["state"] = AgentTaskState.TASK_STATE_WAIT
-            
+
                 tasklist:List[AgentTask]= await self.prviate_workspace.task_mgr.list_task(filter)
 
 
@@ -281,8 +284,8 @@ class AIAgent(BaseAIAgent):
                     if len(tasklist) > 0:
                         simple_list:List[Dict] = []
                         for task in tasklist:
-                            simple_list.append(task.to_simple_dict()) 
-                            
+                            simple_list.append(task.to_simple_dict())
+
                         input_parms = {
                             "tasklist":simple_list,
                             "context_info": await self._get_context_info()
@@ -294,7 +297,7 @@ class AIAgent(BaseAIAgent):
                             logger.info(f"llm process triage_tasks ignore!")
                         else:
                             logger.info(f"llm process triage_tasks ok!,think is:{llm_result.resp}")
-                        self.agent_energy -= 3  
+                        self.agent_energy -= 3
 
                     # for agent_task in tasklist:
                     #     if self.agent_energy <= 0:
@@ -314,7 +317,7 @@ class AIAgent(BaseAIAgent):
                     #         else:
                     #             determine = llm_result.raw_result.get("determine")
                     #             logger.info(f"llm process review_task ok!,think is:{determine}")
-                    #         self.agent_energy -= 1  
+                    #         self.agent_energy -= 1
 
     async def llm_do_todo(self, todo: AgentTodo):
         llm_process : BaseLLMProcess = self.behaviors.get("do")
@@ -350,7 +353,7 @@ class AIAgent(BaseAIAgent):
                 logger.info(f"llm process check_todo ok!,think is:{llm_result.resp}")
             self.agent_energy -= 1
 
-            return 
+            return
 
     async def llm_plan_task(self,task:AgentTask):
         llm_process : BaseLLMProcess = self.behaviors.get("plan_task")
@@ -398,7 +401,7 @@ class AIAgent(BaseAIAgent):
 
     async def _on_timer(self):
         await asyncio.sleep(5)
-        while True:   
+        while True:
             try:
                 now = time.time()
                 if self.last_recover_time is None:
@@ -419,7 +422,7 @@ class AIAgent(BaseAIAgent):
                 #filter["state"] = AgentTaskState.TASK_STATE_WAIT
                 filter = None
                 task_list:List[AgentTask] = await self.prviate_workspace.task_mgr.list_task(filter)
-                
+
                 for task in task_list:
                     if self.agent_energy <= 0:
                         break
@@ -456,18 +459,18 @@ class AIAgent(BaseAIAgent):
                     task = await self.prviate_workspace.task_mgr.get_task(task.task_id)
                     if task.state == AgentTaskState.TASK_STATE_WAITING_REVIEW:
                         await self.llm_review_task(task)
-                    
+
                 await self._self_imporve()
-                
-               
-                
+
+
+
             except Exception as e:
                 tb_str = traceback.format_exc()
                 logger.error(f"agent {self.agent_id} on timer error:{e},{tb_str}")
-            
+
             # Because the LLM itself is very slow, the accuracy of the system processing task is in minutes.
-            await asyncio.sleep(30) 
-                
+            await asyncio.sleep(30)
+
 
 
 

src/aios/agent/llm_process.py

@@ -1,5 +1,8 @@
 # Old name is behavior, I belive new name "llm_process" is better
 # pylint:disable=E0402
+import os.path
+
+from .chatsession import AIChatSession
 from ..utils import video_utils,image_utils
 
 from ..proto.compute_task import LLMPrompt,LLMResult,ComputeTaskResult,ComputeTaskResultCode
@@ -31,11 +34,11 @@ class BaseLLMProcess(ABC):
         self.goal:str = None #目标
         self.input_example:str= None #输入样例
         self.result_example:str = None #llm_result样例
-        
+
         self.enable_json_resp = False
         #None means system default,
         # TODO: support abcstract model name like: local-hight,local-low,local-medium,remote-hight,remote-low,remote-medium
-        self.model_name = None 
+        self.model_name = None
         self.max_token = 1000 # result_token
         self.max_prompt_token = 1000 # not include input prompt
         self.timeout = 1800 # 30 min
@@ -55,8 +58,8 @@ class BaseLLMProcess(ABC):
 
     @abstractmethod
     def prepare_inner_function_context_for_exec(self,inner_func_name:str,parameters:Dict):
-        return 
-    
+        return
+
     @abstractmethod
     async def post_llm_process(self,actions:List[ActionNode],input:Dict,llm_result:LLMResult) -> bool:
         pass
@@ -76,14 +79,14 @@ class BaseLLMProcess(ABC):
             self.max_token = config.get("max_token")
         if config.get("timeout"):
             self.timeout = config.get("timeout")
-        
+
 
         return True
-    
+
     @abstractmethod
     async def initial(self,params:Dict = None) -> bool:
         pass
-    
+
     def _format_content_by_env_value(self,content:str,env)->str:
         return content.format_map(env)
 
@@ -120,12 +123,12 @@ class BaseLLMProcess(ABC):
             task_result.result_code = ComputeTaskResultCode.ERROR
             task_result.error_str = f"prompt too long,can not predict"
             return task_result
-        
+
         if stack_limit > 0:
             inner_functions=prompt.inner_functions
         else:
             inner_functions = None
-       
+
 
         task_result: ComputeTaskResult = await (ComputeKernel.get_instance().do_llm_completion(
             prompt,
@@ -140,7 +143,7 @@ class BaseLLMProcess(ABC):
             return task_result
 
         inner_func_call_node = None
- 
+
         result_message : dict = task_result.result.get("message")
         if result_message:
             inner_func_call_node = result_message.get("function_call")
@@ -163,10 +166,10 @@ class BaseLLMProcess(ABC):
 
         # Action define in prompt, will be execute after llm compute
         prompt = await self.prepare_prompt(input)
-        max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.model_name)
+        max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.get_llm_model_name())
         #if max_result_token < MIN_PREDICT_TOKEN_LEN:
         #    return LLMResult.from_error_str(f"prompt too long,can not predict")
-        
+
         task_result: ComputeTaskResult = await (ComputeKernel.get_instance().do_llm_completion(
                 prompt,
                 resp_mode=resp_mode,
@@ -174,12 +177,12 @@ class BaseLLMProcess(ABC):
                 max_token=max_result_token,
                 inner_functions=prompt.inner_functions, #NOTICE: inner_function in prompt can be a subset of get_inner_function
                 timeout=self.timeout))
-        
+
         if task_result.result_code != ComputeTaskResultCode.OK:
             err_str = f"do_llm_completion error:{task_result.error_str}"
             logger.error(err_str)
             return LLMResult.from_error_str(err_str)
-        
+
         result_message = task_result.result.get("message")
         inner_func_call_node = None
         if result_message:
@@ -194,7 +197,11 @@ class BaseLLMProcess(ABC):
 
         # parse task_result to LLM Result
         if self.enable_json_resp:
-            llm_result = LLMResult.from_json_str(task_result.result_str)
+            try:
+                llm_result = LLMResult.from_json_str(task_result.result_str)
+            except Exception as e:
+                logger.error(f"parse llm result error:{e}")
+                llm_result = LLMResult.from_str(task_result.result_str)
         else:
             llm_result = LLMResult.from_str(task_result.result_str)
 
@@ -202,7 +209,7 @@ class BaseLLMProcess(ABC):
         await self.post_llm_process(llm_result.action_list,input,llm_result)
 
         return llm_result
-    
+
 class LLMAgentBaseProcess(BaseLLMProcess):
     def __init__(self) -> None:
         super().__init__()
@@ -211,11 +218,11 @@ class LLMAgentBaseProcess(BaseLLMProcess):
         self.process_description:str = None
         self.reply_format:str = None
         self.context : str = None
-        
+
         self.workspace : AgentWorkspace = None # If Workspace is not none , enable Agent Tasklist
         self.memory : AgentMemory = None
         self.enable_kb : bool = False
-        self.kb = None    
+        self.kb = None
 
     async def initial(self,params:Dict = None) -> bool:
         self.memory = params.get("memory")
@@ -227,23 +234,23 @@ class LLMAgentBaseProcess(BaseLLMProcess):
         return True
     async def load_default_config(self) -> bool:
         return True
-        
-        
+
+
     async def load_from_config(self, config: dict,is_load_default=True) -> Coroutine[Any, Any, bool]:
         if is_load_default:
             await self.load_default_config()
 
         if await super().load_from_config(config) is False:
             return False
-        
+
         self.role_description = config.get("role_desc")
         if self.role_description is None:
             logger.error(f"role_description not found in config")
             return False
-        
+
         if config.get("process_description"):
             self.process_description = config.get("process_description")
-        
+
         if config.get("reply_format"):
             self.reply_format = config.get("reply_format")
 
@@ -282,7 +289,7 @@ class LLMAgentBaseProcess(BaseLLMProcess):
         return system_prompt_dict
 
     def prepare_inner_function_context_for_exec(self,inner_func_name:str,parameters:Dict):
-        parameters["_workspace"] = self.workspace  
+        parameters["_workspace"] = self.workspace
 
     def get_action_desc(self) -> Dict:
         result = {}
@@ -290,17 +297,17 @@ class LLMAgentBaseProcess(BaseLLMProcess):
         for action in actions_list:
             result[action.get_name()] = action.get_description()
         return result
-    
+
     async def get_inner_function_for_exec(self,func_name:str) -> AIFunction:
         return self.llm_context.get_ai_function(func_name)
-    
+
     async def _execute_actions(self,actions:List[ActionNode],action_params:Dict):
         for action_item in actions:
             op : AIAction = self.llm_context.get_ai_action(action_item.name)
             if op:
                 if action_item.parms is None:
                     action_item.parms = {}
-                
+
                 real_parms = {**action_params,**action_item.parms}
 
                 action_item.parms["_result"] = await op.execute(real_parms)
@@ -309,17 +316,19 @@ class LLMAgentBaseProcess(BaseLLMProcess):
                 logger.warn(f"action {action_item.name} not found")
                 return False
 
-    
+
 class AgentMessageProcess(LLMAgentBaseProcess):
     def __init__(self) -> None:
         super().__init__()
         self.mutil_model = None
         self.enable_media2text = False
         self.is_mutil_model = False
+        self.asr_model = None
+        self.tts_model = None
 
     async def load_default_config(self) -> bool:
         return True
-        
+
     async def load_from_config(self, config: dict,is_load_default=True) -> Coroutine[Any, Any, bool]:
         if is_load_default:
             await self.load_default_config()
@@ -331,23 +340,26 @@ class AgentMessageProcess(LLMAgentBaseProcess):
 
         if config.get("mutil_model"):
             self.mutil_model = config.get("mutil_model")
-         
+
+        self.asr_model = config.get("asr_model")
+        self.tts_model = config.get("tts_model")
+
     def get_llm_model_name(self) -> str:
         if self.is_mutil_model:
             return self.mutil_model
         else:
             return self.model_name
-    
+
     def check_and_to_base64(self, image_path: str) -> str:
         if image_utils.is_file(image_path):
             return image_utils.to_base64(image_path, (1024, 1024))
         else:
             return image_path
-             
+
     async def get_prompt_from_msg(self,msg:AgentMsg) -> LLMPrompt:
         msg_prompt = LLMPrompt()
         self.is_mutil_model = False
-        if msg.is_image_msg():  
+        if msg.is_image_msg():
             if self.enable_media2text:
                 logger.error(f"enable_media2text is not supported yet")
             else:
@@ -358,35 +370,60 @@ class AgentMessageProcess(LLMAgentBaseProcess):
                     content = [{"type": "text", "text": image_prompt}]
                     content.extend([{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images])
                     msg_prompt.messages = [{"role": "user", "content": content}]
-                
+
                 if self.mutil_model:
                     self.is_mutil_model = True
                 else:
                     logger.warning(f"mutil_model is not set!")
-                
+
         elif msg.is_video_msg():
-            video_prompt, video = msg.get_video_body()
-            frames = video_utils.extract_frames(video, (1024, 1024))
-            if video_prompt is None:
-                msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": frame}} for frame in frames]}]
+            if self.enable_media2text:
+                logger.error(f"enable_media2text is not supported yet")
             else:
-                content = [{"type": "text", "text": video_prompt}]
+                video_prompt, video = msg.get_video_body()
+                frames = video_utils.extract_frames(video, (1024, 1024))
+                audio_file = os.path.splitext(video)[0] + ".mp3"
+                video_utils.extract_audio(video, audio_file)
+
+                voice_content = None
+                if self.asr_model is not None:
+                    resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, model=self.asr_model, prompt=None, response_format="text"))
+                    if resp.result_code == ComputeTaskResultCode.OK:
+                        voice_content = resp.result_str
+
+                content = []
+                if video_prompt is not None:
+                    content.append({"type": "text", "text": video_prompt})
+                if voice_content is not None and voice_content != "":
+                    content.append({"type": "text", "text": f"Voice content in video:{voice_content}"})
+
                 content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
                 msg_prompt.messages = [{"role": "user", "content": content}]
+                if self.mutil_model:
+                    self.is_mutil_model = True
+                else:
+                    logger.warning(f"mutil_model is not set!")
         elif msg.is_audio_msg():
-            audio_file = msg.body
-            resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text"))
-            if resp.result_code != ComputeTaskResultCode.OK:
-                error_resp = msg.create_error_resp(resp.error_str)
-                return error_resp
+            if self.enable_media2text:
+                logger.error(f"enable_media2text is not supported yet")
             else:
-                msg.body = resp.result_str
-                msg_prompt.messages = [{"role":"user","content":resp.result_str}]
+                prompt, audio_file = msg.get_audio_body()
+                resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, model=self.asr_model, prompt=None, response_format="text"))
+                if resp.result_code != ComputeTaskResultCode.OK:
+                    error_resp = msg.create_error_resp(resp.error_str)
+                    return error_resp
+                else:
+                    if prompt == "":
+                        msg.body = resp.result_str
+                        msg_prompt.messages = [{"role":"user","content":resp.result_str}]
+                    else:
+                        msg.body = f"{prompt}\nVoice content:{resp.result_str}"
+                        msg_prompt.messages = [{"role":"user","content": prompt}, {"role": "user", "content": f"Voice content:{resp.result_str}"}]
         else:
             msg_prompt.messages = [{"role":"user","content":msg.body}]
 
         return msg_prompt
-    
+
     async def sender_info(self,msg:AgentMsg)->str:
         sender_id = msg.sender
         #TODO Is sender an agent?
@@ -400,14 +437,14 @@ class AgentMessageProcess(LLMAgentBaseProcess):
 
     async def get_log_summary(self,msg:AgentMsg)->str:
         return None
-        
+
 
     async def get_extend_known_info(self,msg:AgentMsg,prompt:LLMPrompt)->str:
         return None
 
     async def prepare_prompt(self,input:Dict) -> LLMPrompt:
         prompt = LLMPrompt()
-        # User Prompt 
+        # User Prompt
         ## Input Msg
         msg : AgentMsg = input.get("msg")
         context_info = input.get("context_info")
@@ -422,8 +459,8 @@ class AgentMessageProcess(LLMAgentBaseProcess):
 
         ## 通用的角色相关的系统提示词
         system_prompt_dict = self.prepare_role_system_prompt(context_info)
-               
-        ## 已知信息  
+
+        ## 已知信息
         known_info = {}
         #prompt.append_system_message(self.known_info_tips)
         ### 信息发送者资料
@@ -442,23 +479,23 @@ class AgentMessageProcess(LLMAgentBaseProcess):
                 known_info["summary"] = summary
         #prompt.append_system_message(await self.get_log_summary(self,msg))
         system_prompt_dict["known_info"] = known_info
-        
+
         prompt.inner_functions =LLMProcessContext.aifunctions_to_inner_functions(self.llm_context.get_all_ai_functions())
         if self.workspace:
             #TODO eanble workspace functions?
             logger.info(f"workspace is not none,enable workspace functions")
 
-        ## 给予查询KB的权限    
-        if self.enable_kb:        
+        ## 给予查询KB的权限
+        if self.enable_kb:
             logger.info(f"enable kb")
-           
+
 
         prompt.append_system_message(json.dumps(system_prompt_dict,ensure_ascii=False))
         ## 扩展已知信息 (这可能是一个LLM过程)
         prompt.append_system_message(await self.get_extend_known_info(msg,prompt))
 
         return prompt
-    
+
 
     async def post_llm_process(self,actions:List[ActionNode],input:Dict,llm_result:LLMResult) -> bool:
         msg:AgentMsg = input.get("msg")
@@ -466,14 +503,15 @@ class AgentMessageProcess(LLMAgentBaseProcess):
             resp_msg = msg.create_group_resp_msg(self.memory.agent_id,llm_result.resp)
         else:
             resp_msg = msg.create_resp_msg(llm_result.resp)
-        
-        llm_result.raw_result["_resp_msg"] = resp_msg
+
+        if llm_result.raw_result is not None:
+            llm_result.raw_result["_resp_msg"] = resp_msg
 
         action_params = {}
         action_params["_input"] = input
         action_params["_memory"] = self.memory
         action_params["_workspace"] = self.workspace
-        action_params["_resp_msg"] = resp_msg  
+        action_params["_resp_msg"] = resp_msg
         action_params["_llm_result"] = llm_result
         action_params["_agentid"] = self.memory.agent_id
         action_params["_start_at"] = datetime.now()
@@ -482,7 +520,7 @@ class AgentMessageProcess(LLMAgentBaseProcess):
 
         chatsession = self.memory.get_session_from_msg(msg)
         chatsession.append(msg)
-        chatsession.append(resp_msg)  
+        chatsession.append(resp_msg)
 
         return True
 
@@ -567,11 +605,11 @@ class AgentSelfThinking(LLMAgentBaseProcess):
 
         record_list = input.get("record_list")
         context_info = input.get("context_info")
-        
+
         if record_list is None:
             logger.error(f"AgentSelfThinking prepare_prompt failed! input  not found")
             return None
-        
+
         prompt.append_user_message(json.dumps(record_list,ensure_ascii=False))
         system_prompt_dict = self.prepare_role_system_prompt(context_info)
 
@@ -594,7 +632,7 @@ class AgentSelfThinking(LLMAgentBaseProcess):
         if known_experience_list:
             known_info["known_experience_list"] = known_experience_list
             have_known_info = True
-        
+
         if have_known_info:
             system_prompt_dict["known_info"] = known_info
 
@@ -626,7 +664,7 @@ class AgentSelfLearning(BaseLLMProcess):
 
     async def prepare_prompt(self) -> LLMPrompt:
         prompt = LLMPrompt()
-        pass  
+        pass
 
     async def get_inner_function_for_exec(self,func_name:str) -> AIFunction:
         pass
@@ -636,7 +674,7 @@ class AgentSelfLearning(BaseLLMProcess):
 
 class AgentSelfImprove(BaseLLMProcess):
     def __init__(self) -> None:
-        super().__init__()    
+        super().__init__()
 
 
 

src/aios/proto/compute_task.py

@@ -80,16 +80,16 @@ class LLMPrompt:
     def append_system_message(self,content:str):
         if content is None:
             return
-        
+
         if self.system_message is None:
             self.system_message = {"role":"system","content":content}
         else:
             self.system_message["content"] += content
-    
+
     def append_user_message(self,content:str):
         if content is None:
             return
-        
+
         self.messages.append({"role":"user","content":content})
 
     def as_str(self)->str:
@@ -109,13 +109,13 @@ class LLMPrompt:
             result.append(self.system_message)
         result.extend(self.messages)
         return result
-    
-    
+
+
 
     def append(self,prompt:'LLMPrompt'):
         if prompt is None:
             return
-        
+
         if prompt.inner_functions:
             if self.inner_functions is None:
                 self.inner_functions = copy.deepcopy(prompt.inner_functions)
@@ -164,8 +164,8 @@ class LLMResult:
     @classmethod
     def from_error_str(self,error_str:str) -> 'LLMResult':
         r = LLMResult()
-        r.state = "error"
-        r.compute_error_str = error_str
+        r.state = LLMResultStates.ERROR
+        r.error_str = error_str
         return r
 
     @classmethod
@@ -177,7 +177,7 @@ class LLMResult:
         if llm_json_str == "**IGNORE**":
             r.state = LLMResultStates.IGNORE
             return r
-        
+
         r.state = LLMResultStates.OK
 
         llm_json = json.loads(llm_json_str)
@@ -198,7 +198,7 @@ class LLMResult:
         func_name = str_list[0]
         params = str_list[1:]
         return func_name, params
-    
+
     @classmethod
     def from_str(self,llm_result_str:str,valid_func:List[str]=None) -> 'LLMResult':
         r = LLMResult()
@@ -210,8 +210,14 @@ class LLMResult:
             r.state = LLMResultStates.IGNORE
             return r
 
-        if llm_result_str[0] == "{":
-            return LLMResult.from_json_str(llm_result_str)
+        try:
+            if llm_result_str[0] == "{":
+                return LLMResult.from_json_str(llm_result_str)
+
+            if llm_result_str.lstrip().rstrip().startswith("```json"):
+                return LLMResult.from_json_str(llm_result_str[7:-3])
+        except:
+            pass
 
         lines = llm_result_str.splitlines()
         is_need_wait = False
@@ -226,10 +232,10 @@ class LLMResult:
                     target_id = action_item.args[0]
                     msg_content = action_item.body
                     new_msg.set("",target_id,msg_content)
-                    
+
                     return True
 
-                    
+
             return False
 
 
@@ -255,6 +261,8 @@ class LLMResult:
                 r.resp += current_action.dumps()
             else:
                 r.action_list.append(current_action)
+
+        r.state = LLMResultStates.OK
         return r
 
 class ComputeTask:

src/aios/utils/video_utils.py

@@ -3,6 +3,7 @@ from typing import List, Tuple
 
 import cv2
 import numpy as np
+import moviepy.editor as mp
 
 
 def precess_image(image):
@@ -120,3 +121,8 @@ def extract_frames(video_path: str, resize: Tuple[int, int] = None, smooth=False
         i += 1
     vidcap.release()
     return frames
+
+
+def extract_audio(video_path: str, audio_path: str):
+    my_clip = mp.VideoFileClip(video_path)
+    my_clip.audio.write_audiofile(audio_path)

src/component/common_environment/local_document.py

@@ -13,6 +13,7 @@ import PyPDF2
 import datetime
 from typing import Optional, List
 from aios import *
+from aios.environment.workspace_env import TodoListEnvironment, TodoListType
 from .local_file_system import FilesystemEnvironment
 
 logger = logging.getLogger(__name__)
@@ -21,7 +22,7 @@ class MetaDatabase:
     def __init__(self,db_path:str):
         self.db_path = db_path
         self._get_conn()
-        
+
     def _get_conn(self):
         """ get db connection """
         local = threading.local()
@@ -43,7 +44,7 @@ class MetaDatabase:
             self._create_tables(conn)
 
         return conn
-    
+
     def _create_tables(self,conn):
         cursor = conn.cursor()
         cursor.execute('''
@@ -68,7 +69,7 @@ class MetaDatabase:
                 create_time TEXT
             )
         ''')
-            
+
         cursor.execute('''
             CREATE INDEX IF NOT EXISTS idx_documents_doc_hash
             ON documents (doc_hash)
@@ -110,7 +111,7 @@ class MetaDatabase:
             WHERE doc_path = ?
         ''', (doc_hash, doc_path))
         conn.commit()
-    
+
     def get_docs_without_hash(self,limit:int=1024) -> List[str]:
         conn = self._get_conn()
         cursor = conn.cursor()
@@ -186,7 +187,7 @@ class MetaDatabase:
         row = cursor.fetchone()
         if row is None:
             return None
-        
+
         # get doc path
         cursor.execute('''
             SELECT doc_path
@@ -197,7 +198,7 @@ class MetaDatabase:
         if row2 is None:
             return None
         doc_path = row2[0]
-    
+
 
         return {
             "full_path": doc_path,
@@ -261,7 +262,7 @@ class LearningCache:
     def remove(self, key):
         with self.cache_lock:
             return self.cache.pop(key, None)
-    
+
 
 class LocalKnowledgeBase(CompositeEnvironment):
     def __init__(self, workspace: str) -> None:
@@ -275,10 +276,10 @@ class LocalKnowledgeBase(CompositeEnvironment):
         async def learn(op:dict):
             full_path = op.get("original_path")
             if not full_path:
-                return 
+                return
             meta = self.learning_cache.get(full_path)
             meta.update(op)
-            
+
         self.add_ai_operation(SimpleAIAction(
             op="learn",
             description="update knowledge llm summary",
@@ -287,16 +288,16 @@ class LocalKnowledgeBase(CompositeEnvironment):
 
         self.fs = FilesystemEnvironment(self.root_path)
         self.add_env(self.fs)
-    
+
     async def get_knowledege_catalog(self,path:str=None,only_dir =True,max_depth:int=5)->str:
         if path:
             full_path = f"{self.root_path}/{path}"
         else:
             full_path = self.root_path
-        
+
         catlogs,file_count = await self.get_directory_structure(full_path,max_depth,only_dir)
         return catlogs
- 
+
     async def get_directory_structure(self,root_dir, max_depth:int=4, only_dir=True, indent=1):
         file_count = 0
         structure_str = ''
@@ -315,11 +316,11 @@ class LocalKnowledgeBase(CompositeEnvironment):
 
                 if only_dir is False:
                     for file_name in sub_files:
-                        structure_str = structure_str + '  ' * (indent+1) + file_name + '\n' 
+                        structure_str = structure_str + '  ' * (indent+1) + file_name + '\n'
 
             dir_name = os.path.basename(root_dir)
             dir_info = f"{dir_name} <count: {file_count}>"
-        
+
 
             structure_str = '  ' * indent + dir_info + '\n' + structure_str
 
@@ -328,7 +329,7 @@ class LocalKnowledgeBase(CompositeEnvironment):
         else:
             return structure_str, file_count
 
-    # inner_function    
+    # inner_function
     async def get_knowledge_meta(self,path:str) -> str:
         full_path = f"{self.root_path}/{path}"
         if os.islink(full_path):
@@ -336,9 +337,9 @@ class LocalKnowledgeBase(CompositeEnvironment):
             hash = self.meta_db.get_hash_by_doc_path(org_path)
             if hash:
                 return self.meta_db.get_knowledge(org_path)
-        
+
         return "not found"
-    
+
     async def load_knowledge_content(self,path:str,pos:int=0,length:int=None) -> str:
         if path.endswith("pdf"):
             logger.info("load_knowledge_content:pdf")
@@ -367,12 +368,12 @@ class ScanLocalDocument:
         workspace = string.Template(config["workspace"]).substitute(myai_dir=AIStorage.get_instance().get_myai_dir())
         path = string.Template(config["path"]).substitute(myai_dir=AIStorage.get_instance().get_myai_dir())
         self.knowledge_base = LocalKnowledgeBase(workspace)
-        self.path = path  
+        self.path = path
 
     def _support_file(self,file_name:str) -> bool:
         if file_name.startswith("."):
             return False
-        
+
         if file_name.endswith(".pdf"):
             return True
         if file_name.endswith(".md"):
@@ -380,7 +381,7 @@ class ScanLocalDocument:
         if file_name.endswith(".txt"):
             return True
         return False
-        
+
     async def next(self):
         while True:
             for root, dirs, files in os.walk(self.path):
@@ -391,10 +392,10 @@ class ScanLocalDocument:
                         if self.knowledge_base.meta_db.is_doc_exist(full_path):
                             continue
                         yield(full_path, full_path)
-                    else: 
+                    else:
                         continue
             yield(None, None)
-                        
+
 
 
 class ParseLocalDocument:
@@ -425,7 +426,7 @@ class ParseLocalDocument:
                 await self.knowledge_base.fs.symlink(full_path, new_path)
                 logger.info(f"create soft link {full_path} -> {new_path}")
         return full_path
-        
+
     async def _get_meta_prompt(self,meta: dict,temp_meta = None,need_catalogs = False) -> str:
         kb_tree = await self.knowledge_base.get_knowledege_catalog()
 
@@ -473,7 +474,7 @@ class ParseLocalDocument:
         full_content_len = self._token_len(full_content)
         full_path = meta["original_path"]
         self.knowledge_base.learning_cache.add(full_path, meta)
-       
+
 
         if full_content_len < self.token_limit:
             # 短文章不用总结catalog
@@ -521,7 +522,7 @@ class ParseLocalDocument:
                 if item.title:
                     new_item = {}
                     new_item["page"] = item.page.idnum
-                    new_item["title"] = item.title 
+                    new_item["title"] = item.title
                     my_childs = []
                     if item.childs:
                         if len(item.childs) > 0:
@@ -573,7 +574,7 @@ class ParseLocalDocument:
         return {}
 
     def _parse_md(self,doc_path:str):
-        metadata = {} 
+        metadata = {}
         cur_encode = "utf-8"
         with open(doc_path,'rb') as f:
             cur_encode = chardet.detect(f.read(1024))['encoding']
@@ -588,7 +589,7 @@ class ParseLocalDocument:
             toc = md.toc
             if toc:
                 metadata['catalogs'] = toc
-            
+
         return metadata
 
     def _parse_document(self,doc_path:str):
@@ -614,5 +615,4 @@ class ParseLocalDocument:
             meta_data["title"] = title
         logger.info("parse document %s!",doc_path)
         return hash_result, meta_data
-        
-    
+

src/component/openai_node/open_ai_node.py

@@ -206,7 +206,7 @@ class OpenAI_ComputeNode(ComputeNode):
                 if mode_name == "gpt-4-vision-preview":
                     response_format = NOT_GIVEN
                     llm_inner_functions = None
-                    if max_token_size > 4096:
+                    if max_token_size > 4096 or max_token_size < 50:
                         result_token = 4096
                     else:
                         result_token = -1
@@ -216,14 +216,16 @@ class OpenAI_ComputeNode(ComputeNode):
                 client = AsyncOpenAI(api_key=self.openai_api_key)
                 try:
                     if llm_inner_functions is None or len(llm_inner_functions) == 0:
-                        logger.info(f"call openai {mode_name} prompts: {prompts}")
+                        if mode_name != "gpt-4-vision-preview":
+                            logger.info(f"call openai {mode_name} prompts: {prompts}")
                         resp = await client.chat.completions.create(model=mode_name,
                                                         messages=prompts,
                                                         response_format = response_format,
                                                         max_tokens=result_token,
                                                         )
                     else:
-                        logger.info(f"call openai {mode_name} prompts: \n\t {prompts} \nfunctions: \n\t{json.dumps(llm_inner_functions,ensure_ascii=False)}")
+                        if mode_name != "gpt-4-vision-preview":
+                            logger.info(f"call openai {mode_name} prompts: \n\t {prompts} \nfunctions: \n\t{json.dumps(llm_inner_functions,ensure_ascii=False)}")
                         resp = await client.chat.completions.create(model=mode_name,
                                                             messages=prompts,
                                                             response_format = response_format,
@@ -239,7 +241,7 @@ class OpenAI_ComputeNode(ComputeNode):
 
                 #logger.info(f"openai response: {resp}")
                 #TODO: gpt-4v api is image_2_text ?
-                if mode_name == "gpt-4-vision-preview": 
+                if mode_name == "gpt-4-vision-preview":
                     status_code = resp.choices[0].finish_reason
                     if status_code is None:
                         status_code = resp.choices[0].finish_details['type']
@@ -267,7 +269,7 @@ class OpenAI_ComputeNode(ComputeNode):
 
                 if token_usage:
                     result.result_refers["token_usage"] = token_usage
-                    
+
                 logger.info(f"openai success response: {result.result_str}")
                 return result
             case _:

src/component/slack_tunnel.py

@@ -119,6 +119,7 @@ class SlackTunnel(AgentTunnel):
                             continue
                         await download_file(file_info["file"]["url_private_download"], file_path, self.token)
 
+                        mime_type = file["mimetype"]
                         if file["mimetype"].startswith("image/"):
                             if file_type is None:
                                 file_type = "image"

src/requirements.txt

@@ -156,3 +156,4 @@ opencv-python
 discord.py
 slack_bolt
 wget
+moviepy