fix multimodal input bug

src/aios/agent/agent.py

@@ -84,7 +84,7 @@ class AIAgent(BaseAIAgent):
         todo_prompts = {}
         todo_prompts[TodoListType.TO_WORK] = {
             "do": None,
-            "check": None, 
+            "check": None,
             "review": None,
         }
         todo_prompts[TodoListType.TO_LEARN] = {
@@ -103,12 +103,12 @@ class AIAgent(BaseAIAgent):
         self.prviate_workspace : AgentWorkspace = None
 
         self.behaviors:Dict[str,BaseLLMProcess] = {}
-        
+
     async def initial(self,params:Dict = None):
         self.base_dir = f"{AIStorage.get_instance().get_myai_dir()}/agent_data/{self.agent_id}"
         memory_base_dir = f"{self.base_dir}/memory"
         self.memory = AgentMemory(self.agent_id,memory_base_dir)
-        self.prviate_workspace = AgentWorkspace(self.agent_id) 
+        self.prviate_workspace = AgentWorkspace(self.agent_id)
         init_params = {}
         init_params["memory"] = self.memory
         init_params["workspace"] = self.prviate_workspace
@@ -117,7 +117,7 @@ class AIAgent(BaseAIAgent):
             if init_result is False:
                 logger.error(f"llm process {process_name} initial failed! initial return False")
                 return False
-        
+
         self.wake_up()
         return True
 
@@ -151,7 +151,7 @@ class AIAgent(BaseAIAgent):
             self.enable_timestamp = bool(config["enable_timestamp"])
         if config.get("history_len"):
             self.history_len = int(config.get("history_len"))
- 
+
         #load all LLMProcess
         self.behaviors = {}
         behaviors = config.get("behavior")
@@ -201,12 +201,12 @@ class AIAgent(BaseAIAgent):
         context_info["owner"] = AIStorage.get_instance().get_user_config().get_value("username")
 
         return context_info
-    
+
     async def llm_process_msg(self,msg:AgentMsg) -> AgentMsg:
         need_process:bool = True
         if msg.msg_type == AgentMsgType.TYPE_GROUPMSG:
             need_process = False
-           
+
             session_topic = msg.target + "#" + msg.topic
             chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.memory.memory_db)
             if msg.mentions is not None:
@@ -218,7 +218,7 @@ class AIAgent(BaseAIAgent):
                 chatsession.append(msg)
                 resp_msg = msg.create_group_resp_msg(self.agent_id,"")
                 return resp_msg
-        
+
         context_info = await self._get_context_info()
         input_parms = {
             "msg":msg,
@@ -232,8 +232,11 @@ class AIAgent(BaseAIAgent):
         elif llm_result.state == LLMResultStates.IGNORE:
             return None
         else: # OK
-            resp_msg = llm_result.raw_result.get("_resp_msg")
+            if llm_result.raw_result is not None:
-            return resp_msg
+                resp_msg = llm_result.raw_result.get("_resp_msg")
+                return resp_msg
+            else:
+                return msg.create_resp_msg(llm_result.resp)
 
     async def _process_msg(self,msg:AgentMsg,workspace = None) -> AgentMsg:
         return await self.llm_process_msg(msg)
@@ -264,7 +267,7 @@ class AIAgent(BaseAIAgent):
             else:
                 logger.info(f"llm process self thinking  ok!,think is:{llm_result.resp}")
                 self.memory.set_last_think_time(time.time())
-            self.agent_energy -= 2  
+            self.agent_energy -= 2
             return
 
     async def llm_triage_tasklist(self):
@@ -273,7 +276,7 @@ class AIAgent(BaseAIAgent):
             if self.prviate_workspace:
                 filter = {}
                 filter["state"] = AgentTaskState.TASK_STATE_WAIT
-            
+
                 tasklist:List[AgentTask]= await self.prviate_workspace.task_mgr.list_task(filter)
 
 
@@ -281,8 +284,8 @@ class AIAgent(BaseAIAgent):
                     if len(tasklist) > 0:
                         simple_list:List[Dict] = []
                         for task in tasklist:
-                            simple_list.append(task.to_simple_dict()) 
+                            simple_list.append(task.to_simple_dict())
-                            
+
                         input_parms = {
                             "tasklist":simple_list,
                             "context_info": await self._get_context_info()
@@ -294,7 +297,7 @@ class AIAgent(BaseAIAgent):
                             logger.info(f"llm process triage_tasks ignore!")
                         else:
                             logger.info(f"llm process triage_tasks ok!,think is:{llm_result.resp}")
-                        self.agent_energy -= 3  
+                        self.agent_energy -= 3
 
                     # for agent_task in tasklist:
                     #     if self.agent_energy <= 0:
@@ -314,7 +317,7 @@ class AIAgent(BaseAIAgent):
                     #         else:
                     #             determine = llm_result.raw_result.get("determine")
                     #             logger.info(f"llm process review_task ok!,think is:{determine}")
-                    #         self.agent_energy -= 1  
+                    #         self.agent_energy -= 1
 
     async def llm_do_todo(self, todo: AgentTodo):
         llm_process : BaseLLMProcess = self.behaviors.get("do")
@@ -350,7 +353,7 @@ class AIAgent(BaseAIAgent):
                 logger.info(f"llm process check_todo ok!,think is:{llm_result.resp}")
             self.agent_energy -= 1
 
-            return 
+            return
 
     async def llm_plan_task(self,task:AgentTask):
         llm_process : BaseLLMProcess = self.behaviors.get("plan_task")
@@ -398,7 +401,7 @@ class AIAgent(BaseAIAgent):
 
     async def _on_timer(self):
         await asyncio.sleep(5)
-        while True:   
+        while True:
             try:
                 now = time.time()
                 if self.last_recover_time is None:
@@ -419,7 +422,7 @@ class AIAgent(BaseAIAgent):
                 #filter["state"] = AgentTaskState.TASK_STATE_WAIT
                 filter = None
                 task_list:List[AgentTask] = await self.prviate_workspace.task_mgr.list_task(filter)
-                
+
                 for task in task_list:
                     if self.agent_energy <= 0:
                         break
@@ -456,18 +459,18 @@ class AIAgent(BaseAIAgent):
                     task = await self.prviate_workspace.task_mgr.get_task(task.task_id)
                     if task.state == AgentTaskState.TASK_STATE_WAITING_REVIEW:
                         await self.llm_review_task(task)
-                    
+
                 await self._self_imporve()
-                
+
-               
+
-                
+
             except Exception as e:
                 tb_str = traceback.format_exc()
                 logger.error(f"agent {self.agent_id} on timer error:{e},{tb_str}")
-            
+
             # Because the LLM itself is very slow, the accuracy of the system processing task is in minutes.
-            await asyncio.sleep(30) 
+            await asyncio.sleep(30)
-                
+
 
 
 

src/aios/agent/llm_process.py

@@ -2,6 +2,7 @@
 # pylint:disable=E0402
 import os.path
 
+from .chatsession import AIChatSession
 from ..utils import video_utils,image_utils
 
 from ..proto.compute_task import LLMPrompt,LLMResult,ComputeTaskResult,ComputeTaskResultCode
@@ -165,7 +166,7 @@ class BaseLLMProcess(ABC):
 
         # Action define in prompt, will be execute after llm compute
         prompt = await self.prepare_prompt(input)
-        max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.model_name)
+        max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.get_llm_model_name())
         #if max_result_token < MIN_PREDICT_TOKEN_LEN:
         #    return LLMResult.from_error_str(f"prompt too long,can not predict")
 
@@ -196,7 +197,11 @@ class BaseLLMProcess(ABC):
 
         # parse task_result to LLM Result
         if self.enable_json_resp:
-            llm_result = LLMResult.from_json_str(task_result.result_str)
+            try:
+                llm_result = LLMResult.from_json_str(task_result.result_str)
+            except Exception as e:
+                logger.error(f"parse llm result error:{e}")
+                llm_result = LLMResult.from_str(task_result.result_str)
         else:
             llm_result = LLMResult.from_str(task_result.result_str)
 
@@ -402,14 +407,18 @@ class AgentMessageProcess(LLMAgentBaseProcess):
             if self.enable_media2text:
                 logger.error(f"enable_media2text is not supported yet")
             else:
-                audio_file = msg.body
+                prompt, audio_file = msg.get_audio_body()
                 resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, model=self.asr_model, prompt=None, response_format="text"))
                 if resp.result_code != ComputeTaskResultCode.OK:
                     error_resp = msg.create_error_resp(resp.error_str)
                     return error_resp
                 else:
-                    msg.body = resp.result_str
+                    if prompt == "":
-                    msg_prompt.messages = [{"role":"user","content":resp.result_str}]
+                        msg.body = resp.result_str
+                        msg_prompt.messages = [{"role":"user","content":resp.result_str}]
+                    else:
+                        msg.body = f"{prompt}\nVoice content:{resp.result_str}"
+                        msg_prompt.messages = [{"role":"user","content": prompt}, {"role": "user", "content": f"Voice content:{resp.result_str}"}]
         else:
             msg_prompt.messages = [{"role":"user","content":msg.body}]
 
@@ -495,7 +504,8 @@ class AgentMessageProcess(LLMAgentBaseProcess):
         else:
             resp_msg = msg.create_resp_msg(llm_result.resp)
 
-        llm_result.raw_result["_resp_msg"] = resp_msg
+        if llm_result.raw_result is not None:
+            llm_result.raw_result["_resp_msg"] = resp_msg
 
         action_params = {}
         action_params["_input"] = input

src/aios/proto/compute_task.py

@@ -210,8 +210,14 @@ class LLMResult:
             r.state = LLMResultStates.IGNORE
             return r
 
-        if llm_result_str[0] == "{":
+        try:
-            return LLMResult.from_json_str(llm_result_str)
+            if llm_result_str[0] == "{":
+                return LLMResult.from_json_str(llm_result_str)
+
+            if llm_result_str.lstrip().rstrip().startswith("```json"):
+                return LLMResult.from_json_str(llm_result_str[7:-3])
+        except:
+            pass
 
         lines = llm_result_str.splitlines()
         is_need_wait = False
@@ -255,6 +261,8 @@ class LLMResult:
                 r.resp += current_action.dumps()
             else:
                 r.action_list.append(current_action)
+
+        r.state = LLMResultStates.OK
         return r
 
 class ComputeTask:

src/component/openai_node/open_ai_node.py

@@ -206,7 +206,7 @@ class OpenAI_ComputeNode(ComputeNode):
                 if mode_name == "gpt-4-vision-preview":
                     response_format = NOT_GIVEN
                     llm_inner_functions = None
-                    if max_token_size > 4096:
+                    if max_token_size > 4096 or max_token_size < 50:
                         result_token = 4096
                     else:
                         result_token = -1