fix multimodal input bug

src/aios/agent/agent.py

@@ -232,8 +232,11 @@ class AIAgent(BaseAIAgent):
         elif llm_result.state == LLMResultStates.IGNORE:
             return None
         else: # OK
+            if llm_result.raw_result is not None:
                 resp_msg = llm_result.raw_result.get("_resp_msg")
                 return resp_msg
+            else:
+                return msg.create_resp_msg(llm_result.resp)
 
     async def _process_msg(self,msg:AgentMsg,workspace = None) -> AgentMsg:
         return await self.llm_process_msg(msg)

src/aios/agent/llm_process.py

@@ -2,6 +2,7 @@
 # pylint:disable=E0402
 import os.path
 
+from .chatsession import AIChatSession
 from ..utils import video_utils,image_utils
 
 from ..proto.compute_task import LLMPrompt,LLMResult,ComputeTaskResult,ComputeTaskResultCode
@@ -165,7 +166,7 @@ class BaseLLMProcess(ABC):
 
         # Action define in prompt, will be execute after llm compute
         prompt = await self.prepare_prompt(input)
-        max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.model_name)
+        max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.get_llm_model_name())
         #if max_result_token < MIN_PREDICT_TOKEN_LEN:
         #    return LLMResult.from_error_str(f"prompt too long,can not predict")
 
@@ -196,7 +197,11 @@ class BaseLLMProcess(ABC):
 
         # parse task_result to LLM Result
         if self.enable_json_resp:
+            try:
                 llm_result = LLMResult.from_json_str(task_result.result_str)
+            except Exception as e:
+                logger.error(f"parse llm result error:{e}")
+                llm_result = LLMResult.from_str(task_result.result_str)
         else:
             llm_result = LLMResult.from_str(task_result.result_str)
 
@@ -402,14 +407,18 @@ class AgentMessageProcess(LLMAgentBaseProcess):
             if self.enable_media2text:
                 logger.error(f"enable_media2text is not supported yet")
             else:
-                audio_file = msg.body
+                prompt, audio_file = msg.get_audio_body()
                 resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, model=self.asr_model, prompt=None, response_format="text"))
                 if resp.result_code != ComputeTaskResultCode.OK:
                     error_resp = msg.create_error_resp(resp.error_str)
                     return error_resp
                 else:
+                    if prompt == "":
                         msg.body = resp.result_str
                         msg_prompt.messages = [{"role":"user","content":resp.result_str}]
+                    else:
+                        msg.body = f"{prompt}\nVoice content:{resp.result_str}"
+                        msg_prompt.messages = [{"role":"user","content": prompt}, {"role": "user", "content": f"Voice content:{resp.result_str}"}]
         else:
             msg_prompt.messages = [{"role":"user","content":msg.body}]
 
@@ -495,6 +504,7 @@ class AgentMessageProcess(LLMAgentBaseProcess):
         else:
             resp_msg = msg.create_resp_msg(llm_result.resp)
 
+        if llm_result.raw_result is not None:
             llm_result.raw_result["_resp_msg"] = resp_msg
 
         action_params = {}

src/aios/proto/compute_task.py

@@ -210,9 +210,15 @@ class LLMResult:
             r.state = LLMResultStates.IGNORE
             return r
 
+        try:
             if llm_result_str[0] == "{":
                 return LLMResult.from_json_str(llm_result_str)
 
+            if llm_result_str.lstrip().rstrip().startswith("```json"):
+                return LLMResult.from_json_str(llm_result_str[7:-3])
+        except:
+            pass
+
         lines = llm_result_str.splitlines()
         is_need_wait = False
 
@@ -255,6 +261,8 @@ class LLMResult:
                 r.resp += current_action.dumps()
             else:
                 r.action_list.append(current_action)
+
+        r.state = LLMResultStates.OK
         return r
 
 class ComputeTask:

src/component/openai_node/open_ai_node.py

@@ -206,7 +206,7 @@ class OpenAI_ComputeNode(ComputeNode):
                 if mode_name == "gpt-4-vision-preview":
                     response_format = NOT_GIVEN
                     llm_inner_functions = None
-                    if max_token_size > 4096:
+                    if max_token_size > 4096 or max_token_size < 50:
                         result_token = 4096
                     else:
                         result_token = -1