Merge pull request #121 from wugren/MVP

Support multimodal input

rootfs/agents/Jarvis/agent.toml

@@ -16,6 +16,8 @@ Only clearly specifying the task you completed can be completed independently.
 type="AgentMessageProcess"
 # TODO: 是否应该自动记录 inner function和action的执行细节
 mutil_model="gpt-4-vision-preview"
+asr_model="openai-whisper"
+tts_model="tts-1"
 
 process_description="""
 1. Based on your role and the existing information, please think and then make a brief and efficient reply.

src/aios/agent/agent.py

@@ -232,8 +232,11 @@ class AIAgent(BaseAIAgent):
         elif llm_result.state == LLMResultStates.IGNORE:
             return None
         else: # OK
-            resp_msg = llm_result.raw_result.get("_resp_msg")
+            if llm_result.raw_result is not None:
-            return resp_msg
+                resp_msg = llm_result.raw_result.get("_resp_msg")
+                return resp_msg
+            else:
+                return msg.create_resp_msg(llm_result.resp)
 
     async def _process_msg(self,msg:AgentMsg,workspace = None) -> AgentMsg:
         return await self.llm_process_msg(msg)

src/aios/agent/llm_process.py

@@ -1,5 +1,8 @@
 # Old name is behavior, I belive new name "llm_process" is better
 # pylint:disable=E0402
+import os.path
+
+from .chatsession import AIChatSession
 from ..utils import video_utils,image_utils
 
 from ..proto.compute_task import LLMPrompt,LLMResult,ComputeTaskResult,ComputeTaskResultCode
@@ -163,7 +166,7 @@ class BaseLLMProcess(ABC):
 
         # Action define in prompt, will be execute after llm compute
         prompt = await self.prepare_prompt(input)
-        max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.model_name)
+        max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.get_llm_model_name())
         #if max_result_token < MIN_PREDICT_TOKEN_LEN:
         #    return LLMResult.from_error_str(f"prompt too long,can not predict")
 
@@ -194,7 +197,11 @@ class BaseLLMProcess(ABC):
 
         # parse task_result to LLM Result
         if self.enable_json_resp:
-            llm_result = LLMResult.from_json_str(task_result.result_str)
+            try:
+                llm_result = LLMResult.from_json_str(task_result.result_str)
+            except Exception as e:
+                logger.error(f"parse llm result error:{e}")
+                llm_result = LLMResult.from_str(task_result.result_str)
         else:
             llm_result = LLMResult.from_str(task_result.result_str)
 
@@ -316,6 +323,8 @@ class AgentMessageProcess(LLMAgentBaseProcess):
         self.mutil_model = None
         self.enable_media2text = False
         self.is_mutil_model = False
+        self.asr_model = None
+        self.tts_model = None
 
     async def load_default_config(self) -> bool:
         return True
@@ -332,6 +341,9 @@ class AgentMessageProcess(LLMAgentBaseProcess):
         if config.get("mutil_model"):
             self.mutil_model = config.get("mutil_model")
 
+        self.asr_model = config.get("asr_model")
+        self.tts_model = config.get("tts_model")
+
     def get_llm_model_name(self) -> str:
         if self.is_mutil_model:
             return self.mutil_model
@@ -365,23 +377,48 @@ class AgentMessageProcess(LLMAgentBaseProcess):
                     logger.warning(f"mutil_model is not set!")
 
         elif msg.is_video_msg():
-            video_prompt, video = msg.get_video_body()
+            if self.enable_media2text:
-            frames = video_utils.extract_frames(video, (1024, 1024))
+                logger.error(f"enable_media2text is not supported yet")
-            if video_prompt is None:
-                msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": frame}} for frame in frames]}]
             else:
-                content = [{"type": "text", "text": video_prompt}]
+                video_prompt, video = msg.get_video_body()
+                frames = video_utils.extract_frames(video, (1024, 1024))
+                audio_file = os.path.splitext(video)[0] + ".mp3"
+                video_utils.extract_audio(video, audio_file)
+
+                voice_content = None
+                if self.asr_model is not None:
+                    resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, model=self.asr_model, prompt=None, response_format="text"))
+                    if resp.result_code == ComputeTaskResultCode.OK:
+                        voice_content = resp.result_str
+
+                content = []
+                if video_prompt is not None:
+                    content.append({"type": "text", "text": video_prompt})
+                if voice_content is not None and voice_content != "":
+                    content.append({"type": "text", "text": f"Voice content in video:{voice_content}"})
+
                 content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
                 msg_prompt.messages = [{"role": "user", "content": content}]
+                if self.mutil_model:
+                    self.is_mutil_model = True
+                else:
+                    logger.warning(f"mutil_model is not set!")
         elif msg.is_audio_msg():
-            audio_file = msg.body
+            if self.enable_media2text:
-            resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text"))
+                logger.error(f"enable_media2text is not supported yet")
-            if resp.result_code != ComputeTaskResultCode.OK:
-                error_resp = msg.create_error_resp(resp.error_str)
-                return error_resp
             else:
-                msg.body = resp.result_str
+                prompt, audio_file = msg.get_audio_body()
-                msg_prompt.messages = [{"role":"user","content":resp.result_str}]
+                resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, model=self.asr_model, prompt=None, response_format="text"))
+                if resp.result_code != ComputeTaskResultCode.OK:
+                    error_resp = msg.create_error_resp(resp.error_str)
+                    return error_resp
+                else:
+                    if prompt == "":
+                        msg.body = resp.result_str
+                        msg_prompt.messages = [{"role":"user","content":resp.result_str}]
+                    else:
+                        msg.body = f"{prompt}\nVoice content:{resp.result_str}"
+                        msg_prompt.messages = [{"role":"user","content": prompt}, {"role": "user", "content": f"Voice content:{resp.result_str}"}]
         else:
             msg_prompt.messages = [{"role":"user","content":msg.body}]
 
@@ -467,7 +504,8 @@ class AgentMessageProcess(LLMAgentBaseProcess):
         else:
             resp_msg = msg.create_resp_msg(llm_result.resp)
 
-        llm_result.raw_result["_resp_msg"] = resp_msg
+        if llm_result.raw_result is not None:
+            llm_result.raw_result["_resp_msg"] = resp_msg
 
         action_params = {}
         action_params["_input"] = input

src/aios/proto/compute_task.py

@@ -164,8 +164,8 @@ class LLMResult:
     @classmethod
     def from_error_str(self,error_str:str) -> 'LLMResult':
         r = LLMResult()
-        r.state = "error"
+        r.state = LLMResultStates.ERROR
-        r.compute_error_str = error_str
+        r.error_str = error_str
         return r
 
     @classmethod
@@ -210,8 +210,14 @@ class LLMResult:
             r.state = LLMResultStates.IGNORE
             return r
 
-        if llm_result_str[0] == "{":
+        try:
-            return LLMResult.from_json_str(llm_result_str)
+            if llm_result_str[0] == "{":
+                return LLMResult.from_json_str(llm_result_str)
+
+            if llm_result_str.lstrip().rstrip().startswith("```json"):
+                return LLMResult.from_json_str(llm_result_str[7:-3])
+        except:
+            pass
 
         lines = llm_result_str.splitlines()
         is_need_wait = False
@@ -255,6 +261,8 @@ class LLMResult:
                 r.resp += current_action.dumps()
             else:
                 r.action_list.append(current_action)
+
+        r.state = LLMResultStates.OK
         return r
 
 class ComputeTask:

src/aios/utils/video_utils.py

@@ -3,6 +3,7 @@ from typing import List, Tuple
 
 import cv2
 import numpy as np
+import moviepy.editor as mp
 
 
 def precess_image(image):
@@ -120,3 +121,8 @@ def extract_frames(video_path: str, resize: Tuple[int, int] = None, smooth=False
         i += 1
     vidcap.release()
     return frames
+
+
+def extract_audio(video_path: str, audio_path: str):
+    my_clip = mp.VideoFileClip(video_path)
+    my_clip.audio.write_audiofile(audio_path)

src/component/common_environment/local_document.py

@@ -13,6 +13,7 @@ import PyPDF2
 import datetime
 from typing import Optional, List
 from aios import *
+from aios.environment.workspace_env import TodoListEnvironment, TodoListType
 from .local_file_system import FilesystemEnvironment
 
 logger = logging.getLogger(__name__)
@@ -615,4 +616,3 @@ class ParseLocalDocument:
         logger.info("parse document %s!",doc_path)
         return hash_result, meta_data
 
-    

src/component/openai_node/open_ai_node.py

@@ -206,7 +206,7 @@ class OpenAI_ComputeNode(ComputeNode):
                 if mode_name == "gpt-4-vision-preview":
                     response_format = NOT_GIVEN
                     llm_inner_functions = None
-                    if max_token_size > 4096:
+                    if max_token_size > 4096 or max_token_size < 50:
                         result_token = 4096
                     else:
                         result_token = -1
@@ -216,14 +216,16 @@ class OpenAI_ComputeNode(ComputeNode):
                 client = AsyncOpenAI(api_key=self.openai_api_key)
                 try:
                     if llm_inner_functions is None or len(llm_inner_functions) == 0:
-                        logger.info(f"call openai {mode_name} prompts: {prompts}")
+                        if mode_name != "gpt-4-vision-preview":
+                            logger.info(f"call openai {mode_name} prompts: {prompts}")
                         resp = await client.chat.completions.create(model=mode_name,
                                                         messages=prompts,
                                                         response_format = response_format,
                                                         max_tokens=result_token,
                                                         )
                     else:
-                        logger.info(f"call openai {mode_name} prompts: \n\t {prompts} \nfunctions: \n\t{json.dumps(llm_inner_functions,ensure_ascii=False)}")
+                        if mode_name != "gpt-4-vision-preview":
+                            logger.info(f"call openai {mode_name} prompts: \n\t {prompts} \nfunctions: \n\t{json.dumps(llm_inner_functions,ensure_ascii=False)}")
                         resp = await client.chat.completions.create(model=mode_name,
                                                             messages=prompts,
                                                             response_format = response_format,

src/component/slack_tunnel.py

@@ -119,6 +119,7 @@ class SlackTunnel(AgentTunnel):
                             continue
                         await download_file(file_info["file"]["url_private_download"], file_path, self.token)
 
+                        mime_type = file["mimetype"]
                         if file["mimetype"].startswith("image/"):
                             if file_type is None:
                                 file_type = "image"

src/requirements.txt

@@ -156,3 +156,4 @@ opencv-python
 discord.py
 slack_bolt
 wget
+moviepy