Support multimodal input

rootfs/agents/Jarvis/agent.toml

@@ -16,6 +16,8 @@ Only clearly specifying the task you completed can be completed independently.
 type="AgentMessageProcess"
 # TODO: 是否应该自动记录 inner function和action的执行细节
 mutil_model="gpt-4-vision-preview"
+asr_model="openai-whisper"
+tts_model="tts-1"
 
 process_description="""
 1. Based on your role and the existing information, please think and then make a brief and efficient reply.

src/aios/agent/llm_process.py

@@ -1,5 +1,7 @@
 # Old name is behavior, I belive new name "llm_process" is better
 # pylint:disable=E0402
+import os.path
+
 from ..utils import video_utils,image_utils
 
 from ..proto.compute_task import LLMPrompt,LLMResult,ComputeTaskResult,ComputeTaskResultCode
@@ -316,6 +318,8 @@ class AgentMessageProcess(LLMAgentBaseProcess):
         self.mutil_model = None
         self.enable_media2text = False
         self.is_mutil_model = False
+        self.asr_model = None
+        self.tts_model = None
 
     async def load_default_config(self) -> bool:
         return True
@@ -332,6 +336,9 @@ class AgentMessageProcess(LLMAgentBaseProcess):
         if config.get("mutil_model"):
             self.mutil_model = config.get("mutil_model")
 
+        self.asr_model = config.get("asr_model")
+        self.tts_model = config.get("tts_model")
+
     def get_llm_model_name(self) -> str:
         if self.is_mutil_model:
             return self.mutil_model
@@ -365,23 +372,44 @@ class AgentMessageProcess(LLMAgentBaseProcess):
                     logger.warning(f"mutil_model is not set!")
 
         elif msg.is_video_msg():
-            video_prompt, video = msg.get_video_body()
+            if self.enable_media2text:
-            frames = video_utils.extract_frames(video, (1024, 1024))
+                logger.error(f"enable_media2text is not supported yet")
-            if video_prompt is None:
-                msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": frame}} for frame in frames]}]
             else:
-                content = [{"type": "text", "text": video_prompt}]
+                video_prompt, video = msg.get_video_body()
+                frames = video_utils.extract_frames(video, (1024, 1024))
+                audio_file = os.path.splitext(video)[0] + ".mp3"
+                video_utils.extract_audio(video, audio_file)
+
+                voice_content = None
+                if self.asr_model is not None:
+                    resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, model=self.asr_model, prompt=None, response_format="text"))
+                    if resp.result_code == ComputeTaskResultCode.OK:
+                        voice_content = resp.result_str
+
+                content = []
+                if video_prompt is not None:
+                    content.append({"type": "text", "text": video_prompt})
+                if voice_content is not None and voice_content != "":
+                    content.append({"type": "text", "text": f"Voice content in video:{voice_content}"})
+
                 content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
                 msg_prompt.messages = [{"role": "user", "content": content}]
+                if self.mutil_model:
+                    self.is_mutil_model = True
+                else:
+                    logger.warning(f"mutil_model is not set!")
         elif msg.is_audio_msg():
-            audio_file = msg.body
+            if self.enable_media2text:
-            resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text"))
+                logger.error(f"enable_media2text is not supported yet")
-            if resp.result_code != ComputeTaskResultCode.OK:
-                error_resp = msg.create_error_resp(resp.error_str)
-                return error_resp
             else:
-                msg.body = resp.result_str
+                audio_file = msg.body
-                msg_prompt.messages = [{"role":"user","content":resp.result_str}]
+                resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, model=self.asr_model, prompt=None, response_format="text"))
+                if resp.result_code != ComputeTaskResultCode.OK:
+                    error_resp = msg.create_error_resp(resp.error_str)
+                    return error_resp
+                else:
+                    msg.body = resp.result_str
+                    msg_prompt.messages = [{"role":"user","content":resp.result_str}]
         else:
             msg_prompt.messages = [{"role":"user","content":msg.body}]
 

src/aios/proto/compute_task.py

@@ -164,8 +164,8 @@ class LLMResult:
     @classmethod
     def from_error_str(self,error_str:str) -> 'LLMResult':
         r = LLMResult()
-        r.state = "error"
+        r.state = LLMResultStates.ERROR
-        r.compute_error_str = error_str
+        r.error_str = error_str
         return r
 
     @classmethod

src/aios/utils/video_utils.py

@@ -3,6 +3,7 @@ from typing import List, Tuple
 
 import cv2
 import numpy as np
+import moviepy.editor as mp
 
 
 def precess_image(image):
@@ -120,3 +121,8 @@ def extract_frames(video_path: str, resize: Tuple[int, int] = None, smooth=False
         i += 1
     vidcap.release()
     return frames
+
+
+def extract_audio(video_path: str, audio_path: str):
+    my_clip = mp.VideoFileClip(video_path)
+    my_clip.audio.write_audiofile(audio_path)

src/component/common_environment/local_document.py

@@ -13,6 +13,7 @@ import PyPDF2
 import datetime
 from typing import Optional, List
 from aios import *
+from aios.environment.workspace_env import TodoListEnvironment, TodoListType
 from .local_file_system import FilesystemEnvironment
 
 logger = logging.getLogger(__name__)
@@ -615,4 +616,3 @@ class ParseLocalDocument:
         logger.info("parse document %s!",doc_path)
         return hash_result, meta_data
 
-    

src/component/openai_node/open_ai_node.py

@@ -216,14 +216,16 @@ class OpenAI_ComputeNode(ComputeNode):
                 client = AsyncOpenAI(api_key=self.openai_api_key)
                 try:
                     if llm_inner_functions is None or len(llm_inner_functions) == 0:
-                        logger.info(f"call openai {mode_name} prompts: {prompts}")
+                        if mode_name != "gpt-4-vision-preview":
+                            logger.info(f"call openai {mode_name} prompts: {prompts}")
                         resp = await client.chat.completions.create(model=mode_name,
                                                         messages=prompts,
                                                         response_format = response_format,
                                                         max_tokens=result_token,
                                                         )
                     else:
-                        logger.info(f"call openai {mode_name} prompts: \n\t {prompts} \nfunctions: \n\t{json.dumps(llm_inner_functions,ensure_ascii=False)}")
+                        if mode_name != "gpt-4-vision-preview":
+                            logger.info(f"call openai {mode_name} prompts: \n\t {prompts} \nfunctions: \n\t{json.dumps(llm_inner_functions,ensure_ascii=False)}")
                         resp = await client.chat.completions.create(model=mode_name,
                                                             messages=prompts,
                                                             response_format = response_format,

src/component/slack_tunnel.py

@@ -119,6 +119,7 @@ class SlackTunnel(AgentTunnel):
                             continue
                         await download_file(file_info["file"]["url_private_download"], file_path, self.token)
 
+                        mime_type = file["mimetype"]
                         if file["mimetype"].startswith("image/"):
                             if file_type is None:
                                 file_type = "image"

src/requirements.txt

@@ -156,3 +156,4 @@ opencv-python
 discord.py
 slack_bolt
 wget
+moviepy