gyt/opendan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #121 from wugren/MVP

Browse Source 
Support multimodal input
This commit is contained in:
Liu Zhicong
2024-02-27 10:31:36 -08:00
committed by GitHub
parent 3e9f8ac0ed ed0adbede9
commit 7e9b29eef9
9 changed files with 211 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rootfs/agents/Jarvis/agent.toml
+2
View File
@@ -16,6 +16,8 @@ Only clearly specifying the task you completed can be completed independently.
type="AgentMessageProcess"
# TODO: 是否应该自动记录 inner function和action的执行细节
mutil_model="gpt-4-vision-preview"
asr_model="openai-whisper"
tts_model="tts-1"
process_description="""
1. Based on your role and the existing information, please think and then make a brief and efficient reply.
src/aios/agent/agent.py
+5 -2
View File
@@ -232,8 +232,11 @@ class AIAgent(BaseAIAgent):
elif llm_result.state == LLMResultStates.IGNORE:
return None
else: # OK
resp_msg = llm_result.raw_result.get("_resp_msg")
return resp_msg
if llm_result.raw_result is not None:
resp_msg = llm_result.raw_result.get("_resp_msg")
return resp_msg
else:
return msg.create_resp_msg(llm_result.resp)
async def _process_msg(self,msg:AgentMsg,workspace = None) -> AgentMsg:
return await self.llm_process_msg(msg)
src/aios/agent/llm_process.py
+53 -15
View File
@@ -1,5 +1,8 @@
# Old name is behavior, I belive new name "llm_process" is better
# pylint:disable=E0402
import os.path
from .chatsession import AIChatSession
from ..utils import video_utils,image_utils
from ..proto.compute_task import LLMPrompt,LLMResult,ComputeTaskResult,ComputeTaskResultCode
@@ -163,7 +166,7 @@ class BaseLLMProcess(ABC):
# Action define in prompt, will be execute after llm compute
prompt = await self.prepare_prompt(input)
max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.model_name)
max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.get_llm_model_name())
#if max_result_token < MIN_PREDICT_TOKEN_LEN:
# return LLMResult.from_error_str(f"prompt too long,can not predict")
@@ -194,7 +197,11 @@ class BaseLLMProcess(ABC):
# parse task_result to LLM Result
if self.enable_json_resp:
llm_result = LLMResult.from_json_str(task_result.result_str)
try:
llm_result = LLMResult.from_json_str(task_result.result_str)
except Exception as e:
logger.error(f"parse llm result error:{e}")
llm_result = LLMResult.from_str(task_result.result_str)
else:
llm_result = LLMResult.from_str(task_result.result_str)
@@ -316,6 +323,8 @@ class AgentMessageProcess(LLMAgentBaseProcess):
self.mutil_model = None
self.enable_media2text = False
self.is_mutil_model = False
self.asr_model = None
self.tts_model = None
async def load_default_config(self) -> bool:
return True
@@ -332,6 +341,9 @@ class AgentMessageProcess(LLMAgentBaseProcess):
if config.get("mutil_model"):
self.mutil_model = config.get("mutil_model")
self.asr_model = config.get("asr_model")
self.tts_model = config.get("tts_model")
def get_llm_model_name(self) -> str:
if self.is_mutil_model:
return self.mutil_model
@@ -365,23 +377,48 @@ class AgentMessageProcess(LLMAgentBaseProcess):
logger.warning(f"mutil_model is not set!")
elif msg.is_video_msg():
video_prompt, video = msg.get_video_body()
frames = video_utils.extract_frames(video, (1024, 1024))
if video_prompt is None:
msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": frame}} for frame in frames]}]
if self.enable_media2text:
logger.error(f"enable_media2text is not supported yet")
else:
content = [{"type": "text", "text": video_prompt}]
video_prompt, video = msg.get_video_body()
frames = video_utils.extract_frames(video, (1024, 1024))
audio_file = os.path.splitext(video)[0] + ".mp3"
video_utils.extract_audio(video, audio_file)
voice_content = None
if self.asr_model is not None:
resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, model=self.asr_model, prompt=None, response_format="text"))
if resp.result_code == ComputeTaskResultCode.OK:
voice_content = resp.result_str
content = []
if video_prompt is not None:
content.append({"type": "text", "text": video_prompt})
if voice_content is not None and voice_content != "":
content.append({"type": "text", "text": f"Voice content in video:{voice_content}"})
content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
msg_prompt.messages = [{"role": "user", "content": content}]
if self.mutil_model:
self.is_mutil_model = True
else:
logger.warning(f"mutil_model is not set!")
elif msg.is_audio_msg():
audio_file = msg.body
resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text"))
if resp.result_code != ComputeTaskResultCode.OK:
error_resp = msg.create_error_resp(resp.error_str)
return error_resp
if self.enable_media2text:
logger.error(f"enable_media2text is not supported yet")
else:
msg.body = resp.result_str
msg_prompt.messages = [{"role":"user","content":resp.result_str}]
prompt, audio_file = msg.get_audio_body()
resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, model=self.asr_model, prompt=None, response_format="text"))
if resp.result_code != ComputeTaskResultCode.OK:
error_resp = msg.create_error_resp(resp.error_str)
return error_resp
else:
if prompt == "":
msg.body = resp.result_str
msg_prompt.messages = [{"role":"user","content":resp.result_str}]
else:
msg.body = f"{prompt}\nVoice content:{resp.result_str}"
msg_prompt.messages = [{"role":"user","content": prompt}, {"role": "user", "content": f"Voice content:{resp.result_str}"}]
else:
msg_prompt.messages = [{"role":"user","content":msg.body}]
@@ -467,7 +504,8 @@ class AgentMessageProcess(LLMAgentBaseProcess):
else:
resp_msg = msg.create_resp_msg(llm_result.resp)
llm_result.raw_result["_resp_msg"] = resp_msg
if llm_result.raw_result is not None:
llm_result.raw_result["_resp_msg"] = resp_msg
action_params = {}
action_params["_input"] = input
src/aios/proto/compute_task.py
+12 -4
View File
@@ -164,8 +164,8 @@ class LLMResult:
@classmethod
def from_error_str(self,error_str:str) -> 'LLMResult':
r = LLMResult()
r.state = "error"
r.compute_error_str = error_str
r.state = LLMResultStates.ERROR
r.error_str = error_str
return r
@classmethod
@@ -210,8 +210,14 @@ class LLMResult:
r.state = LLMResultStates.IGNORE
return r
if llm_result_str[0] == "{":
return LLMResult.from_json_str(llm_result_str)
try:
if llm_result_str[0] == "{":
return LLMResult.from_json_str(llm_result_str)
if llm_result_str.lstrip().rstrip().startswith("```json"):
return LLMResult.from_json_str(llm_result_str[7:-3])
except:
pass
lines = llm_result_str.splitlines()
is_need_wait = False
@@ -255,6 +261,8 @@ class LLMResult:
r.resp += current_action.dumps()
else:
r.action_list.append(current_action)
r.state = LLMResultStates.OK
return r
class ComputeTask:
src/aios/utils/video_utils.py
+6
View File
@@ -3,6 +3,7 @@ from typing import List, Tuple
import cv2
import numpy as np
import moviepy.editor as mp
def precess_image(image):
@@ -120,3 +121,8 @@ def extract_frames(video_path: str, resize: Tuple[int, int] = None, smooth=False
i += 1
vidcap.release()
return frames
def extract_audio(video_path: str, audio_path: str):
my_clip = mp.VideoFileClip(video_path)
my_clip.audio.write_audiofile(audio_path)
src/component/common_environment/local_document.py
+1 -1
View File
@@ -13,6 +13,7 @@ import PyPDF2
import datetime
from typing import Optional, List
from aios import *
from aios.environment.workspace_env import TodoListEnvironment, TodoListType
from .local_file_system import FilesystemEnvironment
logger = logging.getLogger(__name__)
@@ -615,4 +616,3 @@ class ParseLocalDocument:
logger.info("parse document %s!",doc_path)
return hash_result, meta_data
src/component/openai_node/open_ai_node.py
+5 -3
View File
@@ -206,7 +206,7 @@ class OpenAI_ComputeNode(ComputeNode):
if mode_name == "gpt-4-vision-preview":
response_format = NOT_GIVEN
llm_inner_functions = None
if max_token_size > 4096:
if max_token_size > 4096 or max_token_size < 50:
result_token = 4096
else:
result_token = -1
@@ -216,14 +216,16 @@ class OpenAI_ComputeNode(ComputeNode):
client = AsyncOpenAI(api_key=self.openai_api_key)
try:
if llm_inner_functions is None or len(llm_inner_functions) == 0:
logger.info(f"call openai {mode_name} prompts: {prompts}")
if mode_name != "gpt-4-vision-preview":
logger.info(f"call openai {mode_name} prompts: {prompts}")
resp = await client.chat.completions.create(model=mode_name,
messages=prompts,
response_format = response_format,
max_tokens=result_token,
)
else:
logger.info(f"call openai {mode_name} prompts: \n\t {prompts} \nfunctions: \n\t{json.dumps(llm_inner_functions,ensure_ascii=False)}")
if mode_name != "gpt-4-vision-preview":
logger.info(f"call openai {mode_name} prompts: \n\t {prompts} \nfunctions: \n\t{json.dumps(llm_inner_functions,ensure_ascii=False)}")
resp = await client.chat.completions.create(model=mode_name,
messages=prompts,
response_format = response_format,
src/component/slack_tunnel.py
+1
View File
@@ -119,6 +119,7 @@ class SlackTunnel(AgentTunnel):
continue
await download_file(file_info["file"]["url_private_download"], file_path, self.token)
mime_type = file["mimetype"]
if file["mimetype"].startswith("image/"):
if file_type is None:
file_type = "image"
src/requirements.txt
+1
View File
@@ -156,3 +156,4 @@ opencv-python
discord.py
slack_bolt
wget
moviepy