gyt/opendan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mime

Browse Source
This commit is contained in:
wugren
2023-12-01 14:22:34 +08:00
parent eb67980537
commit 9cf4613d31
11 changed files with 206 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/aios/agent/agent.py
+49 -2
View File
@@ -27,6 +27,7 @@ from ..environment.workspace_env import WorkspaceEnvironment
from ..storage.storage import AIStorage
from ..knowledge import *
from . import video_utils, image_utils
logger = logging.getLogger(__name__)
@@ -423,11 +424,39 @@ class AIAgent(BaseAIAgent):
async def _create_openai_thread(self) -> str:
return None
def check_and_to_base64(self, image_path: str) -> str:
if image_utils.is_file(image_path):
return image_utils.image_to_base64(image_path)
else:
return image_path
async def _process_msg(self,msg:AgentMsg,workspace = None) -> AgentMsg:
msg_prompt = AgentPrompt()
if msg.msg_type == AgentMsgType.TYPE_GROUPMSG:
need_process = False
msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{msg.body}"}]
if msg.is_image_msg():
image_prompt, images = msg.get_image_body()
if image_prompt is None:
content = [[{"type": "text", "text": f"{msg.sender}'s message"}]]
content.extend([{"type": "image_url", "url": self.check_and_to_base64(image)} for image in images])
msg_prompt.messages = [{"role": "user", "content": content}]
else:
content = [{"type": "text", "text": f"{msg.sender}:{image_prompt}"}]
content.extend([{"type": "image_url", "url": self.check_and_to_base64(image)} for image in images])
msg_prompt.messages = [{"role": "user", "content": content}]
elif msg.is_video_msg():
video_prompt, video = msg.get_video_body()
frames = video_utils.extract_frames(video)
if video_prompt is None:
content = [{"type": "text", "text": f"{msg.sender}'s message"}]
content.extend([{"type": "image_url", "url": frame} for frame in frames])
msg_prompt.messages = [{"role": "user", "content": content}]
else:
content = [{"type": "text", "text": f"{msg.sender}:{video_prompt}"}]
content.extend([{"type": "image_url", "url": frame} for frame in frames])
msg_prompt.messages = [{"role": "user", "content": content}]
else:
msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{msg.body}"}]
session_topic = msg.target + "#" + msg.topic
chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.chat_db)
@@ -441,7 +470,25 @@ class AIAgent(BaseAIAgent):
resp_msg = msg.create_group_resp_msg(self.agent_id,"")
return resp_msg
else:
msg_prompt.messages = [{"role":"user","content":msg.body}]
if msg.is_image_msg():
image_prompt, images = msg.get_image_body()
if image_prompt is None:
msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "url": image} for image in images]}]
else:
content = [{"type": "text", "text": image_prompt}]
content.extend([{"type": "image_url", "url": image} for image in images])
msg_prompt.messages = [{"role": "user", "content": content}]
elif msg.is_video_msg():
video_prompt, video = msg.get_video_body()
frames = video_utils.extract_frames(video)
if video_prompt is None:
msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "url": frame} for frame in frames]}]
else:
content = [{"type": "text", "text": video_prompt}]
content.extend([{"type": "image_url", "url": frame} for frame in frames])
msg_prompt.messages = [{"role": "user", "content": content}]
else:
msg_prompt.messages = [{"role":"user","content":msg.body}]
session_topic = msg.get_sender() + "#" + msg.topic
chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.chat_db)
if self.enable_thread:
src/aios/agent/agent_base.py
+26 -3
View File
@@ -9,7 +9,7 @@ import time
import re
import shlex
import json
from typing import List
from typing import List, Tuple
from .ai_function import FunctionItem, AIFunction
from ..proto.agent_msg import AgentMsg, AgentMsgType
@@ -410,6 +410,10 @@ class BaseAIAgent(abc.ABC):
def get_max_token_size(self) -> int:
pass
@abstractmethod
async def _process_msg(self,msg:AgentMsg,workspace = None) -> AgentMsg:
pass
@classmethod
def get_inner_functions(cls, env:Environment) -> (dict,int):
if env is None:
@@ -445,10 +449,29 @@ class BaseAIAgent(abc.ABC):
#logger.debug(f"Agent {self.agent_id} do llm token static system:{system_prompt_len},function:{function_token_len},history:{history_token_len},input:{input_len}, totoal prompt:{system_prompt_len + function_token_len + history_token_len} ")
if inner_functions is None and env is not None:
inner_functions,_ = BaseAIAgent.get_inner_functions(env)
model_name = self.get_llm_model_name()
if org_msg.is_video_msg() or org_msg.is_image_msg():
if model_name.startswith("gpt4"):
model_name = "gpt-4-vision-preview"
if is_json_resp:
task_result:ComputeTaskResult = await ComputeKernel.get_instance().do_llm_completion(prompt,resp_mode="json",mode_name=self.get_llm_model_name(),max_token=self.get_max_token_size(),inner_functions=inner_functions,timeout=None)
task_result: ComputeTaskResult = await (ComputeKernel.get_instance()
.do_llm_completion(
prompt,
resp_mode="json",
mode_name=model_name,
max_token=self.get_max_token_size(),
inner_functions=inner_functions,
timeout=None))
else:
task_result:ComputeTaskResult = await ComputeKernel.get_instance().do_llm_completion(prompt,resp_mode="text",mode_name=self.get_llm_model_name(),max_token=self.get_max_token_size(),inner_functions=inner_functions,timeout=None)
task_result: ComputeTaskResult = await (ComputeKernel.get_instance()
.do_llm_completion(
prompt,
resp_mode="text",
mode_name=model_name,
max_token=self.get_max_token_size(),
inner_functions=inner_functions,
timeout=None))
if task_result.result_code != ComputeTaskResultCode.OK:
logger.error(f"_do_llm_complection llm compute error:{task_result.error_str}")
#error_resp = msg.create_error_resp(task_result.error_str)