gyt/opendan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix multimodal input bug

Browse Source
This commit is contained in:
wugren
2024-02-17 13:37:03 +08:00
parent 42c4f97a94
commit ed0adbede9
4 changed files with 56 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/aios/agent/agent.py
+29 -26
View File
@@ -84,7 +84,7 @@ class AIAgent(BaseAIAgent):
todo_prompts = {}
todo_prompts[TodoListType.TO_WORK] = {
"do": None,
"check": None,
"check": None,
"review": None,
}
todo_prompts[TodoListType.TO_LEARN] = {
@@ -103,12 +103,12 @@ class AIAgent(BaseAIAgent):
self.prviate_workspace : AgentWorkspace = None
self.behaviors:Dict[str,BaseLLMProcess] = {}
async def initial(self,params:Dict = None):
self.base_dir = f"{AIStorage.get_instance().get_myai_dir()}/agent_data/{self.agent_id}"
memory_base_dir = f"{self.base_dir}/memory"
self.memory = AgentMemory(self.agent_id,memory_base_dir)
self.prviate_workspace = AgentWorkspace(self.agent_id)
self.prviate_workspace = AgentWorkspace(self.agent_id)
init_params = {}
init_params["memory"] = self.memory
init_params["workspace"] = self.prviate_workspace
@@ -117,7 +117,7 @@ class AIAgent(BaseAIAgent):
if init_result is False:
logger.error(f"llm process {process_name} initial failed! initial return False")
return False
self.wake_up()
return True
@@ -151,7 +151,7 @@ class AIAgent(BaseAIAgent):
self.enable_timestamp = bool(config["enable_timestamp"])
if config.get("history_len"):
self.history_len = int(config.get("history_len"))
#load all LLMProcess
self.behaviors = {}
behaviors = config.get("behavior")
@@ -201,12 +201,12 @@ class AIAgent(BaseAIAgent):
context_info["owner"] = AIStorage.get_instance().get_user_config().get_value("username")
return context_info
async def llm_process_msg(self,msg:AgentMsg) -> AgentMsg:
need_process:bool = True
if msg.msg_type == AgentMsgType.TYPE_GROUPMSG:
need_process = False
session_topic = msg.target + "#" + msg.topic
chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.memory.memory_db)
if msg.mentions is not None:
@@ -218,7 +218,7 @@ class AIAgent(BaseAIAgent):
chatsession.append(msg)
resp_msg = msg.create_group_resp_msg(self.agent_id,"")
return resp_msg
context_info = await self._get_context_info()
input_parms = {
"msg":msg,
@@ -232,8 +232,11 @@ class AIAgent(BaseAIAgent):
elif llm_result.state == LLMResultStates.IGNORE:
return None
else: # OK
resp_msg = llm_result.raw_result.get("_resp_msg")
return resp_msg
if llm_result.raw_result is not None:
resp_msg = llm_result.raw_result.get("_resp_msg")
return resp_msg
else:
return msg.create_resp_msg(llm_result.resp)
async def _process_msg(self,msg:AgentMsg,workspace = None) -> AgentMsg:
return await self.llm_process_msg(msg)
@@ -264,7 +267,7 @@ class AIAgent(BaseAIAgent):
else:
logger.info(f"llm process self thinking ok!,think is:{llm_result.resp}")
self.memory.set_last_think_time(time.time())
self.agent_energy -= 2
self.agent_energy -= 2
return
async def llm_triage_tasklist(self):
@@ -273,7 +276,7 @@ class AIAgent(BaseAIAgent):
if self.prviate_workspace:
filter = {}
filter["state"] = AgentTaskState.TASK_STATE_WAIT
tasklist:List[AgentTask]= await self.prviate_workspace.task_mgr.list_task(filter)
@@ -281,8 +284,8 @@ class AIAgent(BaseAIAgent):
if len(tasklist) > 0:
simple_list:List[Dict] = []
for task in tasklist:
simple_list.append(task.to_simple_dict())
simple_list.append(task.to_simple_dict())
input_parms = {
"tasklist":simple_list,
"context_info": await self._get_context_info()
@@ -294,7 +297,7 @@ class AIAgent(BaseAIAgent):
logger.info(f"llm process triage_tasks ignore!")
else:
logger.info(f"llm process triage_tasks ok!,think is:{llm_result.resp}")
self.agent_energy -= 3
self.agent_energy -= 3
# for agent_task in tasklist:
# if self.agent_energy <= 0:
@@ -314,7 +317,7 @@ class AIAgent(BaseAIAgent):
# else:
# determine = llm_result.raw_result.get("determine")
# logger.info(f"llm process review_task ok!,think is:{determine}")
# self.agent_energy -= 1
# self.agent_energy -= 1
async def llm_do_todo(self, todo: AgentTodo):
llm_process : BaseLLMProcess = self.behaviors.get("do")
@@ -350,7 +353,7 @@ class AIAgent(BaseAIAgent):
logger.info(f"llm process check_todo ok!,think is:{llm_result.resp}")
self.agent_energy -= 1
return
return
async def llm_plan_task(self,task:AgentTask):
llm_process : BaseLLMProcess = self.behaviors.get("plan_task")
@@ -398,7 +401,7 @@ class AIAgent(BaseAIAgent):
async def _on_timer(self):
await asyncio.sleep(5)
while True:
while True:
try:
now = time.time()
if self.last_recover_time is None:
@@ -419,7 +422,7 @@ class AIAgent(BaseAIAgent):
#filter["state"] = AgentTaskState.TASK_STATE_WAIT
filter = None
task_list:List[AgentTask] = await self.prviate_workspace.task_mgr.list_task(filter)
for task in task_list:
if self.agent_energy <= 0:
break
@@ -456,18 +459,18 @@ class AIAgent(BaseAIAgent):
task = await self.prviate_workspace.task_mgr.get_task(task.task_id)
if task.state == AgentTaskState.TASK_STATE_WAITING_REVIEW:
await self.llm_review_task(task)
await self._self_imporve()
except Exception as e:
tb_str = traceback.format_exc()
logger.error(f"agent {self.agent_id} on timer error:{e},{tb_str}")
# Because the LLM itself is very slow, the accuracy of the system processing task is in minutes.
await asyncio.sleep(30)
await asyncio.sleep(30)
src/aios/agent/llm_process.py
+16 -6
View File
@@ -2,6 +2,7 @@
# pylint:disable=E0402
import os.path
from .chatsession import AIChatSession
from ..utils import video_utils,image_utils
from ..proto.compute_task import LLMPrompt,LLMResult,ComputeTaskResult,ComputeTaskResultCode
@@ -165,7 +166,7 @@ class BaseLLMProcess(ABC):
# Action define in prompt, will be execute after llm compute
prompt = await self.prepare_prompt(input)
max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.model_name)
max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.get_llm_model_name())
#if max_result_token < MIN_PREDICT_TOKEN_LEN:
# return LLMResult.from_error_str(f"prompt too long,can not predict")
@@ -196,7 +197,11 @@ class BaseLLMProcess(ABC):
# parse task_result to LLM Result
if self.enable_json_resp:
llm_result = LLMResult.from_json_str(task_result.result_str)
try:
llm_result = LLMResult.from_json_str(task_result.result_str)
except Exception as e:
logger.error(f"parse llm result error:{e}")
llm_result = LLMResult.from_str(task_result.result_str)
else:
llm_result = LLMResult.from_str(task_result.result_str)
@@ -402,14 +407,18 @@ class AgentMessageProcess(LLMAgentBaseProcess):
if self.enable_media2text:
logger.error(f"enable_media2text is not supported yet")
else:
audio_file = msg.body
prompt, audio_file = msg.get_audio_body()
resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, model=self.asr_model, prompt=None, response_format="text"))
if resp.result_code != ComputeTaskResultCode.OK:
error_resp = msg.create_error_resp(resp.error_str)
return error_resp
else:
msg.body = resp.result_str
msg_prompt.messages = [{"role":"user","content":resp.result_str}]
if prompt == "":
msg.body = resp.result_str
msg_prompt.messages = [{"role":"user","content":resp.result_str}]
else:
msg.body = f"{prompt}\nVoice content:{resp.result_str}"
msg_prompt.messages = [{"role":"user","content": prompt}, {"role": "user", "content": f"Voice content:{resp.result_str}"}]
else:
msg_prompt.messages = [{"role":"user","content":msg.body}]
@@ -495,7 +504,8 @@ class AgentMessageProcess(LLMAgentBaseProcess):
else:
resp_msg = msg.create_resp_msg(llm_result.resp)
llm_result.raw_result["_resp_msg"] = resp_msg
if llm_result.raw_result is not None:
llm_result.raw_result["_resp_msg"] = resp_msg
action_params = {}
action_params["_input"] = input
src/aios/proto/compute_task.py
+10 -2
View File
@@ -210,8 +210,14 @@ class LLMResult:
r.state = LLMResultStates.IGNORE
return r
if llm_result_str[0] == "{":
return LLMResult.from_json_str(llm_result_str)
try:
if llm_result_str[0] == "{":
return LLMResult.from_json_str(llm_result_str)
if llm_result_str.lstrip().rstrip().startswith("```json"):
return LLMResult.from_json_str(llm_result_str[7:-3])
except:
pass
lines = llm_result_str.splitlines()
is_need_wait = False
@@ -255,6 +261,8 @@ class LLMResult:
r.resp += current_action.dumps()
else:
r.action_list.append(current_action)
r.state = LLMResultStates.OK
return r
class ComputeTask:
src/component/openai_node/open_ai_node.py
+1 -1
View File
@@ -206,7 +206,7 @@ class OpenAI_ComputeNode(ComputeNode):
if mode_name == "gpt-4-vision-preview":
response_format = NOT_GIVEN
llm_inner_functions = None
if max_token_size > 4096:
if max_token_size > 4096 or max_token_size < 50:
result_token = 4096
else:
result_token = -1