diff --git a/src/aios/agent/agent.py b/src/aios/agent/agent.py index 655d7f4..9e8ad16 100644 --- a/src/aios/agent/agent.py +++ b/src/aios/agent/agent.py @@ -84,7 +84,7 @@ class AIAgent(BaseAIAgent): todo_prompts = {} todo_prompts[TodoListType.TO_WORK] = { "do": None, - "check": None, + "check": None, "review": None, } todo_prompts[TodoListType.TO_LEARN] = { @@ -103,12 +103,12 @@ class AIAgent(BaseAIAgent): self.prviate_workspace : AgentWorkspace = None self.behaviors:Dict[str,BaseLLMProcess] = {} - + async def initial(self,params:Dict = None): self.base_dir = f"{AIStorage.get_instance().get_myai_dir()}/agent_data/{self.agent_id}" memory_base_dir = f"{self.base_dir}/memory" self.memory = AgentMemory(self.agent_id,memory_base_dir) - self.prviate_workspace = AgentWorkspace(self.agent_id) + self.prviate_workspace = AgentWorkspace(self.agent_id) init_params = {} init_params["memory"] = self.memory init_params["workspace"] = self.prviate_workspace @@ -117,7 +117,7 @@ class AIAgent(BaseAIAgent): if init_result is False: logger.error(f"llm process {process_name} initial failed! initial return False") return False - + self.wake_up() return True @@ -151,7 +151,7 @@ class AIAgent(BaseAIAgent): self.enable_timestamp = bool(config["enable_timestamp"]) if config.get("history_len"): self.history_len = int(config.get("history_len")) - + #load all LLMProcess self.behaviors = {} behaviors = config.get("behavior") @@ -201,12 +201,12 @@ class AIAgent(BaseAIAgent): context_info["owner"] = AIStorage.get_instance().get_user_config().get_value("username") return context_info - + async def llm_process_msg(self,msg:AgentMsg) -> AgentMsg: need_process:bool = True if msg.msg_type == AgentMsgType.TYPE_GROUPMSG: need_process = False - + session_topic = msg.target + "#" + msg.topic chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.memory.memory_db) if msg.mentions is not None: @@ -218,7 +218,7 @@ class AIAgent(BaseAIAgent): chatsession.append(msg) resp_msg = msg.create_group_resp_msg(self.agent_id,"") return resp_msg - + context_info = await self._get_context_info() input_parms = { "msg":msg, @@ -232,8 +232,11 @@ class AIAgent(BaseAIAgent): elif llm_result.state == LLMResultStates.IGNORE: return None else: # OK - resp_msg = llm_result.raw_result.get("_resp_msg") - return resp_msg + if llm_result.raw_result is not None: + resp_msg = llm_result.raw_result.get("_resp_msg") + return resp_msg + else: + return msg.create_resp_msg(llm_result.resp) async def _process_msg(self,msg:AgentMsg,workspace = None) -> AgentMsg: return await self.llm_process_msg(msg) @@ -264,7 +267,7 @@ class AIAgent(BaseAIAgent): else: logger.info(f"llm process self thinking ok!,think is:{llm_result.resp}") self.memory.set_last_think_time(time.time()) - self.agent_energy -= 2 + self.agent_energy -= 2 return async def llm_triage_tasklist(self): @@ -273,7 +276,7 @@ class AIAgent(BaseAIAgent): if self.prviate_workspace: filter = {} filter["state"] = AgentTaskState.TASK_STATE_WAIT - + tasklist:List[AgentTask]= await self.prviate_workspace.task_mgr.list_task(filter) @@ -281,8 +284,8 @@ class AIAgent(BaseAIAgent): if len(tasklist) > 0: simple_list:List[Dict] = [] for task in tasklist: - simple_list.append(task.to_simple_dict()) - + simple_list.append(task.to_simple_dict()) + input_parms = { "tasklist":simple_list, "context_info": await self._get_context_info() @@ -294,7 +297,7 @@ class AIAgent(BaseAIAgent): logger.info(f"llm process triage_tasks ignore!") else: logger.info(f"llm process triage_tasks ok!,think is:{llm_result.resp}") - self.agent_energy -= 3 + self.agent_energy -= 3 # for agent_task in tasklist: # if self.agent_energy <= 0: @@ -314,7 +317,7 @@ class AIAgent(BaseAIAgent): # else: # determine = llm_result.raw_result.get("determine") # logger.info(f"llm process review_task ok!,think is:{determine}") - # self.agent_energy -= 1 + # self.agent_energy -= 1 async def llm_do_todo(self, todo: AgentTodo): llm_process : BaseLLMProcess = self.behaviors.get("do") @@ -350,7 +353,7 @@ class AIAgent(BaseAIAgent): logger.info(f"llm process check_todo ok!,think is:{llm_result.resp}") self.agent_energy -= 1 - return + return async def llm_plan_task(self,task:AgentTask): llm_process : BaseLLMProcess = self.behaviors.get("plan_task") @@ -398,7 +401,7 @@ class AIAgent(BaseAIAgent): async def _on_timer(self): await asyncio.sleep(5) - while True: + while True: try: now = time.time() if self.last_recover_time is None: @@ -419,7 +422,7 @@ class AIAgent(BaseAIAgent): #filter["state"] = AgentTaskState.TASK_STATE_WAIT filter = None task_list:List[AgentTask] = await self.prviate_workspace.task_mgr.list_task(filter) - + for task in task_list: if self.agent_energy <= 0: break @@ -456,18 +459,18 @@ class AIAgent(BaseAIAgent): task = await self.prviate_workspace.task_mgr.get_task(task.task_id) if task.state == AgentTaskState.TASK_STATE_WAITING_REVIEW: await self.llm_review_task(task) - + await self._self_imporve() - - - + + + except Exception as e: tb_str = traceback.format_exc() logger.error(f"agent {self.agent_id} on timer error:{e},{tb_str}") - + # Because the LLM itself is very slow, the accuracy of the system processing task is in minutes. - await asyncio.sleep(30) - + await asyncio.sleep(30) + diff --git a/src/aios/agent/llm_process.py b/src/aios/agent/llm_process.py index e90c7ec..9e2e882 100644 --- a/src/aios/agent/llm_process.py +++ b/src/aios/agent/llm_process.py @@ -2,6 +2,7 @@ # pylint:disable=E0402 import os.path +from .chatsession import AIChatSession from ..utils import video_utils,image_utils from ..proto.compute_task import LLMPrompt,LLMResult,ComputeTaskResult,ComputeTaskResultCode @@ -165,7 +166,7 @@ class BaseLLMProcess(ABC): # Action define in prompt, will be execute after llm compute prompt = await self.prepare_prompt(input) - max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.model_name) + max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.get_llm_model_name()) #if max_result_token < MIN_PREDICT_TOKEN_LEN: # return LLMResult.from_error_str(f"prompt too long,can not predict") @@ -196,7 +197,11 @@ class BaseLLMProcess(ABC): # parse task_result to LLM Result if self.enable_json_resp: - llm_result = LLMResult.from_json_str(task_result.result_str) + try: + llm_result = LLMResult.from_json_str(task_result.result_str) + except Exception as e: + logger.error(f"parse llm result error:{e}") + llm_result = LLMResult.from_str(task_result.result_str) else: llm_result = LLMResult.from_str(task_result.result_str) @@ -402,14 +407,18 @@ class AgentMessageProcess(LLMAgentBaseProcess): if self.enable_media2text: logger.error(f"enable_media2text is not supported yet") else: - audio_file = msg.body + prompt, audio_file = msg.get_audio_body() resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, model=self.asr_model, prompt=None, response_format="text")) if resp.result_code != ComputeTaskResultCode.OK: error_resp = msg.create_error_resp(resp.error_str) return error_resp else: - msg.body = resp.result_str - msg_prompt.messages = [{"role":"user","content":resp.result_str}] + if prompt == "": + msg.body = resp.result_str + msg_prompt.messages = [{"role":"user","content":resp.result_str}] + else: + msg.body = f"{prompt}\nVoice content:{resp.result_str}" + msg_prompt.messages = [{"role":"user","content": prompt}, {"role": "user", "content": f"Voice content:{resp.result_str}"}] else: msg_prompt.messages = [{"role":"user","content":msg.body}] @@ -495,7 +504,8 @@ class AgentMessageProcess(LLMAgentBaseProcess): else: resp_msg = msg.create_resp_msg(llm_result.resp) - llm_result.raw_result["_resp_msg"] = resp_msg + if llm_result.raw_result is not None: + llm_result.raw_result["_resp_msg"] = resp_msg action_params = {} action_params["_input"] = input diff --git a/src/aios/proto/compute_task.py b/src/aios/proto/compute_task.py index e483d0e..f7f4371 100644 --- a/src/aios/proto/compute_task.py +++ b/src/aios/proto/compute_task.py @@ -210,8 +210,14 @@ class LLMResult: r.state = LLMResultStates.IGNORE return r - if llm_result_str[0] == "{": - return LLMResult.from_json_str(llm_result_str) + try: + if llm_result_str[0] == "{": + return LLMResult.from_json_str(llm_result_str) + + if llm_result_str.lstrip().rstrip().startswith("```json"): + return LLMResult.from_json_str(llm_result_str[7:-3]) + except: + pass lines = llm_result_str.splitlines() is_need_wait = False @@ -255,6 +261,8 @@ class LLMResult: r.resp += current_action.dumps() else: r.action_list.append(current_action) + + r.state = LLMResultStates.OK return r class ComputeTask: diff --git a/src/component/openai_node/open_ai_node.py b/src/component/openai_node/open_ai_node.py index 63849a1..345c6c4 100644 --- a/src/component/openai_node/open_ai_node.py +++ b/src/component/openai_node/open_ai_node.py @@ -206,7 +206,7 @@ class OpenAI_ComputeNode(ComputeNode): if mode_name == "gpt-4-vision-preview": response_format = NOT_GIVEN llm_inner_functions = None - if max_token_size > 4096: + if max_token_size > 4096 or max_token_size < 50: result_token = 4096 else: result_token = -1