diff --git a/src/aios/agent/agent.py b/src/aios/agent/agent.py index 2b97326..dc4588f 100644 --- a/src/aios/agent/agent.py +++ b/src/aios/agent/agent.py @@ -13,7 +13,6 @@ import copy import sys from ..proto.agent_msg import AgentMsg -from ..proto.compute_task import ComputeTaskResult,ComputeTaskResultCode from .agent_base import * from .chatsession import * @@ -28,6 +27,7 @@ from ..storage.storage import AIStorage from ..knowledge import * from ..utils import video_utils, image_utils +from ..proto.compute_task import ComputeTaskResult,ComputeTaskResultCode logger = logging.getLogger(__name__) @@ -455,6 +455,15 @@ class AIAgent(BaseAIAgent): content = [{"type": "text", "text": f"{msg.sender}:{video_prompt}"}] content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames]) msg_prompt.messages = [{"role": "user", "content": content}] + elif msg.is_audio_msg(): + audio_file = msg.body + resp = await ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text") + if resp.result_code != ComputeTaskResultCode.OK: + error_resp = msg.create_error_resp(resp.error_str) + return error_resp + else: + msg.body = resp.result_str + msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{resp.result_str}"}] else: msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{msg.body}"}] session_topic = msg.target + "#" + msg.topic @@ -487,6 +496,15 @@ class AIAgent(BaseAIAgent): content = [{"type": "text", "text": video_prompt}] content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames]) msg_prompt.messages = [{"role": "user", "content": content}] + elif msg.is_audio_msg(): + audio_file = msg.body + resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text")) + if resp.result_code != ComputeTaskResultCode.OK: + error_resp = msg.create_error_resp(resp.error_str) + return error_resp + else: + msg.body = resp.result_str + msg_prompt.messages = [{"role":"user","content":resp.result_str}] else: msg_prompt.messages = [{"role":"user","content":msg.body}] session_topic = msg.get_sender() + "#" + msg.topic diff --git a/src/aios/proto/agent_msg.py b/src/aios/proto/agent_msg.py index c7209bf..21ec3b6 100644 --- a/src/aios/proto/agent_msg.py +++ b/src/aios/proto/agent_msg.py @@ -210,6 +210,13 @@ class AgentMsg: return True return False + def is_audio_msg(self) -> bool: + if self.body_mime is None: + return False + if self.body_mime.startswith("audio/"): + return True + return False + def get_msg_id(self) -> str: return self.msg_id diff --git a/src/aios/proto/compute_task.py b/src/aios/proto/compute_task.py index c226b51..3390465 100644 --- a/src/aios/proto/compute_task.py +++ b/src/aios/proto/compute_task.py @@ -75,7 +75,7 @@ class ComputeTask: else: self.params["model_name"] = "text-embedding-ada-002" self.params["input"] = input - + def set_image_embedding_params(self, input = Union[ObjectID, bytes], model_name=None, callchain_id = None): self.task_type = ComputeTaskType.IMAGE_EMBEDDING self.create_time = time.time() @@ -86,7 +86,7 @@ class ComputeTask: else: self.params["model_name"] = None self.params["input"] = input - + def set_text_2_image_params(self, prompt: str, model_name, negative_prompt="", callchain_id=None): self.task_type = ComputeTaskType.TEXT_2_IMAGE self.create_time = time.time() @@ -126,15 +126,15 @@ class ComputeTaskResult: self.task_id: str = None self.callchain_id: str = None self.worker_id: str = None - self.error_str : str = None - self.result_code: int = 0 + self.error_str : str = None + self.result_code: int = ComputeTaskResultCode.OK self.result_str: str = None # easy to use,can read from result self.result : dict = {} self.result_refers: dict = {} self.pading_data: bytearray = None - + def set_from_task(self, task: ComputeTask): self.task_id = task.task_id diff --git a/src/component/tg_tunnel.py b/src/component/tg_tunnel.py index fa0e5e5..4406591 100644 --- a/src/component/tg_tunnel.py +++ b/src/component/tg_tunnel.py @@ -63,7 +63,7 @@ class TelegramTunnel(AgentTunnel): for update in updates: next_update_id = update.update_id + 1 - if update.message and (update.message.text or (update.message.photo and len(update.message.photo) > 0) or update.message.video): + if update.message and (update.message.text or (update.message.photo and len(update.message.photo) > 0) or update.message.video or update.message.voice or update.message.audio): await self.on_message(bot,update) return next_update_id @@ -89,6 +89,10 @@ class TelegramTunnel(AgentTunnel): update_id = (await self.bot.get_updates())[0].update_id except IndexError: update_id = None + except Exception as e: + logger.error(f"tg_tunnel error:{e}") + logger.exception(e) + update_id = None #logger.info("listening for new messages...") while True: @@ -179,6 +183,20 @@ class TelegramTunnel(AgentTunnel): await video_file.download_to_drive(file_path) agent_msg.body = agent_msg.create_video_body(file_path, message.caption) agent_msg.body_mime = f"video/{ext}" + elif message.audio is not None: + audio_file = await message.audio.get_file() + ext = audio_file.file_path.rsplit(".")[-1] + file_path = os.path.join(self.get_cache_path(), audio_file.file_id + f".{ext}") + await audio_file.download_to_drive(file_path) + agent_msg.body = file_path + agent_msg.body_mime = f"audio/{ext}" + elif message.voice is not None: + audio_file = await message.voice.get_file() + ext = audio_file.file_path.rsplit(".")[-1] + file_path = os.path.join(self.get_cache_path(), audio_file.file_id + f".{ext}") + await audio_file.download_to_drive(file_path) + agent_msg.body = file_path + agent_msg.body_mime = f"audio/{ext}" agent_msg.create_time = time.time() messag_type = message.chat.type