AgentMsg support audio type

src/aios/agent/agent.py

@@ -13,7 +13,6 @@ import copy
 import sys
 
 from ..proto.agent_msg import AgentMsg
-from ..proto.compute_task import ComputeTaskResult,ComputeTaskResultCode
 
 from .agent_base import *
 from .chatsession import *
@@ -28,6 +27,7 @@ from ..storage.storage import AIStorage
 
 from ..knowledge import *
 from ..utils import video_utils, image_utils
+from ..proto.compute_task import ComputeTaskResult,ComputeTaskResultCode
 
 logger = logging.getLogger(__name__)
 
@@ -455,6 +455,15 @@ class AIAgent(BaseAIAgent):
                     content = [{"type": "text", "text": f"{msg.sender}:{video_prompt}"}]
                     content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
                     msg_prompt.messages = [{"role": "user", "content": content}]
+            elif msg.is_audio_msg():
+                audio_file = msg.body
+                resp = await ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text")
+                if resp.result_code != ComputeTaskResultCode.OK:
+                    error_resp = msg.create_error_resp(resp.error_str)
+                    return error_resp
+                else:
+                    msg.body = resp.result_str
+                    msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{resp.result_str}"}]
             else:
                 msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{msg.body}"}]
             session_topic = msg.target + "#" + msg.topic
@@ -487,6 +496,15 @@ class AIAgent(BaseAIAgent):
                     content = [{"type": "text", "text": video_prompt}]
                     content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
                     msg_prompt.messages = [{"role": "user", "content": content}]
+            elif msg.is_audio_msg():
+                audio_file = msg.body
+                resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text"))
+                if resp.result_code != ComputeTaskResultCode.OK:
+                    error_resp = msg.create_error_resp(resp.error_str)
+                    return error_resp
+                else:
+                    msg.body = resp.result_str
+                    msg_prompt.messages = [{"role":"user","content":resp.result_str}]
             else:
                 msg_prompt.messages = [{"role":"user","content":msg.body}]
             session_topic = msg.get_sender() + "#" + msg.topic

src/aios/proto/agent_msg.py

@@ -210,6 +210,13 @@ class AgentMsg:
             return True
         return False
 
+    def is_audio_msg(self) -> bool:
+        if self.body_mime is None:
+            return False
+        if self.body_mime.startswith("audio/"):
+            return True
+        return False
+
     def get_msg_id(self) -> str:
         return self.msg_id
 

src/aios/proto/compute_task.py

@@ -127,7 +127,7 @@ class ComputeTaskResult:
         self.callchain_id: str = None
         self.worker_id: str = None
         self.error_str : str = None
-        self.result_code: int = 0
+        self.result_code: int = ComputeTaskResultCode.OK
         self.result_str: str = None # easy to use,can read from result
 
         self.result : dict = {}

src/component/tg_tunnel.py

@@ -63,7 +63,7 @@ class TelegramTunnel(AgentTunnel):
         for update in updates:
             next_update_id = update.update_id + 1
 
-            if update.message and (update.message.text or (update.message.photo and len(update.message.photo) > 0) or update.message.video):
+            if update.message and (update.message.text or (update.message.photo and len(update.message.photo) > 0) or update.message.video or update.message.voice or update.message.audio):
 
                 await self.on_message(bot,update)
             return next_update_id
@@ -89,6 +89,10 @@ class TelegramTunnel(AgentTunnel):
                 update_id = (await self.bot.get_updates())[0].update_id
             except IndexError:
                 update_id = None
+            except Exception as e:
+                logger.error(f"tg_tunnel error:{e}")
+                logger.exception(e)
+                update_id = None
 
             #logger.info("listening for new messages...")
             while True:
@@ -179,6 +183,20 @@ class TelegramTunnel(AgentTunnel):
             await video_file.download_to_drive(file_path)
             agent_msg.body = agent_msg.create_video_body(file_path, message.caption)
             agent_msg.body_mime = f"video/{ext}"
+        elif message.audio is not None:
+            audio_file = await message.audio.get_file()
+            ext = audio_file.file_path.rsplit(".")[-1]
+            file_path = os.path.join(self.get_cache_path(), audio_file.file_id + f".{ext}")
+            await audio_file.download_to_drive(file_path)
+            agent_msg.body = file_path
+            agent_msg.body_mime = f"audio/{ext}"
+        elif message.voice is not None:
+            audio_file = await message.voice.get_file()
+            ext = audio_file.file_path.rsplit(".")[-1]
+            file_path = os.path.join(self.get_cache_path(), audio_file.file_id + f".{ext}")
+            await audio_file.download_to_drive(file_path)
+            agent_msg.body = file_path
+            agent_msg.body_mime = f"audio/{ext}"
 
         agent_msg.create_time = time.time()
         messag_type = message.chat.type