Add discord tunnel

src/aios/agent/agent.py

@@ -311,14 +311,20 @@ class AIAgent(BaseAIAgent):
                     content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
                     msg_prompt.messages = [{"role": "user", "content": content}]
             elif msg.is_audio_msg():
-                audio_file = msg.body
+                prompt, audio_file = msg.get_audio_body()
                 resp = await ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text")
                 if resp.result_code != ComputeTaskResultCode.OK:
                     error_resp = msg.create_error_resp(resp.error_str)
                     return error_resp
                 else:
-                    msg.body = resp.result_str
-                    msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{resp.result_str}"}]
+                    if prompt is None or prompt == "":
+                        msg.body_mime = "text/plain"
+                        msg.body = resp.result_str
+                        msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{resp.result_str}"}]
+                    else:
+                        msg.body_mime = "text/plain"
+                        msg.body = f"{msg.sender} prompt:{prompt}\nasr response:{resp.result_str}"
+                        msg_prompt.messages = [{"role": "user", "content": msg.body}]
             else:
                 msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{msg.body}"}]
             session_topic = msg.target + "#" + msg.topic
@@ -352,14 +358,20 @@ class AIAgent(BaseAIAgent):
                     content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
                     msg_prompt.messages = [{"role": "user", "content": content}]
             elif msg.is_audio_msg():
-                audio_file = msg.body
+                prompt, audio_file = msg.get_audio_body()
                 resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text"))
                 if resp.result_code != ComputeTaskResultCode.OK:
                     error_resp = msg.create_error_resp(resp.error_str)
                     return error_resp
                 else:
-                    msg.body = resp.result_str
-                    msg_prompt.messages = [{"role":"user","content":resp.result_str}]
+                    if prompt is None or prompt == "":
+                        msg.body_mime = "text/plain"
+                        msg.body = resp.result_str
+                        msg_prompt.messages = [{"role":"user","content":resp.result_str}]
+                    else:
+                        msg.body_mime = "text/plain"
+                        msg.body = f"user prompt:{prompt}\nasr response:{resp.result_str}"
+                        msg_prompt.messages = [{"role": "user", "content": msg.body}]
             else:
                 msg_prompt.messages = [{"role":"user","content":msg.body}]
             session_topic = msg.get_sender() + "#" + msg.topic

src/aios/frame/tunnel.py

@@ -34,7 +34,7 @@ class AgentTunnel(ABC):
                     tunnel.connect_to(AIBus.get_default_bus(),tunnel.target_id)
                     await tunnel.start()
                 else:
-                    logger.error(f"load tunnel {tunnel_id} failed") 
+                    logger.error(f"load tunnel {tunnel_id} failed")
             else:
                 logger.error(f"load tunnel {tunnel_id} failed,loader not found")
 
@@ -49,12 +49,12 @@ class AgentTunnel(ABC):
                 await tunnel.start()
                 return True
             else:
-                logger.error(f"load tunnel {tunnel_config['tunnel_id']} failed") 
+                logger.error(f"load tunnel {tunnel_config['tunnel_id']} failed")
         else:
             logger.error(f"load tunnel {tunnel_config['type']} failed,loader not found")
 
         return False
-    
+
     @classmethod
     async def get_tunnel_by_agentid(cls,agent_id:str):
         result = []
@@ -62,14 +62,14 @@ class AgentTunnel(ABC):
             if tunnel.target_id == agent_id:
                 result.append(tunnel)
         return result
-    
+
 
     def __init__(self) -> None:
         super().__init__()
         self.tunnel_id = None
         self.target_id = None
         self.target_type = None
-        self.ai_bus = None
+        self.ai_bus: AIBus = None
         self.is_connected = False
 
     def connect_to(self, ai_bus:AIBus,target_id: str) -> None:
@@ -87,7 +87,7 @@ class AgentTunnel(ABC):
     @abstractmethod
     def post_message(self, msg: AgentMsg) -> None:
         pass
-        
+
 
     @abstractmethod
     async def start(self) -> bool:

src/aios/proto/agent_msg.py

@@ -164,6 +164,15 @@ class AgentMsg:
         body = json.loads(video_body)
         return body.get("prompt"), body.get("video")
 
+    @staticmethod
+    def create_audio_body(audio: str, prompt: str = None):
+        return json.dumps({"audio": audio, "prompt": prompt})
+
+    @staticmethod
+    def parse_audio_body(audio_body: str) -> Tuple[str, str]:
+        body = json.loads(audio_body)
+        return body.get("prompt"), body.get("audio")
+
     def set_image(self, sender: str, target: str, image_format: str, images: [str], prompt: str = None, topic: str = None):
         self.sender = sender
         self.target = target
@@ -210,6 +219,22 @@ class AgentMsg:
             return True
         return False
 
+    def set_audio(self, sender: str, target: str, audio_format: str, audio: str, prompt: str = None, topic: str = None):
+        self.sender = sender
+        self.target = target
+        self.create_time = time.time()
+        self.body_mime = f"audio/{audio_format}"
+        self.body = self.create_audio_body(audio, prompt)
+        if topic:
+            self.topic = topic
+
+    def get_audio_body(self) -> Tuple[str, str]:
+        if self.body_mime is None:
+            return None
+        if self.body_mime.startswith("audio/"):
+            return self.parse_audio_body(self.body)
+        return None
+
     def is_audio_msg(self) -> bool:
         if self.body_mime is None:
             return False