Merge branch 'MVP' into mvp-dev

src/aios/agent/agent.py

@@ -13,7 +13,6 @@ import copy
 import sys
 
 from ..proto.agent_msg import AgentMsg
-from ..proto.compute_task import ComputeTaskResult,ComputeTaskResultCode
 
 from .agent_base import *
 from .chatsession import *
@@ -29,6 +28,7 @@ from ..storage.storage import AIStorage
 
 from ..knowledge import *
 from ..utils import video_utils, image_utils
+from ..proto.compute_task import ComputeTaskResult,ComputeTaskResultCode
 
 logger = logging.getLogger(__name__)
 
@@ -308,6 +308,15 @@ class AIAgent(BaseAIAgent):
                     content = [{"type": "text", "text": f"{msg.sender}:{video_prompt}"}]
                     content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
                     msg_prompt.messages = [{"role": "user", "content": content}]
+            elif msg.is_audio_msg():
+                audio_file = msg.body
+                resp = await ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text")
+                if resp.result_code != ComputeTaskResultCode.OK:
+                    error_resp = msg.create_error_resp(resp.error_str)
+                    return error_resp
+                else:
+                    msg.body = resp.result_str
+                    msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{resp.result_str}"}]
             else:
                 msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{msg.body}"}]
             session_topic = msg.target + "#" + msg.topic
@@ -340,6 +349,15 @@ class AIAgent(BaseAIAgent):
                     content = [{"type": "text", "text": video_prompt}]
                     content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
                     msg_prompt.messages = [{"role": "user", "content": content}]
+            elif msg.is_audio_msg():
+                audio_file = msg.body
+                resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text"))
+                if resp.result_code != ComputeTaskResultCode.OK:
+                    error_resp = msg.create_error_resp(resp.error_str)
+                    return error_resp
+                else:
+                    msg.body = resp.result_str
+                    msg_prompt.messages = [{"role":"user","content":resp.result_str}]
             else:
                 msg_prompt.messages = [{"role":"user","content":msg.body}]
             session_topic = msg.get_sender() + "#" + msg.topic

src/aios/knowledge/object/object_id.py

@@ -17,10 +17,10 @@ class ObjectType(IntEnum):
 
     def is_user_def(self) -> bool:
         return self.value >= 200
-    
+
     def get_user_def_type_code(self):
         return (self.value - 200) if self.is_user_def() else None
-    
+
     @classmethod
     def from_user_def_type_code(cls, value):
         return value + 200
@@ -34,7 +34,7 @@ class ObjectID:  # pylint: disable=too-few-public-methods
 
     def __str__(self):
         return self.to_base58()
-    
+
     def to_base58(self):
         return base58.b58encode(self.value).decode()
 
@@ -57,13 +57,16 @@ class ObjectID:  # pylint: disable=too-few-public-methods
     def new_chunk_id(chunk_hash: HashValue):
         assert len(chunk_hash.value) == 32, "ObjectID must be 32 bytes long"
         return ObjectID(bytes([ObjectType.Chunk]) + chunk_hash.value[1:])
-    
+
     def get_object_type(self) -> ObjectType:
         return ObjectType(self.value[0])
-    
+
     @staticmethod
     def hash_data(data: bytes):
         return ObjectID.new_chunk_id(HashValue.hash_data(data))
-    
+
     def __eq__(self, other) -> bool:
-        return self.value == other.value
+        return self.value == other.value
+
+    def __hash__(self):
+        return hash(self.value)

src/aios/proto/agent_msg.py

@@ -210,6 +210,13 @@ class AgentMsg:
             return True
         return False
 
+    def is_audio_msg(self) -> bool:
+        if self.body_mime is None:
+            return False
+        if self.body_mime.startswith("audio/"):
+            return True
+        return False
+
     def get_msg_id(self) -> str:
         return self.msg_id
 

src/aios/proto/compute_task.py

@@ -75,7 +75,7 @@ class ComputeTask:
         else:
             self.params["model_name"] = "text-embedding-ada-002"
         self.params["input"] = input
-        
+
     def set_image_embedding_params(self, input = Union[ObjectID, bytes], model_name=None, callchain_id = None):
         self.task_type = ComputeTaskType.IMAGE_EMBEDDING
         self.create_time = time.time()
@@ -86,7 +86,7 @@ class ComputeTask:
         else:
             self.params["model_name"] = None
         self.params["input"] = input
-    
+
     def set_text_2_image_params(self, prompt: str, model_name, negative_prompt="", callchain_id=None):
         self.task_type = ComputeTaskType.TEXT_2_IMAGE
         self.create_time = time.time()
@@ -126,15 +126,15 @@ class ComputeTaskResult:
         self.task_id: str = None
         self.callchain_id: str = None
         self.worker_id: str = None
-        self.error_str : str = None 
-        self.result_code: int = 0
+        self.error_str : str = None
+        self.result_code: int = ComputeTaskResultCode.OK
         self.result_str: str = None # easy to use,can read from result
 
         self.result : dict = {}
 
         self.result_refers: dict = {}
         self.pading_data: bytearray = None
-        
+
 
     def set_from_task(self, task: ComputeTask):
         self.task_id = task.task_id