Merge pull request #62 from wugren/MVP

Add story maker

rootfs/agents/fairy_tale_writer/agent.toml

@@ -0,0 +1,8 @@
+instance_id = "fairy_tale_writer"
+fullname = "tracy wang"
+llm_model_name = "gpt-3.5-turbo-16k-0613"
+enable_function = []
+
+[[prompt]]
+role = "system"
+content = "你是一个童话做作家，能够写出各种有趣的童话。"

rootfs/agents/manager/agent.toml

@@ -1,5 +1,7 @@
 instance_id = "agent:xxxxxxabcde"
 fullname = "musk"
+enable_function = []
+
 [[prompt]]
 role = "system"
 content = "你有丰富的管理技能，擅长将复杂工作拆解成简单的任务，让团队成员高效协作。"

rootfs/agents/speecher/agent.toml

@@ -0,0 +1,8 @@
+instance_id = "speecher"
+fullname = "tracy wang"
+llm_model_name = "gpt-3.5-turbo-16k-0613"
+enable_function = ["text_to_speech"]
+
+[[prompt]]
+role = "system"
+content = "你是一个故事播音员，可以将故事演播成音频，演播前需要将故事改编成播音剧本，提取旁白和角色台词，以及每个角色需要有性别、年龄、以及每句台词的语气等。如果生成了音频文件则告知你的用户。"

rootfs/workflows/story_maker/workflow.toml

@@ -0,0 +1,42 @@
+name = "story_maker"
+
+
+[filter]
+"*" = "manager"
+
+[roles.manager]
+name = "manager"
+fullname = "总导演"
+agent="manager"
+enable_function = []
+
+[[roles.manager.prompt]]
+role="system"
+content="""
+你是一个语音故事制作总导演,与客户对接并向团队下达指令。你的团队分为下面几个成员:writer,speecher。一个故事制作分成两个阶段：让writer写出故事，再交由speecher演播故事生成音频文件。你的基本工作模式是：
+1. 收到客户的明确的指令后,让writer写出故事
+2. 将writer写出的故事交给speecher演播
+3. 获得音频文件之后，将音频文件的存放路径交给客户
+4. 当你决定要和成员通信时，请使用下面形式输出需要通信的消息
+```
+##/send_msg 成员名称
+内容
+```
+"""
+
+[roles.writer]
+name = "writer"
+agent = "fairy_tale_writer"
+fullname = "作家"
+enable_function = []
+[[roles.writer.prompt]]
+role="system"
+content=""
+
+[roles.speecher]
+name = "speecher"
+agent = "speecher"
+enable_function = ["text_to_speech"]
+[[roles.speecher.prompt]]
+role="system"
+content=""

src/aios_kernel/__init__.py

@@ -19,6 +19,7 @@ from .tg_tunnel import TelegramTunnel
 from .email_tunnel import EmailTunnel
 from .storage import ResourceLocation,AIStorage,UserConfig,UserConfigItem
 from .contact_manager import ContactManager,Contact,FamilyMember  
+from .text_to_speech_function import TextToSpeechFunction
 from .workspace_env import WorkspaceEnvironment
 
 AIOS_Version = "0.5.1, build 2023-9-17"

src/aios_kernel/agent.py

@@ -290,12 +290,13 @@ class AIAgent:
         result_len = 0
         for inner_func in all_inner_function:
             func_name = inner_func.get_name()
-            if self.enable_function_list:
+            if self.enable_function_list is not None:
                 if len(self.enable_function_list) > 0:
                     if func_name not in self.enable_function_list:
                         logger.debug(f"ageint {self.agent_id} ignore inner func:{func_name}")
                         continue
-
+                else:
+                    continue
             this_func = {}
             this_func["name"] = func_name
             this_func["description"] = inner_func.get_description()
@@ -324,6 +325,7 @@ class AIAgent:
             logger.error(f"llm execute inner func:{func_name} error:{e}")
 
 
+        logger.info("llm execute inner func result:" + result_str)
         inner_functions,inner_function_len = self._get_inner_functions()
         prompt.messages.append({"role":"function","content":result_str,"name":func_name})
         task_result:ComputeTaskResult = await ComputeKernel.get_instance().do_llm_completion(prompt,self.llm_model_name,self.max_token_size,inner_functions)

src/aios_kernel/bus.py

@@ -87,7 +87,7 @@ class AIBus:
 
             await asyncio.sleep(0.2)
             retry_times += 1
-            if retry_times > 5*120: # default timeout is 120 sec
+            if retry_times > 5*240: # default timeout is 240 sec
                 msg.status = AgentMsgStatus.ERROR
                 return None
 
@@ -107,12 +107,14 @@ class AIBus:
             # Wait for a message
             message = await handler.queue.get()
 
-            #try:
+            try:
                 # Try to handle the message
                 await handler.handle_message(message)
-            #except Exception as e:
+            except Exception as e:
                 # If an error occurs, put the message back into the queue
-            #    logger.error(f"handle message {message.msg_id} failed! {e}")
+                logger.error(f"handle message {message.msg_id} failed! {e}")
+                logger.exception(e)
+                raise e
                 #self.queues[name].put_nowait(message)
 
         return

src/aios_kernel/chatsession.py

@@ -264,6 +264,7 @@ class AIChatSession:
         return self.owner_id
 
     def read_history(self, number:int=10,offset=0) -> [AgentMsg]:
+        return []
         msgs = self.db.get_messages(self.session_id, number, offset)
         result = []
         for msg in msgs:

src/aios_kernel/compute_kernel.py

@@ -7,7 +7,7 @@ from asyncio import Queue
 
 from .agent import AgentPrompt
 from .compute_node import ComputeNode
-from .compute_task import ComputeTask, ComputeTaskState, ComputeTaskResult
+from .compute_task import ComputeTask, ComputeTaskState, ComputeTaskResult, ComputeTaskType
 
 logger = logging.getLogger(__name__)
 
@@ -73,7 +73,7 @@ class ComputeKernel:
         hit_pos = random.randint(0, total_weights - 1)
         for i in range(min(len(support_nodes) - 1, hit_pos), -1, -1):
             if support_nodes[i]["pos"] <= hit_pos:
-                return node
+                return support_nodes[i]["node"]
 
         logger.warning(
             f"task {task.display()} is not support by any compute node")
@@ -118,7 +118,7 @@ class ComputeKernel:
                 if task_req.state == ComputeTaskState.ERROR:
                     break
 
-                if check_times >= 20:
+                if check_times >= 120:
                     task_req.state = ComputeTaskState.ERROR
                     break
 
@@ -129,7 +129,7 @@ class ComputeKernel:
         if task_req.state == ComputeTaskState.DONE:
             return task_req.result
 
-        return "error!"
+        raise Exception("error!")
 
     
     def text_embedding(self,input:str,model_name:Optional[str] = None):
@@ -162,4 +162,38 @@ class ComputeKernel:
 
         return "error!"
 
+    async def do_text_to_speech(self,
+                       input:str,
+                       language_code:Optional[str] = None,
+                       gender: Optional[str] = None,
+                       age: Optional[str] = None,
+                       voice_name: Optional[str] = None,
+                       tone: Optional[str] = None):
+        task_req = ComputeTask()
+        task_req.params["text"] = input
+        task_req.params["language_code"] = language_code
+        task_req.params["gender"] = gender
+        task_req.params["age"] = age
+        task_req.params["voice_name"] = voice_name
+        task_req.params["tone"] = tone
+        task_req.task_type = ComputeTaskType.TEXT_2_VOICE
+        self.run(task_req)
+
+        check_times = 0
+        while True:
+            if task_req.state == ComputeTaskState.DONE:
+                break
+            if task_req.state == ComputeTaskState.ERROR:
+                break
+            if check_times >= 60:
+                task_req.state = ComputeTaskState.ERROR
+                break
+            await asyncio.sleep(0.5)
+            check_times += 1
+
+        if task_req.state == ComputeTaskState.DONE:
+            return task_req.result.result
+        else:
+            raise Exception("do_text_to_speech failed!")
+
 

src/aios_kernel/google_text_to_speech_node.py

@@ -3,9 +3,11 @@ import os
 import asyncio
 from asyncio import Queue
 import logging
+from typing import Optional
 
 from google.cloud import texttospeech
 
+from .storage import AIStorage
 from .compute_task import ComputeTask, ComputeTaskResult, ComputeTaskState, ComputeTaskType
 from .compute_node import ComputeNode
 
@@ -21,26 +23,78 @@ see:https://cloud.google.com/text-to-speech/docs/before-you-begin
 class GoogleTextToSpeechNode(ComputeNode):
     _instance = None
 
-    def __new__(cls, *args, **kwargs):
+    @classmethod
+    def get_instance(cls):
         if cls._instance is None:
-            cls._instance = super(GoogleTextToSpeechNode, cls).__new__(cls)
+            cls._instance = cls()
-            cls._instance.is_start = False
         return cls._instance
 
     def __init__(self):
         super().__init__()
-        if self.is_start is True:
-            logger.warn("GoogleTextToSpeechNode is already start")
-            return
-
-        self.is_start = True
         self.node_id = "google_text_to_speech_node"
         self.task_queue = Queue()
+        self.client: Optional[texttospeech.TextToSpeechClient] = None
 
-        self.client = texttospeech.TextToSpeechClient()
+        self.language_list = {
-
+            "cnm-CN": {
+                "female": ["cmn-CN-Standard-A",
+                           "cmn-CN-Standard-D",
+                           "cmn-CN-Wavenet-A",
+                           "cmn-CN-Wavenet-D",
+                           "cmn-TW-Standard-A",
+                           "cmn-TW-Wavenet-A"],
+                "man": ["cmn-CN-Standard-B",
+                        "cmn-CN-Standard-C",
+                        "cmn-CN-Wavenet-B",
+                        "cmn-CN-Wavenet-C",
+                        "cmn-TW-Standard-B",
+                        "cmn-TW-Standard-C",
+                        "cmn-TW-Wavenet-B",
+                        "cmn-TW-Wavenet-C"]
+            },
+            "en-US": {
+                "female": ["en-US-Neural2-C",
+                           "en-US-Neural2-E",
+                           "en-US-Neural2-F",
+                           "en-US-Neural2-G",
+                           "en-US-Neural2-H",
+                           "en-US-News-K",
+                           "en-US-News-L",
+                           "en-US-Standard-C",
+                           "en-US-Standard-E",
+                           "en-US-Standard-F",
+                           "en-US-Standard-G",
+                           "en-US-Standard-H",
+                           "en-US-Studio-O",
+                           "en-US-Wavenet-C",
+                           "en-US-Wavenet-E",
+                           "en-US-Wavenet-F",
+                           "en-US-Wavenet-G",
+                           "en-US-Wavenet-H"],
+                "man": ["en-US-Polyglot-1",
+                        "en-US-Standard-A",
+                        "en-US-Standard-B",
+                        "en-US-Standard-D",
+                        "en-US-Standard-I",
+                        "en-US-Standard-J",
+                        "en-US-Studio-M",
+                        "en-US-Wavenet-A",
+                        "en-US-Wavenet-B",
+                        "en-US-Wavenet-D",
+                        "en-US-Wavenet-I",
+                        "en-US-Wavenet-J"]
+            }
+        }
         self.start()
 
+    def init(self):
+        user_config = AIStorage.get_instance().get_user_config()
+        google_application_credentials = user_config.get_value("google_application_credentials")
+        if google_application_credentials is None:
+            raise Exception("google_application_credentials is None!")
+        os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = google_application_credentials
+        self.client = texttospeech.TextToSpeechClient()
+
     def start(self):
         async def _run_task_loop():
             while True:
@@ -64,10 +118,24 @@ class GoogleTextToSpeechNode(ComputeNode):
         task.state = ComputeTaskState.RUNNING
         language_code = task.params["language_code"]
         text = task.params["text"]
+        voice_name = task.params["voice_name"]
+        gender = task.params["gender"]
+        age = task.params["age"]
+
+        if language_code == "zh":
+            language_code = "cnm-CN"
+        elif language_code == "en":
+            language_code = "en-US"
+        else:
+            raise Exception(f"language_code {language_code} not support")
+
+        lang_list = self.language_list[language_code][gender]
+        voice = lang_list[hash(voice_name) % len(lang_list)]
 
         synthesis_input = texttospeech.SynthesisInput(text=text)
         voice = texttospeech.VoiceSelectionParams(language_code=language_code,
-                                                  ssml_gender=texttospeech.SsmlVoiceGender.NEUTRAL)
+                                                  ssml_gender=texttospeech.SsmlVoiceGender.NEUTRAL,
+                                                  name=voice)
 
         audio_config = texttospeech.AudioConfig(audio_encoding=texttospeech.AudioEncoding.MP3)
 
@@ -95,10 +163,18 @@ class GoogleTextToSpeechNode(ComputeNode):
     def get_capacity(self):
         return 0
 
-    def is_support(self, task_type: ComputeTaskType) -> bool:
+    def is_support(self, task: ComputeTask) -> bool:
-        if task_type == ComputeTaskType.TEXT_2_VOICE:
+        if task.task_type == ComputeTaskType.TEXT_2_VOICE:
             return True
         return False
 
     def is_local(self) -> bool:
         return False
+
+    def declare_user_config(self):
+        if os.getenv("GOOGLE_APPLICATION_CREDENTIALS") is None:
+            user_config = AIStorage.get_instance().get_user_config()
+            user_config.add_user_config("google_application_credentials",
+                                        "google application credentials, please visit:https://cloud.google.com/text-to-speech/docs/before-you-begin",
+                                        False,
+                                        None)

src/aios_kernel/open_ai_node.py

@@ -100,16 +100,16 @@ class OpenAI_ComputeNode(ComputeNode):
                 if max_token_size is None:
                     max_token_size = 4000
 
-                result_token = int(max_token_size * 0.4)
+                result_token = max_token_size
-                    
-                logger.info(f"call openai {mode_name} prompts: {prompts}")
 
                 if llm_inner_functions is None:
+                    logger.info(f"call openai {mode_name} prompts: {prompts}")
                     resp = openai.ChatCompletion.create(model=mode_name,
                                                     messages=prompts,
                                                     max_tokens=result_token,
                                                     temperature=0.7)
                 else:
+                    logger.info(f"call openai {mode_name} prompts: {prompts} functions: {json.dumps(llm_inner_functions)}")
                     resp = openai.ChatCompletion.create(model=mode_name,
                                                         messages=prompts,
                                                         functions=llm_inner_functions,
@@ -139,6 +139,7 @@ class OpenAI_ComputeNode(ComputeNode):
                 result.result_message = resp["choices"][0]["message"]
                 if token_usage:
                     result.result_refers["token_usage"] = token_usage
+                logger.info(f"openai success response: {result.result_str}")
                 return result
             case _:
                 task.state = ComputeTaskState.ERROR

src/aios_kernel/text_to_speech_function.py

@@ -0,0 +1,101 @@
+import io
+import logging
+import os
+import random
+from typing import Dict
+
+from aios_kernel import ComputeKernel
+from aios_kernel.ai_function import AIFunction
+
+from pydub import AudioSegment
+
+logger = logging.getLogger(__name__)
+
+class TextToSpeechFunction(AIFunction):
+    def __init__(self):
+        self.func_id = "text_to_speech"
+        self.description = "根据输入的剧本生成音频文件，成功时会返回音频文件路径"
+
+    def get_name(self) -> str:
+        return self.func_id
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {
+            "type": "object",
+            "properties": {
+                "language": {"type": "string", "description": "演播语言", "enum": ["zh", "en"]},
+                "roles": {"type": "array", "items": {
+                    "type": "object",
+                    "properties": {
+                        "name": {"type": "string", "description": "角色名字"},
+                        "gender": {"type": "string", "description": "角色性别", "enum": ["man", "female"]},
+                        "age": {"type": "string", "description": "年龄", "enum": ["child", "adult"]},
+                    }}},
+                "lines": {"type": "array", "items": {
+                    "type": "object",
+                    "properties": {
+                        "name": {"type": "string", "description": "角色名字"},
+                        "tone": {"type": "string", "description": "演播情感",
+                                 "enum": ["happy", "sad", "angry", "fear", "disgust", "surprise", "neutral"]},
+                        "text": {"type": "string", "description": "台词"},
+                    }
+                }}
+            }
+        }
+
+    async def execute(self, **kwargs) -> str:
+        logger.info(f"execute text_to_speech function: {kwargs}")
+
+        language = kwargs.get("language")
+        if language is None:
+            language = "zh"
+        roles = kwargs.get("roles")
+        lines = kwargs.get("lines")
+
+        audio = None
+        for line in lines:
+            name = line.get("name")
+            tone = line.get("tone")
+            text = line.get("text")
+            gender = None
+            age = None
+            for role in roles:
+                role_name = role.get("name")
+                if role_name == name:
+                    gender = role.get("gender")
+                    age = role.get("age")
+                    break
+            i = 0
+            while i < 3:
+                try:
+                    data = await ComputeKernel.get_instance().do_text_to_speech(text, language, gender, age, name, tone)
+                    if audio is None:
+                        audio = AudioSegment.from_mp3(io.BytesIO(data))
+                    else:
+                        audio = audio + AudioSegment.from_mp3(io.BytesIO(data))
+                    break
+                except Exception as e:
+                    logger.error(f"do_text_to_speech failed: {e}")
+                    i += 1
+                    continue
+
+        if audio is not None:
+            path = os.path.join(os.path.realpath(os.curdir), "{}.mp3".format(''.join(random.sample('zyxwvutsrqponmlkjihgfedcba', 10))))
+            audio.export(path, format="mp3")
+            return "exec text_to_speech OK，speech file store at {}".format(path)
+        else:
+            return "exec text_to_speech failed"
+
+    def is_local(self) -> bool:
+        return True
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False
+
+

src/aios_kernel/workflow.py

@@ -376,10 +376,12 @@ class Workflow:
         result_func = []
         for inner_func in all_inner_function:
             func_name = inner_func.get_name()
-            if the_role.enable_function_list:
+            if the_role.enable_function_list is not None:
                 if len(the_role.enable_function_list) > 0:
                     if func_name not in the_role.enable_function_list:
-                        logger.debug(f"ageint {self.agent_id} ignore inner func:{func_name}")
+                        logger.debug(f"agent {self.agent_id} ignore inner func:{func_name}")
+                        continue
+                else:
                     continue
             this_func = {}
             this_func["name"] = func_name
@@ -404,7 +406,7 @@ class Workflow:
 
         result_str:str = await func_node.execute(**arguments)
 
-        inner_functions = self._get_inner_functions()
+        inner_functions = self._get_inner_functions(the_role)
         prompt.messages.append({"role":"function","content":result_str,"name":func_name})
         task_result:ComputeTaskResult = await ComputeKernel.get_instance().do_llm_completion(prompt,
                                                                                 the_role.agent.llm_model_name,the_role.agent.max_token_size,

src/aios_kernel/workflow_env.py

@@ -7,6 +7,8 @@ from sqlite3 import Error
 import threading
 import logging
 from typing import Optional
+
+from .text_to_speech_function import TextToSpeechFunction
 from .environment import Environment,EnvironmentEvent
 from .ai_function import SimpleAIFunction
 from .storage import AIStorage
@@ -252,6 +254,7 @@ class WorkflowEnvironment(Environment):
         self.db_file = db_file
         self.local = threading.local()
         self.table_name = "WorkflowEnv_" + env_id
+        self.add_ai_function(TextToSpeechFunction())
 
 
     def _get_conn(self):

src/service/aios_shell/aios_shell.py

@@ -59,6 +59,9 @@ class AIOS_Shell:
         user_config.add_user_config("shell.current","last opened target and topic",True,"default@Jarvis")
         proxy.declare_user_config()
 
+        google_text_to_speech = GoogleTextToSpeechNode.get_instance()
+        google_text_to_speech.declare_user_config()
+
 
     async def _handle_no_target_msg(self,bus:AIBus,msg:AgentMsg) -> bool:
         target_id = msg.target.split(".")[0]
@@ -99,6 +102,14 @@ class AIOS_Shell:
             return False
         ComputeKernel.get_instance().add_compute_node(open_ai_node)
 
+        try:
+            google_text_to_speech_node = GoogleTextToSpeechNode.get_instance()
+            google_text_to_speech_node.init()
+            ComputeKernel.get_instance().add_compute_node(google_text_to_speech_node)
+        except Exception as e:
+            logger.error(f"google text to speech node initial failed! {e}")
+            return False
+
         llama_ai_node = LocalLlama_ComputeNode()
         await llama_ai_node.start()
         # ComputeKernel.get_instance().add_compute_node(llama_ai_node)