story maker

rootfs/agents/fairy_tale_writer/agent.toml

@@ -0,0 +1,7 @@
+instance_id = "fairy_tale_writer"
+fullname = "tracy wang"
+llm_model_name = "gpt-3.5-turbo-16k-0613"
+
+[[prompt]]
+role = "system"
+content = "你是一个童话做作家，能够写出各种有趣的童话。"

rootfs/agents/studio_director/agent.toml

@@ -0,0 +1,7 @@
+instance_id = "studio_director"
+fullname = "tracy wang"
+llm_model_name = "gpt-3.5-turbo-16k-0613"
+
+[[prompt]]
+role = "system"
+content = "你是一个演播导演，请将下面故事改编成朗读剧本，提取旁白和角色台词，每个角色需要有性别、年龄、以及每句台词的语气。并调用text_to_speech function生成音频数据。"

rootfs/workflows/story_maker/workflow.toml

@@ -0,0 +1,37 @@
+name = "story_maker"
+
+
+[filter]
+"*" = "manager"
+
+[roles.manager]
+name = "manager"
+fullname = "总导演"
+agent="manager"
+[[roles.manager.prompt]]
+role="system"
+content="""
+你是一个语音故事制作总导演,与客户对接并向团队下达指令。你的团队分为下面几个成员:writer,studio_director。一个故事制作分成两个阶段：让writer写出故事，再交由studio_director演播故事。你的基本工作模式是：
+1. 收到客户的明确的指令后,让writer写出故事
+2. 将writer写出的故事交给studio_director演播
+3. 当你决定要和成员通信时，请使用下面形式输出需要通信的消息
+```
+##/send_msg 成员名称
+内容
+```
+"""
+
+[roles.writer]
+name = "writer"
+agent = "fairy_tale_writer"
+fullname = "作家"
+[[roles.writer.prompt]]
+role="system"
+content=""
+
+[roles.studio_director]
+name = "studio_director"
+agent = "studio_director"
+[[roles.studio_director.prompt]]
+role="system"
+content=""

src/aios_kernel/__init__.py

@@ -19,6 +19,7 @@ from .tg_tunnel import TelegramTunnel
 from .email_tunnel import EmailTunnel
 from .storage import ResourceLocation,AIStorage,UserConfig,UserConfigItem
 from .contact_manager import ContactManager,Contact,FamilyMember  
+from .text_to_speech_function import TextToSpeechFunction
 
 AIOS_Version = "0.5.1, build 2023-9-17"
 

src/aios_kernel/chatsession.py

@@ -264,6 +264,7 @@ class AIChatSession:
         return self.owner_id
 
     def read_history(self, number:int=10,offset=0) -> [AgentMsg]:
+        return []
         msgs = self.db.get_messages(self.session_id, number, offset)
         result = []
         for msg in msgs:

src/aios_kernel/compute_kernel.py

@@ -7,7 +7,7 @@ from asyncio import Queue
 
 from .agent import AgentPrompt
 from .compute_node import ComputeNode
-from .compute_task import ComputeTask, ComputeTaskState, ComputeTaskResult
+from .compute_task import ComputeTask, ComputeTaskState, ComputeTaskResult, ComputeTaskType
 
 logger = logging.getLogger(__name__)
 
@@ -73,7 +73,7 @@ class ComputeKernel:
         hit_pos = random.randint(0, total_weights - 1)
         for i in range(min(len(support_nodes) - 1, hit_pos), -1, -1):
             if support_nodes[i]["pos"] <= hit_pos:
-                return node
+                return support_nodes[i]["node"]
 
         logger.warning(
             f"task {task.display()} is not support by any compute node")
@@ -162,4 +162,38 @@ class ComputeKernel:
 
         return "error!"
 
+    async def do_text_to_speech(self,
+                       input:str,
+                       language_code:Optional[str] = None,
+                       gender: Optional[str] = None,
+                       age: Optional[str] = None,
+                       voice_name: Optional[str] = None,
+                       tone: Optional[str] = None):
+        task_req = ComputeTask()
+        task_req.params["text"] = input
+        task_req.params["language_code"] = language_code
+        task_req.params["gender"] = gender
+        task_req.params["age"] = age
+        task_req.params["voice_name"] = voice_name
+        task_req.params["tone"] = tone
+        task_req.task_type = ComputeTaskType.TEXT_2_VOICE
+        self.run(task_req)
+
+        check_times = 0
+        while True:
+            if task_req.state == ComputeTaskState.DONE:
+                break
+            if task_req.state == ComputeTaskState.ERROR:
+                break
+            if check_times >= 60:
+                task_req.state = ComputeTaskState.ERROR
+                break
+            await asyncio.sleep(0.5)
+            check_times += 1
+
+        if task_req.state == ComputeTaskState.DONE:
+            return task_req.result.result
+        else:
+            raise Exception("do_text_to_speech failed!")
+
 

src/aios_kernel/google_text_to_speech_node.py

@@ -21,24 +21,69 @@ see:https://cloud.google.com/text-to-speech/docs/before-you-begin
 class GoogleTextToSpeechNode(ComputeNode):
     _instance = None
 
-    def __new__(cls, *args, **kwargs):
+    @classmethod
+    def get_instance(cls):
         if cls._instance is None:
-            cls._instance = super(GoogleTextToSpeechNode, cls).__new__(cls)
+            cls._instance = cls()
-            cls._instance.is_start = False
         return cls._instance
 
     def __init__(self):
         super().__init__()
-        if self.is_start is True:
-            logger.warn("GoogleTextToSpeechNode is already start")
-            return
-
-        self.is_start = True
         self.node_id = "google_text_to_speech_node"
         self.task_queue = Queue()
 
         self.client = texttospeech.TextToSpeechClient()
 
+        self.language_list = {
+            "cnm-CN": {
+                "female": ["cmn-CN-Standard-A",
+                           "cmn-CN-Standard-D",
+                           "cmn-CN-Wavenet-A",
+                           "cmn-CN-Wavenet-D",
+                           "cmn-TW-Standard-A",
+                           "cmn-TW-Wavenet-A"],
+                "man": ["cmn-CN-Standard-B",
+                        "cmn-CN-Standard-C",
+                        "cmn-CN-Wavenet-B",
+                        "cmn-CN-Wavenet-C",
+                        "cmn-TW-Standard-B",
+                        "cmn-TW-Standard-C",
+                        "cmn-TW-Wavenet-B",
+                        "cmn-TW-Wavenet-C"]
+            },
+            "en-US": {
+                "female": ["en-US-Neural2-C",
+                           "en-US-Neural2-E",
+                           "en-US-Neural2-F",
+                           "en-US-Neural2-G",
+                           "en-US-Neural2-H",
+                           "en-US-News-K",
+                           "en-US-News-L",
+                           "en-US-Standard-C",
+                           "en-US-Standard-E",
+                           "en-US-Standard-F",
+                           "en-US-Standard-G",
+                           "en-US-Standard-H",
+                           "en-US-Studio-O",
+                           "en-US-Wavenet-C",
+                           "en-US-Wavenet-E",
+                           "en-US-Wavenet-F",
+                           "en-US-Wavenet-G",
+                           "en-US-Wavenet-H"],
+                "man": ["en-US-Polyglot-1",
+                        "en-US-Standard-A",
+                        "en-US-Standard-B",
+                        "en-US-Standard-D",
+                        "en-US-Standard-I",
+                        "en-US-Standard-J",
+                        "en-US-Studio-M",
+                        "en-US-Wavenet-A",
+                        "en-US-Wavenet-B",
+                        "en-US-Wavenet-D",
+                        "en-US-Wavenet-I",
+                        "en-US-Wavenet-J"]
+            }
+        }
         self.start()
 
     def start(self):
@@ -64,10 +109,24 @@ class GoogleTextToSpeechNode(ComputeNode):
         task.state = ComputeTaskState.RUNNING
         language_code = task.params["language_code"]
         text = task.params["text"]
+        voice_name = task.params["voice_name"]
+        gender = task.params["gender"]
+        age = task.params["age"]
+
+        if language_code == "zh":
+            language_code = "cnm-CN"
+        elif language_code == "en":
+            language_code = "en-US"
+        else:
+            raise Exception(f"language_code {language_code} not support")
+
+        lang_list = self.language_list[language_code][gender]
+        voice = lang_list[hash(voice_name) % len(lang_list)]
 
         synthesis_input = texttospeech.SynthesisInput(text=text)
         voice = texttospeech.VoiceSelectionParams(language_code=language_code,
-                                                  ssml_gender=texttospeech.SsmlVoiceGender.NEUTRAL)
+                                                  ssml_gender=texttospeech.SsmlVoiceGender.NEUTRAL,
+                                                  name=voice)
 
         audio_config = texttospeech.AudioConfig(audio_encoding=texttospeech.AudioEncoding.MP3)
 
@@ -95,8 +154,8 @@ class GoogleTextToSpeechNode(ComputeNode):
     def get_capacity(self):
         return 0
 
-    def is_support(self, task_type: ComputeTaskType) -> bool:
+    def is_support(self, task: ComputeTask) -> bool:
-        if task_type == ComputeTaskType.TEXT_2_VOICE:
+        if task.task_type == ComputeTaskType.TEXT_2_VOICE:
             return True
         return False
 

src/aios_kernel/text_to_speech_function.py

@@ -0,0 +1,101 @@
+import io
+import logging
+import os
+import random
+from typing import Dict
+
+from aios_kernel import ComputeKernel
+from aios_kernel.ai_function import AIFunction
+
+from pydub import AudioSegment
+
+logger = logging.getLogger(__name__)
+
+class TextToSpeechFunction(AIFunction):
+    def __init__(self):
+        self.func_id = "text_to_speech"
+        self.description = "根据输入的剧本生成音频数据"
+
+    def get_name(self) -> str:
+        return self.func_id
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {
+            "type": "object",
+            "properties": {
+                "language": {"type": "string", "description": "演播语言", "enum": ["zh", "en"]},
+                "roles": {"type": "array", "items": {
+                    "type": "object",
+                    "properties": {
+                        "name": {"type": "string", "description": "角色名字"},
+                        "gender": {"type": "string", "description": "角色性别", "enum": ["man", "female"]},
+                        "age": {"type": "string", "description": "年龄", "enum": ["child", "adult"]},
+                    }}},
+                "lines": {"type": "array", "items": {
+                    "type": "object",
+                    "properties": {
+                        "name": {"type": "string", "description": "角色名字"},
+                        "tone": {"type": "string", "description": "演播情感",
+                                 "enum": ["happy", "sad", "angry", "fear", "disgust", "surprise", "neutral"]},
+                        "text": {"type": "string", "description": "台词"},
+                    }
+                }}
+            }
+        }
+
+    async def execute(self, **kwargs) -> str:
+        logger.info(f"execute text_to_speech function: {kwargs}")
+
+        language = kwargs.get("language")
+        if language is None:
+            language = "zh"
+        roles = kwargs.get("roles")
+        lines = kwargs.get("lines")
+
+        audio = None
+        for line in lines:
+            name = line.get("name")
+            tone = line.get("tone")
+            text = line.get("text")
+            gender = None
+            age = None
+            for role in roles:
+                role_name = role.get("name")
+                if role_name == name:
+                    gender = role.get("gender")
+                    age = role.get("age")
+                    break
+            i = 0
+            while i < 3:
+                try:
+                    data = await ComputeKernel.get_instance().do_text_to_speech(text, language, gender, age, name, tone)
+                    if audio is None:
+                        audio = AudioSegment.from_mp3(io.BytesIO(data))
+                    else:
+                        audio = audio + AudioSegment.from_mp3(io.BytesIO(data))
+                    break
+                except Exception as e:
+                    logger.error(f"do_text_to_speech failed: {e}")
+                    i += 1
+                    continue
+
+        if audio is not None:
+            path = os.path.join(os.curdir, "{}.mp3".format(random.sample('zyxwvutsrqponmlkjihgfedcba', 10)))
+            audio.export(path, format="mp3")
+            return "complete.file path:{}".format(path)
+        else:
+            return "failed"
+
+    def is_local(self) -> bool:
+        return True
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False
+
+

src/aios_kernel/workflow_env.py

@@ -7,6 +7,8 @@ from sqlite3 import Error
 import threading
 import logging
 from typing import Optional
+
+from .text_to_speech_function import TextToSpeechFunction
 from .environment import Environment,EnvironmentEvent
 from .ai_function import SimpleAIFunction
 from .storage import AIStorage
@@ -252,6 +254,7 @@ class WorkflowEnvironment(Environment):
         self.db_file = db_file
         self.local = threading.local()
         self.table_name = "WorkflowEnv_" + env_id
+        self.add_ai_function(TextToSpeechFunction())
 
 
     def _get_conn(self):

src/service/aios_shell/aios_shell.py

@@ -97,6 +97,9 @@ class AIOS_Shell:
             return False
         ComputeKernel.get_instance().add_compute_node(open_ai_node)
 
+        google_text_to_speech_node = GoogleTextToSpeechNode.get_instance()
+        ComputeKernel.get_instance().add_compute_node(google_text_to_speech_node)
+
         llama_ai_node = LocalLlama_ComputeNode()
         await llama_ai_node.start()
         # ComputeKernel.get_instance().add_compute_node(llama_ai_node)