Merge pull request #62 from wugren/MVP

Add story maker
2023-09-22 00:48:15 -07:00
parent 0d556003c5 3061e00800
commit add4357c24
15 changed files with 521 additions and 227 deletions
@@ -0,0 +1,8 @@
+instance_id = "fairy_tale_writer"
+fullname = "tracy wang"
+llm_model_name = "gpt-3.5-turbo-16k-0613"
+enable_function = []
+
+[[prompt]]
+role = "system"
+content = "你是一个童话做作家，能够写出各种有趣的童话。"
@@ -1,5 +1,7 @@
 instance_id = "agent:xxxxxxabcde"
 fullname = "musk"
+enable_function = []
+
 [[prompt]]
 role = "system"
 content = "你有丰富的管理技能，擅长将复杂工作拆解成简单的任务，让团队成员高效协作。"
@@ -0,0 +1,8 @@
+instance_id = "speecher"
+fullname = "tracy wang"
+llm_model_name = "gpt-3.5-turbo-16k-0613"
+enable_function = ["text_to_speech"]
+
+[[prompt]]
+role = "system"
+content = "你是一个故事播音员，可以将故事演播成音频，演播前需要将故事改编成播音剧本，提取旁白和角色台词，以及每个角色需要有性别、年龄、以及每句台词的语气等。如果生成了音频文件则告知你的用户。"
@@ -0,0 +1,42 @@
+name = "story_maker"
+
+
+[filter]
+"*" = "manager"
+
+[roles.manager]
+name = "manager"
+fullname = "总导演"
+agent="manager"
+enable_function = []
+
+[[roles.manager.prompt]]
+role="system"
+content="""
+你是一个语音故事制作总导演,与客户对接并向团队下达指令。你的团队分为下面几个成员:writer,speecher。一个故事制作分成两个阶段：让writer写出故事，再交由speecher演播故事生成音频文件。你的基本工作模式是：
+1. 收到客户的明确的指令后,让writer写出故事
+2. 将writer写出的故事交给speecher演播
+3. 获得音频文件之后，将音频文件的存放路径交给客户
+4. 当你决定要和成员通信时，请使用下面形式输出需要通信的消息
+```
+##/send_msg 成员名称
+内容
+```
+"""
+
+[roles.writer]
+name = "writer"
+agent = "fairy_tale_writer"
+fullname = "作家"
+enable_function = []
+[[roles.writer.prompt]]
+role="system"
+content=""
+
+[roles.speecher]
+name = "speecher"
+agent = "speecher"
+enable_function = ["text_to_speech"]
+[[roles.speecher.prompt]]
+role="system"
+content=""
@@ -19,6 +19,7 @@ from .tg_tunnel import TelegramTunnel
 from .email_tunnel import EmailTunnel
 from .storage import ResourceLocation,AIStorage,UserConfig,UserConfigItem
 from .contact_manager import ContactManager,Contact,FamilyMember  
+from .text_to_speech_function import TextToSpeechFunction
 from .workspace_env import WorkspaceEnvironment

 AIOS_Version = "0.5.1, build 2023-9-17"
@@ -290,12 +290,13 @@ class AIAgent:
        result_len = 0
        for inner_func in all_inner_function:
            func_name = inner_func.get_name()
-            if self.enable_function_list:
+            if self.enable_function_list is not None:
                if len(self.enable_function_list) > 0:
                    if func_name not in self.enable_function_list:
                        logger.debug(f"ageint {self.agent_id} ignore inner func:{func_name}")
                        continue
-
+                else:
+                    continue
            this_func = {}
            this_func["name"] = func_name
            this_func["description"] = inner_func.get_description()
@@ -324,6 +325,7 @@ class AIAgent:
            logger.error(f"llm execute inner func:{func_name} error:{e}")


+        logger.info("llm execute inner func result:" + result_str)
        inner_functions,inner_function_len = self._get_inner_functions()
        prompt.messages.append({"role":"function","content":result_str,"name":func_name})
        task_result:ComputeTaskResult = await ComputeKernel.get_instance().do_llm_completion(prompt,self.llm_model_name,self.max_token_size,inner_functions)
@@ -87,7 +87,7 @@ class AIBus:

            await asyncio.sleep(0.2)
            retry_times += 1
-            if retry_times > 5*120: # default timeout is 120 sec
+            if retry_times > 5*240: # default timeout is 240 sec
                msg.status = AgentMsgStatus.ERROR
                return None

@@ -107,12 +107,14 @@ class AIBus:
            # Wait for a message
            message = await handler.queue.get()

-            #try:
+            try:
                # Try to handle the message
                await handler.handle_message(message)
-            #except Exception as e:
+            except Exception as e:
                # If an error occurs, put the message back into the queue
-            #    logger.error(f"handle message {message.msg_id} failed! {e}")
+                logger.error(f"handle message {message.msg_id} failed! {e}")
+                logger.exception(e)
+                raise e
                #self.queues[name].put_nowait(message)

        return
@@ -264,6 +264,7 @@ class AIChatSession:
        return self.owner_id

    def read_history(self, number:int=10,offset=0) -> [AgentMsg]:
+        return []
        msgs = self.db.get_messages(self.session_id, number, offset)
        result = []
        for msg in msgs:
@@ -7,7 +7,7 @@ from asyncio import Queue

 from .agent import AgentPrompt
 from .compute_node import ComputeNode
-from .compute_task import ComputeTask, ComputeTaskState, ComputeTaskResult
+from .compute_task import ComputeTask, ComputeTaskState, ComputeTaskResult, ComputeTaskType

 logger = logging.getLogger(__name__)

@@ -73,7 +73,7 @@ class ComputeKernel:
        hit_pos = random.randint(0, total_weights - 1)
        for i in range(min(len(support_nodes) - 1, hit_pos), -1, -1):
            if support_nodes[i]["pos"] <= hit_pos:
-                return node
+                return support_nodes[i]["node"]

        logger.warning(
            f"task {task.display()} is not support by any compute node")
@@ -118,7 +118,7 @@ class ComputeKernel:
                if task_req.state == ComputeTaskState.ERROR:
                    break

-                if check_times >= 20:
+                if check_times >= 120:
                    task_req.state = ComputeTaskState.ERROR
                    break

@@ -129,7 +129,7 @@ class ComputeKernel:
        if task_req.state == ComputeTaskState.DONE:
            return task_req.result

-        return "error!"
+        raise Exception("error!")

    
    def text_embedding(self,input:str,model_name:Optional[str] = None):
@@ -162,4 +162,38 @@ class ComputeKernel:

        return "error!"

+    async def do_text_to_speech(self,
+                       input:str,
+                       language_code:Optional[str] = None,
+                       gender: Optional[str] = None,
+                       age: Optional[str] = None,
+                       voice_name: Optional[str] = None,
+                       tone: Optional[str] = None):
+        task_req = ComputeTask()
+        task_req.params["text"] = input
+        task_req.params["language_code"] = language_code
+        task_req.params["gender"] = gender
+        task_req.params["age"] = age
+        task_req.params["voice_name"] = voice_name
+        task_req.params["tone"] = tone
+        task_req.task_type = ComputeTaskType.TEXT_2_VOICE
+        self.run(task_req)
+
+        check_times = 0
+        while True:
+            if task_req.state == ComputeTaskState.DONE:
+                break
+            if task_req.state == ComputeTaskState.ERROR:
+                break
+            if check_times >= 60:
+                task_req.state = ComputeTaskState.ERROR
+                break
+            await asyncio.sleep(0.5)
+            check_times += 1
+
+        if task_req.state == ComputeTaskState.DONE:
+            return task_req.result.result
+        else:
+            raise Exception("do_text_to_speech failed!")
+

@@ -3,9 +3,11 @@ import os
 import asyncio
 from asyncio import Queue
 import logging
+from typing import Optional

 from google.cloud import texttospeech

+from .storage import AIStorage
 from .compute_task import ComputeTask, ComputeTaskResult, ComputeTaskState, ComputeTaskType
 from .compute_node import ComputeNode

@@ -21,26 +23,78 @@ see:https://cloud.google.com/text-to-speech/docs/before-you-begin
 class GoogleTextToSpeechNode(ComputeNode):
    _instance = None

-    def __new__(cls, *args, **kwargs):
+    @classmethod
+    def get_instance(cls):
        if cls._instance is None:
-            cls._instance = super(GoogleTextToSpeechNode, cls).__new__(cls)
-            cls._instance.is_start = False
+            cls._instance = cls()
        return cls._instance

    def __init__(self):
        super().__init__()
-        if self.is_start is True:
-            logger.warn("GoogleTextToSpeechNode is already start")
-            return
-
-        self.is_start = True
        self.node_id = "google_text_to_speech_node"
        self.task_queue = Queue()
+        self.client: Optional[texttospeech.TextToSpeechClient] = None

-        self.client = texttospeech.TextToSpeechClient()
-
+        self.language_list = {
+            "cnm-CN": {
+                "female": ["cmn-CN-Standard-A",
+                           "cmn-CN-Standard-D",
+                           "cmn-CN-Wavenet-A",
+                           "cmn-CN-Wavenet-D",
+                           "cmn-TW-Standard-A",
+                           "cmn-TW-Wavenet-A"],
+                "man": ["cmn-CN-Standard-B",
+                        "cmn-CN-Standard-C",
+                        "cmn-CN-Wavenet-B",
+                        "cmn-CN-Wavenet-C",
+                        "cmn-TW-Standard-B",
+                        "cmn-TW-Standard-C",
+                        "cmn-TW-Wavenet-B",
+                        "cmn-TW-Wavenet-C"]
+            },
+            "en-US": {
+                "female": ["en-US-Neural2-C",
+                           "en-US-Neural2-E",
+                           "en-US-Neural2-F",
+                           "en-US-Neural2-G",
+                           "en-US-Neural2-H",
+                           "en-US-News-K",
+                           "en-US-News-L",
+                           "en-US-Standard-C",
+                           "en-US-Standard-E",
+                           "en-US-Standard-F",
+                           "en-US-Standard-G",
+                           "en-US-Standard-H",
+                           "en-US-Studio-O",
+                           "en-US-Wavenet-C",
+                           "en-US-Wavenet-E",
+                           "en-US-Wavenet-F",
+                           "en-US-Wavenet-G",
+                           "en-US-Wavenet-H"],
+                "man": ["en-US-Polyglot-1",
+                        "en-US-Standard-A",
+                        "en-US-Standard-B",
+                        "en-US-Standard-D",
+                        "en-US-Standard-I",
+                        "en-US-Standard-J",
+                        "en-US-Studio-M",
+                        "en-US-Wavenet-A",
+                        "en-US-Wavenet-B",
+                        "en-US-Wavenet-D",
+                        "en-US-Wavenet-I",
+                        "en-US-Wavenet-J"]
+            }
+        }
        self.start()

+    def init(self):
+        user_config = AIStorage.get_instance().get_user_config()
+        google_application_credentials = user_config.get_value("google_application_credentials")
+        if google_application_credentials is None:
+            raise Exception("google_application_credentials is None!")
+        os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = google_application_credentials
+        self.client = texttospeech.TextToSpeechClient()
+
    def start(self):
        async def _run_task_loop():
            while True:
@@ -64,10 +118,24 @@ class GoogleTextToSpeechNode(ComputeNode):
        task.state = ComputeTaskState.RUNNING
        language_code = task.params["language_code"]
        text = task.params["text"]
+        voice_name = task.params["voice_name"]
+        gender = task.params["gender"]
+        age = task.params["age"]
+
+        if language_code == "zh":
+            language_code = "cnm-CN"
+        elif language_code == "en":
+            language_code = "en-US"
+        else:
+            raise Exception(f"language_code {language_code} not support")
+
+        lang_list = self.language_list[language_code][gender]
+        voice = lang_list[hash(voice_name) % len(lang_list)]

        synthesis_input = texttospeech.SynthesisInput(text=text)
        voice = texttospeech.VoiceSelectionParams(language_code=language_code,
-                                                  ssml_gender=texttospeech.SsmlVoiceGender.NEUTRAL)
+                                                  ssml_gender=texttospeech.SsmlVoiceGender.NEUTRAL,
+                                                  name=voice)

        audio_config = texttospeech.AudioConfig(audio_encoding=texttospeech.AudioEncoding.MP3)

@@ -95,10 +163,18 @@ class GoogleTextToSpeechNode(ComputeNode):
    def get_capacity(self):
        return 0

-    def is_support(self, task_type: ComputeTaskType) -> bool:
-        if task_type == ComputeTaskType.TEXT_2_VOICE:
+    def is_support(self, task: ComputeTask) -> bool:
+        if task.task_type == ComputeTaskType.TEXT_2_VOICE:
            return True
        return False

    def is_local(self) -> bool:
        return False
+
+    def declare_user_config(self):
+        if os.getenv("GOOGLE_APPLICATION_CREDENTIALS") is None:
+            user_config = AIStorage.get_instance().get_user_config()
+            user_config.add_user_config("google_application_credentials",
+                                        "google application credentials, please visit:https://cloud.google.com/text-to-speech/docs/before-you-begin",
+                                        False,
+                                        None)
@@ -100,16 +100,16 @@ class OpenAI_ComputeNode(ComputeNode):
                if max_token_size is None:
                    max_token_size = 4000

-                result_token = int(max_token_size * 0.4)
-                    
-                logger.info(f"call openai {mode_name} prompts: {prompts}")
+                result_token = max_token_size

                if llm_inner_functions is None:
+                    logger.info(f"call openai {mode_name} prompts: {prompts}")
                    resp = openai.ChatCompletion.create(model=mode_name,
                                                    messages=prompts,
                                                    max_tokens=result_token,
                                                    temperature=0.7)
                else:
+                    logger.info(f"call openai {mode_name} prompts: {prompts} functions: {json.dumps(llm_inner_functions)}")
                    resp = openai.ChatCompletion.create(model=mode_name,
                                                        messages=prompts,
                                                        functions=llm_inner_functions,
@@ -139,6 +139,7 @@ class OpenAI_ComputeNode(ComputeNode):
                result.result_message = resp["choices"][0]["message"]
                if token_usage:
                    result.result_refers["token_usage"] = token_usage
+                logger.info(f"openai success response: {result.result_str}")
                return result
            case _:
                task.state = ComputeTaskState.ERROR
@@ -0,0 +1,101 @@
+import io
+import logging
+import os
+import random
+from typing import Dict
+
+from aios_kernel import ComputeKernel
+from aios_kernel.ai_function import AIFunction
+
+from pydub import AudioSegment
+
+logger = logging.getLogger(__name__)
+
+class TextToSpeechFunction(AIFunction):
+    def __init__(self):
+        self.func_id = "text_to_speech"
+        self.description = "根据输入的剧本生成音频文件，成功时会返回音频文件路径"
+
+    def get_name(self) -> str:
+        return self.func_id
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {
+            "type": "object",
+            "properties": {
+                "language": {"type": "string", "description": "演播语言", "enum": ["zh", "en"]},
+                "roles": {"type": "array", "items": {
+                    "type": "object",
+                    "properties": {
+                        "name": {"type": "string", "description": "角色名字"},
+                        "gender": {"type": "string", "description": "角色性别", "enum": ["man", "female"]},
+                        "age": {"type": "string", "description": "年龄", "enum": ["child", "adult"]},
+                    }}},
+                "lines": {"type": "array", "items": {
+                    "type": "object",
+                    "properties": {
+                        "name": {"type": "string", "description": "角色名字"},
+                        "tone": {"type": "string", "description": "演播情感",
+                                 "enum": ["happy", "sad", "angry", "fear", "disgust", "surprise", "neutral"]},
+                        "text": {"type": "string", "description": "台词"},
+                    }
+                }}
+            }
+        }
+
+    async def execute(self, **kwargs) -> str:
+        logger.info(f"execute text_to_speech function: {kwargs}")
+
+        language = kwargs.get("language")
+        if language is None:
+            language = "zh"
+        roles = kwargs.get("roles")
+        lines = kwargs.get("lines")
+
+        audio = None
+        for line in lines:
+            name = line.get("name")
+            tone = line.get("tone")
+            text = line.get("text")
+            gender = None
+            age = None
+            for role in roles:
+                role_name = role.get("name")
+                if role_name == name:
+                    gender = role.get("gender")
+                    age = role.get("age")
+                    break
+            i = 0
+            while i < 3:
+                try:
+                    data = await ComputeKernel.get_instance().do_text_to_speech(text, language, gender, age, name, tone)
+                    if audio is None:
+                        audio = AudioSegment.from_mp3(io.BytesIO(data))
+                    else:
+                        audio = audio + AudioSegment.from_mp3(io.BytesIO(data))
+                    break
+                except Exception as e:
+                    logger.error(f"do_text_to_speech failed: {e}")
+                    i += 1
+                    continue
+
+        if audio is not None:
+            path = os.path.join(os.path.realpath(os.curdir), "{}.mp3".format(''.join(random.sample('zyxwvutsrqponmlkjihgfedcba', 10))))
+            audio.export(path, format="mp3")
+            return "exec text_to_speech OK，speech file store at {}".format(path)
+        else:
+            return "exec text_to_speech failed"
+
+    def is_local(self) -> bool:
+        return True
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False
+
+
@@ -376,10 +376,12 @@ class Workflow:
        result_func = []
        for inner_func in all_inner_function:
            func_name = inner_func.get_name()
-            if the_role.enable_function_list:
+            if the_role.enable_function_list is not None:
                if len(the_role.enable_function_list) > 0:
                    if func_name not in the_role.enable_function_list:
-                        logger.debug(f"ageint {self.agent_id} ignore inner func:{func_name}")
+                        logger.debug(f"agent {self.agent_id} ignore inner func:{func_name}")
+                        continue
+                else:
                    continue
            this_func = {}
            this_func["name"] = func_name
@@ -404,7 +406,7 @@ class Workflow:

        result_str:str = await func_node.execute(**arguments)

-        inner_functions = self._get_inner_functions()
+        inner_functions = self._get_inner_functions(the_role)
        prompt.messages.append({"role":"function","content":result_str,"name":func_name})
        task_result:ComputeTaskResult = await ComputeKernel.get_instance().do_llm_completion(prompt,
                                                                                the_role.agent.llm_model_name,the_role.agent.max_token_size,
@@ -7,6 +7,8 @@ from sqlite3 import Error
 import threading
 import logging
 from typing import Optional
+
+from .text_to_speech_function import TextToSpeechFunction
 from .environment import Environment,EnvironmentEvent
 from .ai_function import SimpleAIFunction
 from .storage import AIStorage
@@ -252,6 +254,7 @@ class WorkflowEnvironment(Environment):
        self.db_file = db_file
        self.local = threading.local()
        self.table_name = "WorkflowEnv_" + env_id
+        self.add_ai_function(TextToSpeechFunction())


    def _get_conn(self):
@@ -59,6 +59,9 @@ class AIOS_Shell:
        user_config.add_user_config("shell.current","last opened target and topic",True,"default@Jarvis")
        proxy.declare_user_config()

+        google_text_to_speech = GoogleTextToSpeechNode.get_instance()
+        google_text_to_speech.declare_user_config()
+

    async def _handle_no_target_msg(self,bus:AIBus,msg:AgentMsg) -> bool:
        target_id = msg.target.split(".")[0]
@@ -99,6 +102,14 @@ class AIOS_Shell:
            return False
        ComputeKernel.get_instance().add_compute_node(open_ai_node)

+        try:
+            google_text_to_speech_node = GoogleTextToSpeechNode.get_instance()
+            google_text_to_speech_node.init()
+            ComputeKernel.get_instance().add_compute_node(google_text_to_speech_node)
+        except Exception as e:
+            logger.error(f"google text to speech node initial failed! {e}")
+            return False
+
        llama_ai_node = LocalLlama_ComputeNode()
        await llama_ai_node.start()
        # ComputeKernel.get_instance().add_compute_node(llama_ai_node)