TTS and ASR function implemented based on openai api

src/aios_kernel/__init__.py

@@ -6,6 +6,7 @@ from .compute_kernel import ComputeKernel,ComputeTask,ComputeTaskResult,ComputeT
 from .compute_node import ComputeNode,LocalComputeNode
 from .open_ai_node import OpenAI_ComputeNode
 from .role import AIRole,AIRoleGroup
+from .storage import ResourceLocation,AIStorage,UserConfig,UserConfigItem
 from .workflow import Workflow
 from .bus import AIBus
 from .workflow_env import WorkflowEnvironment,CalenderEnvironment,CalenderEvent,PaintEnvironment
@@ -15,7 +16,6 @@ from .google_text_to_speech_node import GoogleTextToSpeechNode
 from .tunnel import AgentTunnel
 from .tg_tunnel import TelegramTunnel
 from .email_tunnel import EmailTunnel
-from .storage import ResourceLocation,AIStorage,UserConfig,UserConfigItem
 from .contact_manager import ContactManager,Contact,FamilyMember
 from .text_to_speech_function import TextToSpeechFunction
 from .image_2_text_function import Image2TextFunction

src/aios_kernel/asr_function.py

@@ -0,0 +1,55 @@
+import logging
+from typing import Dict
+
+from aios_kernel import ComputeKernel
+from aios_kernel.ai_function import AIFunction
+
+logger = logging.getLogger(__name__)
+
+
+class AsrFunction(AIFunction):
+    def __init__(self):
+        self.func_id = "speech_to_text"
+        self.description = "语音识别，将语音转换为文字"
+
+    def get_name(self) -> str:
+        return self.func_id
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {
+            "type": "object",
+            "properties": {
+                "audio_file": {"type": "string", "description": "音频文件路径"},
+                "model": {"type": "string", "description": "识别模型", "enum": ["openai-whisper"]},
+                "prompt": {"type": "string", "description": "提示语句,可以为None"},
+                "response_format": {"type": "string", "description": "返回格式", "enum": ["text", "json", "srt", "verbose_json", "vtt"]},
+            }
+        }
+
+    async def execute(self, **kwargs) -> str:
+        logger.info(f"execute asr function: {kwargs}")
+
+        audio_file = kwargs.get("audio_file")
+        model = kwargs.get("model")
+        prompt = kwargs.get("prompt")
+        response_format = kwargs.get("response_format")
+        if response_format is None:
+            response_format = "text"
+
+        result = await ComputeKernel.get_instance().do_speech_to_text(audio_file, model, prompt, response_format)
+        if result is not None:
+            return f"exec speech_to_text Ok. {response_format} is\n```\n{result.result_str}\n```"
+        else:
+            return "exec speech_to_text failed"
+
+    def is_local(self) -> bool:
+        return True
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False

src/aios_kernel/compute_kernel.py

@@ -197,7 +197,8 @@ class ComputeKernel:
                        gender: Optional[str] = None,
                        age: Optional[str] = None,
                        voice_name: Optional[str] = None,
-                       tone: Optional[str] = None):
+                       tone: Optional[str] = None,
+                       model_name: Optional[str] = None):
         task_req = ComputeTask()
         task_req.params["text"] = input
         task_req.params["language_code"] = language_code
@@ -205,6 +206,7 @@ class ComputeKernel:
         task_req.params["age"] = age
         task_req.params["voice_name"] = voice_name
         task_req.params["tone"] = tone
+        task_req.params["model_name"] = model_name
         task_req.task_type = ComputeTaskType.TEXT_2_VOICE
         self.run(task_req)
 
@@ -213,6 +215,24 @@ class ComputeKernel:
         if task_req.state == ComputeTaskState.DONE:
             return task_result.result
 
+    async def do_speech_to_text(self,
+                                audio: str,
+                                model: str,
+                                prompt: Optional[str],
+                                response_format: Optional[str]):
+        task_req = ComputeTask()
+        task_req.params["file"] = audio
+        task_req.params["model_name"] = model
+        task_req.params["prompt"] = prompt
+        task_req.params["response_format"] = response_format
+        task_req.task_type = ComputeTaskType.VOICE_2_TEXT
+
+        self.run(task_req)
+
+        task_result = await self._wait_task(task_req)
+
+        if task_req.state == ComputeTaskState.DONE:
+            return task_result
 
     def text_2_image(self, prompt:str, model_name:Optional[str] = None, negative_prompt = None):
         task = ComputeTask()

src/aios_kernel/google_text_to_speech_node.py

@@ -117,9 +117,18 @@ class GoogleTextToSpeechNode(ComputeNode):
     def _run_task(self, task: ComputeTask):
         task.state = ComputeTaskState.RUNNING
         language_code = task.params["language_code"]
+        if language_code is None:
+            language_code = "en"
+
         text = task.params["text"]
         voice_name = task.params["voice_name"]
+        if voice_name is None:
+            voice_name = "default"
+
         gender = task.params["gender"]
+        if gender is None:
+            gender = "female"
+
         age = task.params["age"]
 
         if language_code == "zh":

src/aios_kernel/openai_tts_node.py

@@ -0,0 +1,120 @@
+import asyncio
+import io
+import logging
+import os
+from asyncio import Queue
+
+from openai import AsyncOpenAI
+
+from aios_kernel import ComputeNode, ComputeTask, ComputeTaskState, ComputeTaskResult, ComputeTaskType, AIStorage
+
+logger = logging.getLogger(__name__)
+
+
+class OpenAITTSComputeNode(ComputeNode):
+    _instance = None
+
+    @classmethod
+    def get_instance(cls):
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
+
+    def __init__(self):
+        super().__init__()
+        self.is_start = False
+        self.node_id = "openai_tts_node"
+        self.task_queue = Queue()
+        self.voice_list = {
+            "female": ["nova", "shimmer"],
+            "man": ["alloy", "echo", "fable", "onyx"]
+        }
+        if os.getenv("OPENAI_API_KEY") is not None:
+            self.openai_api_key = os.getenv("OPENAI_API_KEY")
+        else:
+            self.openai_api_key = AIStorage.get_instance().get_user_config().get_value("openai_api_key")
+
+        self.start()
+
+    def start(self):
+        if self.is_start is True:
+            logger.warn("OpenAITTSComputeNode is already start")
+            return
+        self.is_start = True
+
+        async def _run_task_loop():
+            while True:
+                task = await self.task_queue.get()
+                try:
+                    result = await self._run_task(task)
+                    if result is not None:
+                        task.state = ComputeTaskState.DONE
+                        task.result = result
+                except Exception as e:
+                    logger.error(f"openai_tts_node run task error: {e}")
+                    task.state = ComputeTaskState.ERROR
+                    task.result = ComputeTaskResult()
+                    task.result.set_from_task(task)
+                    task.result.worker_id = self.node_id
+                    task.result.result_str = str(e)
+
+        asyncio.create_task(_run_task_loop())
+
+    async def _run_task(self,task: ComputeTask):
+        task.state = ComputeTaskState.RUNNING
+        text = task.params["text"]
+        voice_name = task.params["voice_name"]
+        if voice_name is None:
+            voice_name = "default"
+        gender = task.params["gender"]
+        if gender is None:
+            gender = "female"
+
+        voice_list = self.voice_list[gender]
+        voice = voice_list[hash(voice_name)%len(voice_list)]
+
+        model_name = task.params['model_name']
+        if model_name is None:
+            model_name = 'tts-1'
+
+        client = AsyncOpenAI(api_key=self.openai_api_key)
+
+        response = await client.audio.speech.create(model=model_name, voice=voice, input=text)
+
+        cache = io.BytesIO()
+        async for data in await response.aiter_bytes():
+            cache.write(data)
+
+        cache.seek(0)
+
+        result = ComputeTaskResult()
+        result.set_from_task(task)
+        result.worker_id = self.node_id
+        result.result = cache.read()
+        return result
+
+    async def push_task(self, task: ComputeTask, proiority: int = 0):
+        logger.info(f"openai_tts_node push task: {task.display()}")
+        self.task_queue.put_nowait(task)
+
+    async def remove_task(self, task_id: str):
+        pass
+
+    def get_task_state(self, task_id: str):
+        pass
+
+    def display(self) -> str:
+        return f"OpenAITTSComputeNode: {self.node_id}"
+
+    def get_capacity(self):
+        return 0
+
+    def is_support(self, task: ComputeTask) -> bool:
+        if task.task_type == ComputeTaskType.TEXT_2_VOICE:
+            if task.params['model_name'] is None or task.params['model_name'] == 'tts-1' or task.params['model_name'] == 'tts-1-hd':
+                return True
+        return False
+
+
+    def is_local(self) -> bool:
+        return False

src/aios_kernel/script_to_speech_function.py

@@ -0,0 +1,107 @@
+import io
+import logging
+import os
+import random
+from pathlib import Path
+from typing import Dict
+
+from aios_kernel import ComputeKernel, AIStorage
+from aios_kernel.ai_function import AIFunction
+
+from pydub import AudioSegment
+
+logger = logging.getLogger(__name__)
+
+
+class ScriptToSpeechFunction(AIFunction):
+    def __init__(self):
+        self.func_id = "script_to_speech"
+        self.description = "根据输入的剧本生成音频文件，成功时会返回音频文件路径"
+        self.speech_path = os.path.join(AIStorage.get_instance().get_myai_dir(), "tts")
+        Path(self.speech_path).mkdir(exist_ok=True)
+
+    def get_name(self) -> str:
+        return self.func_id
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {
+            "type": "object",
+            "properties": {
+                "language": {"type": "string", "description": "演播语言", "enum": ["zh", "en"]},
+                "model": {"type": "string", "description": "演播模型", "enum": ["tts-1", "tts-1-hd"]},
+                "roles": {"type": "array", "items": {
+                    "type": "object",
+                    "properties": {
+                        "name": {"type": "string", "description": "角色名字"},
+                        "gender": {"type": "string", "description": "角色性别", "enum": ["man", "female"]},
+                        "age": {"type": "string", "description": "年龄", "enum": ["child", "adult"]},
+                    }}},
+                "lines": {"type": "array", "items": {
+                    "type": "object",
+                    "properties": {
+                        "name": {"type": "string", "description": "角色名字"},
+                        "tone": {"type": "string", "description": "演播情感",
+                                 "enum": ["happy", "sad", "angry", "fear", "disgust", "surprise", "neutral"]},
+                        "text": {"type": "string", "description": "台词"},
+                    }
+                }}
+            }
+        }
+
+    async def execute(self, **kwargs) -> str:
+        logger.info(f"execute text_to_speech function: {kwargs}")
+
+        language = kwargs.get("language")
+        if language is None:
+            language = "zh"
+        model = kwargs.get("model")
+        roles = kwargs.get("roles")
+        lines = kwargs.get("lines")
+
+        audio = None
+        for line in lines:
+            name = line.get("name")
+            tone = line.get("tone")
+            text = line.get("text")
+            gender = None
+            age = None
+            for role in roles:
+                role_name = role.get("name")
+                if role_name == name:
+                    gender = role.get("gender")
+                    age = role.get("age")
+                    break
+            i = 0
+            while i < 3:
+                try:
+                    data = await ComputeKernel.get_instance().do_text_to_speech(text, language, gender, age, name, tone, model_name=model)
+                    if audio is None:
+                        audio = AudioSegment.from_mp3(io.BytesIO(data))
+                    else:
+                        audio = audio + AudioSegment.from_mp3(io.BytesIO(data))
+                    break
+                except Exception as e:
+                    logger.error(f"do_text_to_speech failed: {e}")
+                    i += 1
+                    continue
+
+        if audio is not None:
+            path = os.path.join(self.speech_path, "{}.mp3".format(''.join(random.sample('zyxwvutsrqponmlkjihgfedcba', 10))))
+            audio.export(path, format="mp3")
+            return "exec text_to_speech OK，speech file store at ```{}```".format(path)
+        else:
+            return "exec text_to_speech failed"
+
+    def is_local(self) -> bool:
+        return True
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False
+
+

src/aios_kernel/text_to_speech_function.py

@@ -2,19 +2,23 @@ import io
 import logging
 import os
 import random
+from pathlib import Path
 from typing import Dict
 
-from aios_kernel import ComputeKernel
+from aios_kernel import ComputeKernel, AIStorage
 from aios_kernel.ai_function import AIFunction
 
 from pydub import AudioSegment
 
 logger = logging.getLogger(__name__)
 
+
 class TextToSpeechFunction(AIFunction):
     def __init__(self):
         self.func_id = "text_to_speech"
-        self.description = "根据输入的剧本生成音频文件，成功时会返回音频文件路径"
+        self.description = "根据输入的文本生成音频文件，成功时会返回音频文件路径"
+        self.speech_path = os.path.join(AIStorage.get_instance().get_myai_dir(), "tts")
+        Path(self.speech_path).mkdir(exist_ok=True)
 
     def get_name(self) -> str:
         return self.func_id
@@ -27,22 +31,8 @@ class TextToSpeechFunction(AIFunction):
             "type": "object",
             "properties": {
                 "language": {"type": "string", "description": "演播语言", "enum": ["zh", "en"]},
-                "roles": {"type": "array", "items": {
+                "model": {"type": "string", "description": "演播模型", "enum": ["tts-1", "tts-1-hd"]},
-                    "type": "object",
+                "text": {"type": "string", "description": "文本内容"}
-                    "properties": {
-                        "name": {"type": "string", "description": "角色名字"},
-                        "gender": {"type": "string", "description": "角色性别", "enum": ["man", "female"]},
-                        "age": {"type": "string", "description": "年龄", "enum": ["child", "adult"]},
-                    }}},
-                "lines": {"type": "array", "items": {
-                    "type": "object",
-                    "properties": {
-                        "name": {"type": "string", "description": "角色名字"},
-                        "tone": {"type": "string", "description": "演播情感",
-                                 "enum": ["happy", "sad", "angry", "fear", "disgust", "surprise", "neutral"]},
-                        "text": {"type": "string", "description": "台词"},
-                    }
-                }}
             }
         }
 
@@ -51,41 +41,27 @@ class TextToSpeechFunction(AIFunction):
 
         language = kwargs.get("language")
         if language is None:
-            language = "zh"
+            language = "en"
-        roles = kwargs.get("roles")
+        model = kwargs.get("model")
-        lines = kwargs.get("lines")
+        text = kwargs.get("text")
 
-        audio = None
+        i = 0
-        for line in lines:
+        while i < 3:
-            name = line.get("name")
+            try:
-            tone = line.get("tone")
+                data = await ComputeKernel.get_instance().do_text_to_speech(text, language, None, None, None, None,
-            text = line.get("text")
+                                                                            model_name=model)
-            gender = None
+                if data is not None:
-            age = None
+                    audio = AudioSegment.from_mp3(io.BytesIO(data))
-            for role in roles:
+                break
-                role_name = role.get("name")
+            except Exception as e:
-                if role_name == name:
+                logger.error(f"do_text_to_speech failed: {e}")
-                    gender = role.get("gender")
+                i += 1
-                    age = role.get("age")
+                continue
-                    break
-            i = 0
-            while i < 3:
-                try:
-                    data = await ComputeKernel.get_instance().do_text_to_speech(text, language, gender, age, name, tone)
-                    if audio is None:
-                        audio = AudioSegment.from_mp3(io.BytesIO(data))
-                    else:
-                        audio = audio + AudioSegment.from_mp3(io.BytesIO(data))
-                    break
-                except Exception as e:
-                    logger.error(f"do_text_to_speech failed: {e}")
-                    i += 1
-                    continue
 
         if audio is not None:
-            path = os.path.join(os.path.realpath(os.curdir), "{}.mp3".format(''.join(random.sample('zyxwvutsrqponmlkjihgfedcba', 10))))
+            path = os.path.join(self.speech_path, "{}.mp3".format(''.join(random.sample('zyxwvutsrqponmlkjihgfedcba', 10))))
             audio.export(path, format="mp3")
-            return "exec text_to_speech OK，speech file store at {}".format(path)
+            return "exec text_to_speech OK，speech file store at ```{}```".format(path)
         else:
             return "exec text_to_speech failed"
 

src/aios_kernel/whisper_node.py

@@ -1,55 +1,77 @@
+import io
+import json
 from asyncio import Queue
 import asyncio
 import openai
 import os
 import logging
+import srt
+import webvtt
 
+from openai import AsyncOpenAI
+from openai.cli._progress import BufferReader
+from pydub import AudioSegment
+from datetime import timedelta
+
+from . import AIStorage
 from .compute_node import ComputeNode
 from .compute_task import ComputeTask, ComputeTaskResult, ComputeTaskState, ComputeTaskType
 
 logger = logging.getLogger(__name__)
 
+SECONDS_IN_HOUR = 3600
+SECONDS_IN_MINUTE = 60
+HOURS_IN_DAY = 24
+MICROSECONDS_IN_MILLISECOND = 1000
+
+
+def timedelta_to_vtt_timestamp(timedelta_timestamp):
+    hrs, secs_remainder = divmod(timedelta_timestamp.seconds, SECONDS_IN_HOUR)
+    hrs += timedelta_timestamp.days * HOURS_IN_DAY
+    mins, secs = divmod(secs_remainder, SECONDS_IN_MINUTE)
+    msecs = timedelta_timestamp.microseconds // MICROSECONDS_IN_MILLISECOND
+    return "%02d:%02d:%02d.%03d" % (hrs, mins, secs, msecs)
+
 
 class WhisperComputeNode(ComputeNode):
     _instance = None
 
-    def __new__(cls):
+    @classmethod
+    def get_instance(cls):
         if cls._instance is None:
-            cls._instance = super().__new__(cls)
+            cls._instance = cls()
-            cls._instance.is_start = False
         return cls._instance
 
     def __init__(self) -> None:
         super().__init__()
-        if self.is_start is True:
+        self.is_start = False
-            logger.warn("WhisperComputeNode is already start")
-            return
-
-        self.is_start = True
         self.node_id = "whisper_node"
         self.enable = True
         self.task_queue = Queue()
-        self.open_api_key = None
 
-        if self.open_api_key is None and os.getenv("OPENAI_API_KEY") is not None:
+        if os.getenv("OPENAI_API_KEY") is not None:
-            self.open_api_key = os.getenv("OPENAI_API_KEY")
+            self.openai_api_key = os.getenv("OPENAI_API_KEY")
-
+        else:
-        if self.open_api_key is None:
+            self.openai_api_key = AIStorage.get_instance().get_user_config().get_value("openai_api_key")
-            raise Exception("WhisperComputeNode open_api_key is None")
 
         self.start()
 
     def start(self):
+        if self.is_start is True:
+            logger.warn("WhisperComputeNode is already start")
+            return
+        self.is_start = True
         async def _run_task_loop():
             while True:
                 task = await self.task_queue.get()
                 try:
-                    result = self._run_task(task)
+                    result = await self._run_task(task)
                     if result is not None:
                         task.state = ComputeTaskState.DONE
                         task.result = result
                 except Exception as e:
                     logger.error(f"whisper_node run task error: {e}")
+                    logger.exception(e)
                     task.state = ComputeTaskState.ERROR
                     task.result = ComputeTaskResult()
                     task.result.set_from_task(task)
@@ -58,7 +80,7 @@ class WhisperComputeNode(ComputeNode):
 
         asyncio.create_task(_run_task_loop())
 
-    def _run_task(self, task: ComputeTask):
+    async def _run_task(self, task: ComputeTask):
         task.state = ComputeTaskState.RUNNING
         prompt = task.params["prompt"]
         response_format = None
@@ -72,19 +94,111 @@ class WhisperComputeNode(ComputeNode):
             language = task.params["language"]
         file = task.params["file"]
 
-        resp = openai.Audio.transcribe("whisper-1",
+        client = AsyncOpenAI(api_key=self.openai_api_key)
-                                       file,
+
-                                       self.open_api_key,
+        if os.path.getsize(file) > 25 * 1024 * 1024:
-                                       prompt=prompt,
+            audio = AudioSegment.from_file(file)
-                                       response_format=response_format,
+            text = ""
-                                       temperature=temperature,
+            results = []
-                                       language=language)
+            latest_resp = None
-        result = ComputeTaskResult()
+            step = 30 * 1000
-        result.set_from_task(task)
+            for i in range(0, 60 * 1000, step):
-        result.worker_id = self.node_id
+                chunk = audio[i:i + step]
-        result.result_str = resp["text"]
+                seg_file = io.BytesIO()
-        result.result = resp
+                chunk.export(seg_file, format="mp3")
-        return result
+                seg_file.seek(0)
+
+                resp = await client.audio.transcriptions.create(model="whisper-1",
+                                                   file = ("test.mp3", seg_file),
+                                                   language=language,
+                                                   temperature=temperature,
+                                                   prompt=prompt,
+                                                   response_format=response_format)
+                if response_format == "json":
+                    if text == "":
+                        text = resp.text
+                    else:
+                        text += "," + resp.text
+                elif response_format == "text":
+                    if text == "":
+                        text = resp
+                    else:
+                        text += "," + resp
+                elif response_format == "verbose_json":
+                    if text == "":
+                        text = resp.text
+                    else:
+                        text += "," + resp.text
+                    results.extend(resp.segments)
+                elif response_format == "srt":
+                    srt_list = list(srt.parse(resp))
+                    for item in srt_list:
+                        item.start += timedelta(milliseconds=i)
+                        item.end += timedelta(milliseconds=i)
+                        results.append(item)
+                elif response_format == "vtt":
+                    vtt = webvtt.read_buffer(io.StringIO(resp))
+                    for caption in vtt.captions:
+                        start = timedelta_to_vtt_timestamp(
+                            srt.srt_timestamp_to_timedelta(caption.start) + timedelta(milliseconds=i))
+                        end = timedelta_to_vtt_timestamp(
+                            srt.srt_timestamp_to_timedelta(caption.end) + timedelta(milliseconds=i))
+                        results.append(webvtt.Caption(start, end, caption.text))
+                else:
+                    raise Exception(f"not support response_format: {response_format}")
+
+                latest_resp = resp
+
+            result = ComputeTaskResult()
+            result.set_from_task(task)
+            result.worker_id = self.node_id
+            if response_format == "text":
+                result.result_str = text
+                result.result = text
+            elif response_format == "json":
+                result.result_str = json.dumps({"text": text})
+                resp.text = text
+                result.result = resp
+            elif response_format == "verbose_json":
+                result.result_str = json.dumps({"text": text, "segments": results})
+                latest_resp.text = text
+                latest_resp.segments = results
+                result.result = latest_resp
+            elif response_format == "srt":
+                result.result_str = srt.compose(results)
+                result.result = result.result_str
+            elif response_format == "vtt":
+                vtt = webvtt.WebVTT()
+                vtt.captions.extend(results)
+                f = io.StringIO()
+                vtt.write(f)
+                f.seek(0)
+                result.result_str = f.read()
+                result.result = result.result_str
+            return result
+        else:
+            with open(file, "rb") as file_reader:
+                buffer_reader = BufferReader(file_reader.read(), desc="Upload progress")
+
+            resp = await client.audio.transcriptions.create(model="whisper-1",
+                                               file = (file, buffer_reader),
+                                               language=language,
+                                               temperature=temperature,
+                                               prompt=prompt,
+                                               response_format=response_format)
+            result = ComputeTaskResult()
+            result.set_from_task(task)
+            result.worker_id = self.node_id
+            if response_format == "json":
+                result.result_str = json.dumps({"text": resp.text})
+            elif response_format == "verbose_json":
+                result.result_str = json.dumps({"text": resp.text, "segments": resp.segments})
+            elif response_format == "srt" or response_format == "vtt" or response_format == "text":
+                result.result_str = resp
+            else:
+                raise Exception(f"not support response_format: {response_format}")
+            result.result = resp
+            return result
 
     async def push_task(self, task: ComputeTask, proiority: int = 0):
         logger.info(f"whisper_node push task: {task.display()}")
@@ -102,9 +216,10 @@ class WhisperComputeNode(ComputeNode):
     def get_capacity(self):
         return 0
 
-    def is_support(self, task_type: ComputeTaskType) -> bool:
+    def is_support(self, task: ComputeTask) -> bool:
-        if task_type == ComputeTaskType.VOICE_2_TEXT:
+        if task.task_type == ComputeTaskType.VOICE_2_TEXT:
-            return True
+            if task.params['model_name'] is None or task.params['model_name'] == 'openai-whisper':
+                return True
         return False
 
     def is_local(self) -> bool:

src/aios_kernel/workflow_env.py

@@ -8,7 +8,7 @@ import threading
 import logging
 from typing import Optional
 
-from .text_to_speech_function import TextToSpeechFunction
+from .script_to_speech_function import ScriptToSpeechFunction
 from .image_2_text_function import Image2TextFunction
 from .compute_kernel import ComputeKernel, ComputeTaskResultCode
 from .environment import Environment,EnvironmentEvent
@@ -344,7 +344,7 @@ class WorkflowEnvironment(Environment):
         self.db_file = db_file
         self.local = threading.local()
         self.table_name = "WorkflowEnv_" + env_id
-        self.add_ai_function(TextToSpeechFunction())
+        self.add_ai_function(ScriptToSpeechFunction())
         self.add_ai_function(Image2TextFunction())
 
 

src/requirements.txt

@@ -140,3 +140,5 @@ sentence-transformers==2.2.2
 tiktoken
 markdown
 PyPDF2
+srt==3.5.3
+webvtt-py==0.4.6

src/service/aios_shell/aios_shell.py

@@ -20,6 +20,8 @@ from prompt_toolkit.auto_suggest import AutoSuggestFromHistory
 from prompt_toolkit.completion import WordCompleter
 from prompt_toolkit.styles import Style
 
+from aios_kernel.openai_tts_node import OpenAITTSComputeNode
+
 directory = os.path.dirname(__file__)
 sys.path.append(directory + '/../../')
 
@@ -74,8 +76,8 @@ class AIOS_Shell:
         user_config.add_user_config("shell.current","last opened target and topic",True,"default@Jarvis")
         proxy.declare_user_config()
 
-        google_text_to_speech = GoogleTextToSpeechNode.get_instance()
+        # google_text_to_speech = GoogleTextToSpeechNode.get_instance()
-        google_text_to_speech.declare_user_config()
+        # google_text_to_speech.declare_user_config()
 
         Local_Stability_ComputeNode.declare_user_config()
 
@@ -94,7 +96,7 @@ class AIOS_Shell:
             bus.register_message_handler(target_id,a_workflow._process_msg)
             return True
 
-        a_contact = await ContactManager.get_instance().get_contact(target_id)
+        a_contact = ContactManager.get_instance().find_contact_by_name(target_id)
         if a_contact is not None:
             bus.register_message_handler(target_id,a_contact._process_msg)
             return True
@@ -143,6 +145,12 @@ class AIOS_Shell:
             return False
         ComputeKernel.get_instance().add_compute_node(open_ai_node)
 
+        whisper_node = WhisperComputeNode.get_instance()
+        ComputeKernel.get_instance().add_compute_node(whisper_node);
+
+        openai_tts_node = OpenAITTSComputeNode.get_instance()
+        ComputeKernel.get_instance().add_compute_node(openai_tts_node)
+
         llama_nodes = ComputeNodeConfig.get_instance().initial()
         for llama_node in llama_nodes:
             llama_node.start()
@@ -158,14 +166,14 @@ class AIOS_Shell:
                 await AIStorage.get_instance().set_feature_init_result("llama",False)
 
 
-        if await AIStorage.get_instance().is_feature_enable("aigc"):
+        # if await AIStorage.get_instance().is_feature_enable("aigc"):
-            try:
+            # try:
-                google_text_to_speech_node = GoogleTextToSpeechNode.get_instance()
+            #     google_text_to_speech_node = GoogleTextToSpeechNode.get_instance()
-                google_text_to_speech_node.init()
+            #     google_text_to_speech_node.init()
-                ComputeKernel.get_instance().add_compute_node(google_text_to_speech_node)
+            #     ComputeKernel.get_instance().add_compute_node(google_text_to_speech_node)
-            except Exception as e:
+            # except Exception as e:
-                logger.error(f"google text to speech node initial failed! {e}")
+            #     logger.error(f"google text to speech node initial failed! {e}")
-                await AIStorage.get_instance.set_feature_init_result("aigc",False)
+            #     await AIStorage.get_instance.set_feature_init_result("aigc",False)
 
             # stability_api_node = Stability_ComputeNode()
             # if await stability_api_node.initial() is not True: