From b305a963baa2607cc843940078793eaa17d4050b Mon Sep 17 00:00:00 2001 From: wugren Date: Wed, 22 Nov 2023 19:00:41 +0800 Subject: [PATCH 1/2] TTS and ASR function implemented based on openai api --- src/aios_kernel/__init__.py | 4 +- src/aios_kernel/asr_function.py | 55 ++++++ src/aios_kernel/compute_kernel.py | 32 +++- src/aios_kernel/google_text_to_speech_node.py | 9 + src/aios_kernel/openai_tts_node.py | 120 ++++++++++++ src/aios_kernel/script_to_speech_function.py | 107 +++++++++++ src/aios_kernel/text_to_speech_function.py | 74 +++----- src/aios_kernel/whisper_node.py | 179 ++++++++++++++---- src/aios_kernel/workflow_env.py | 28 +-- src/requirements.txt | 4 +- src/service/aios_shell/aios_shell.py | 78 ++++---- 11 files changed, 551 insertions(+), 139 deletions(-) create mode 100644 src/aios_kernel/asr_function.py create mode 100644 src/aios_kernel/openai_tts_node.py create mode 100644 src/aios_kernel/script_to_speech_function.py diff --git a/src/aios_kernel/__init__.py b/src/aios_kernel/__init__.py index 187ee0b..ea00748 100644 --- a/src/aios_kernel/__init__.py +++ b/src/aios_kernel/__init__.py @@ -6,6 +6,7 @@ from .compute_kernel import ComputeKernel,ComputeTask,ComputeTaskResult,ComputeT from .compute_node import ComputeNode,LocalComputeNode from .open_ai_node import OpenAI_ComputeNode from .role import AIRole,AIRoleGroup +from .storage import ResourceLocation,AIStorage,UserConfig,UserConfigItem from .workflow import Workflow from .bus import AIBus from .workflow_env import WorkflowEnvironment,CalenderEnvironment,CalenderEvent,PaintEnvironment @@ -15,8 +16,7 @@ from .google_text_to_speech_node import GoogleTextToSpeechNode from .tunnel import AgentTunnel from .tg_tunnel import TelegramTunnel from .email_tunnel import EmailTunnel -from .storage import ResourceLocation,AIStorage,UserConfig,UserConfigItem -from .contact_manager import ContactManager,Contact,FamilyMember +from .contact_manager import ContactManager,Contact,FamilyMember from .text_to_speech_function import TextToSpeechFunction from .image_2_text_function import Image2TextFunction from .workspace_env import ShellEnvironment diff --git a/src/aios_kernel/asr_function.py b/src/aios_kernel/asr_function.py new file mode 100644 index 0000000..11acde1 --- /dev/null +++ b/src/aios_kernel/asr_function.py @@ -0,0 +1,55 @@ +import logging +from typing import Dict + +from aios_kernel import ComputeKernel +from aios_kernel.ai_function import AIFunction + +logger = logging.getLogger(__name__) + + +class AsrFunction(AIFunction): + def __init__(self): + self.func_id = "speech_to_text" + self.description = "语音识别，将语音转换为文字" + + def get_name(self) -> str: + return self.func_id + + def get_description(self) -> str: + return self.description + + def get_parameters(self) -> Dict: + return { + "type": "object", + "properties": { + "audio_file": {"type": "string", "description": "音频文件路径"}, + "model": {"type": "string", "description": "识别模型", "enum": ["openai-whisper"]}, + "prompt": {"type": "string", "description": "提示语句,可以为None"}, + "response_format": {"type": "string", "description": "返回格式", "enum": ["text", "json", "srt", "verbose_json", "vtt"]}, + } + } + + async def execute(self, **kwargs) -> str: + logger.info(f"execute asr function: {kwargs}") + + audio_file = kwargs.get("audio_file") + model = kwargs.get("model") + prompt = kwargs.get("prompt") + response_format = kwargs.get("response_format") + if response_format is None: + response_format = "text" + + result = await ComputeKernel.get_instance().do_speech_to_text(audio_file, model, prompt, response_format) + if result is not None: + return f"exec speech_to_text Ok. {response_format} is\n```\n{result.result_str}\n```" + else: + return "exec speech_to_text failed" + + def is_local(self) -> bool: + return True + + def is_in_zone(self) -> bool: + return True + + def is_ready_only(self) -> bool: + return False diff --git a/src/aios_kernel/compute_kernel.py b/src/aios_kernel/compute_kernel.py index 6174fa8..1c6219a 100644 --- a/src/aios_kernel/compute_kernel.py +++ b/src/aios_kernel/compute_kernel.py @@ -77,7 +77,7 @@ class ComputeKernel: if len(support_nodes) < 1: logger.warning(f"task {task.display()} is not support by any compute node") return None - + # hit a random node with weight hit_pos = random.randint(0, total_weights - 1) for i in range(min(len(support_nodes) - 1, hit_pos), -1, -1): @@ -126,8 +126,8 @@ class ComputeKernel: task_req.set_llm_params(prompt,resp_mode,mode_name, max_token,inner_functions) self.run(task_req) return task_req - - async def _wait_task(self,task_req:ComputeTask, timeout=60)->ComputeTaskResult: + + async def _wait_task(self,task_req:ComputeTask, timeout=60)->ComputeTaskResult: async def check_timer(): check_times = 0 while True: @@ -136,7 +136,7 @@ class ComputeKernel: if task_req.state == ComputeTaskState.ERROR: break - + if timeout is not None and check_times >= timeout*2: task_req.state = ComputeTaskState.ERROR break @@ -181,7 +181,7 @@ class ComputeKernel: task_req.set_image_embedding_params(input,model_name) self.run(task_req) return task_req - + async def do_image_embedding(self,input:ObjectID,model_name:Optional[str] = None) -> [float]: task_req = self.image_embedding(input,model_name) task_result = await self._wait_task(task_req) @@ -197,7 +197,8 @@ class ComputeKernel: gender: Optional[str] = None, age: Optional[str] = None, voice_name: Optional[str] = None, - tone: Optional[str] = None): + tone: Optional[str] = None, + model_name: Optional[str] = None): task_req = ComputeTask() task_req.params["text"] = input task_req.params["language_code"] = language_code @@ -205,6 +206,7 @@ class ComputeKernel: task_req.params["age"] = age task_req.params["voice_name"] = voice_name task_req.params["tone"] = tone + task_req.params["model_name"] = model_name task_req.task_type = ComputeTaskType.TEXT_2_VOICE self.run(task_req) @@ -213,6 +215,24 @@ class ComputeKernel: if task_req.state == ComputeTaskState.DONE: return task_result.result + async def do_speech_to_text(self, + audio: str, + model: str, + prompt: Optional[str], + response_format: Optional[str]): + task_req = ComputeTask() + task_req.params["file"] = audio + task_req.params["model_name"] = model + task_req.params["prompt"] = prompt + task_req.params["response_format"] = response_format + task_req.task_type = ComputeTaskType.VOICE_2_TEXT + + self.run(task_req) + + task_result = await self._wait_task(task_req) + + if task_req.state == ComputeTaskState.DONE: + return task_result def text_2_image(self, prompt:str, model_name:Optional[str] = None, negative_prompt = None): task = ComputeTask() diff --git a/src/aios_kernel/google_text_to_speech_node.py b/src/aios_kernel/google_text_to_speech_node.py index 0fdba47..ccf9034 100644 --- a/src/aios_kernel/google_text_to_speech_node.py +++ b/src/aios_kernel/google_text_to_speech_node.py @@ -117,9 +117,18 @@ class GoogleTextToSpeechNode(ComputeNode): def _run_task(self, task: ComputeTask): task.state = ComputeTaskState.RUNNING language_code = task.params["language_code"] + if language_code is None: + language_code = "en" + text = task.params["text"] voice_name = task.params["voice_name"] + if voice_name is None: + voice_name = "default" + gender = task.params["gender"] + if gender is None: + gender = "female" + age = task.params["age"] if language_code == "zh": diff --git a/src/aios_kernel/openai_tts_node.py b/src/aios_kernel/openai_tts_node.py new file mode 100644 index 0000000..68ee064 --- /dev/null +++ b/src/aios_kernel/openai_tts_node.py @@ -0,0 +1,120 @@ +import asyncio +import io +import logging +import os +from asyncio import Queue + +from openai import AsyncOpenAI + +from aios_kernel import ComputeNode, ComputeTask, ComputeTaskState, ComputeTaskResult, ComputeTaskType, AIStorage + +logger = logging.getLogger(__name__) + + +class OpenAITTSComputeNode(ComputeNode): + _instance = None + + @classmethod + def get_instance(cls): + if cls._instance is None: + cls._instance = cls() + return cls._instance + + def __init__(self): + super().__init__() + self.is_start = False + self.node_id = "openai_tts_node" + self.task_queue = Queue() + self.voice_list = { + "female": ["nova", "shimmer"], + "man": ["alloy", "echo", "fable", "onyx"] + } + if os.getenv("OPENAI_API_KEY") is not None: + self.openai_api_key = os.getenv("OPENAI_API_KEY") + else: + self.openai_api_key = AIStorage.get_instance().get_user_config().get_value("openai_api_key") + + self.start() + + def start(self): + if self.is_start is True: + logger.warn("OpenAITTSComputeNode is already start") + return + self.is_start = True + + async def _run_task_loop(): + while True: + task = await self.task_queue.get() + try: + result = await self._run_task(task) + if result is not None: + task.state = ComputeTaskState.DONE + task.result = result + except Exception as e: + logger.error(f"openai_tts_node run task error: {e}") + task.state = ComputeTaskState.ERROR + task.result = ComputeTaskResult() + task.result.set_from_task(task) + task.result.worker_id = self.node_id + task.result.result_str = str(e) + + asyncio.create_task(_run_task_loop()) + + async def _run_task(self,task: ComputeTask): + task.state = ComputeTaskState.RUNNING + text = task.params["text"] + voice_name = task.params["voice_name"] + if voice_name is None: + voice_name = "default" + gender = task.params["gender"] + if gender is None: + gender = "female" + + voice_list = self.voice_list[gender] + voice = voice_list[hash(voice_name)%len(voice_list)] + + model_name = task.params['model_name'] + if model_name is None: + model_name = 'tts-1' + + client = AsyncOpenAI(api_key=self.openai_api_key) + + response = await client.audio.speech.create(model=model_name, voice=voice, input=text) + + cache = io.BytesIO() + async for data in await response.aiter_bytes(): + cache.write(data) + + cache.seek(0) + + result = ComputeTaskResult() + result.set_from_task(task) + result.worker_id = self.node_id + result.result = cache.read() + return result + + async def push_task(self, task: ComputeTask, proiority: int = 0): + logger.info(f"openai_tts_node push task: {task.display()}") + self.task_queue.put_nowait(task) + + async def remove_task(self, task_id: str): + pass + + def get_task_state(self, task_id: str): + pass + + def display(self) -> str: + return f"OpenAITTSComputeNode: {self.node_id}" + + def get_capacity(self): + return 0 + + def is_support(self, task: ComputeTask) -> bool: + if task.task_type == ComputeTaskType.TEXT_2_VOICE: + if task.params['model_name'] is None or task.params['model_name'] == 'tts-1' or task.params['model_name'] == 'tts-1-hd': + return True + return False + + + def is_local(self) -> bool: + return False diff --git a/src/aios_kernel/script_to_speech_function.py b/src/aios_kernel/script_to_speech_function.py new file mode 100644 index 0000000..7ccd116 --- /dev/null +++ b/src/aios_kernel/script_to_speech_function.py @@ -0,0 +1,107 @@ +import io +import logging +import os +import random +from pathlib import Path +from typing import Dict + +from aios_kernel import ComputeKernel, AIStorage +from aios_kernel.ai_function import AIFunction + +from pydub import AudioSegment + +logger = logging.getLogger(__name__) + + +class ScriptToSpeechFunction(AIFunction): + def __init__(self): + self.func_id = "script_to_speech" + self.description = "根据输入的剧本生成音频文件，成功时会返回音频文件路径" + self.speech_path = os.path.join(AIStorage.get_instance().get_myai_dir(), "tts") + Path(self.speech_path).mkdir(exist_ok=True) + + def get_name(self) -> str: + return self.func_id + + def get_description(self) -> str: + return self.description + + def get_parameters(self) -> Dict: + return { + "type": "object", + "properties": { + "language": {"type": "string", "description": "演播语言", "enum": ["zh", "en"]}, + "model": {"type": "string", "description": "演播模型", "enum": ["tts-1", "tts-1-hd"]}, + "roles": {"type": "array", "items": { + "type": "object", + "properties": { + "name": {"type": "string", "description": "角色名字"}, + "gender": {"type": "string", "description": "角色性别", "enum": ["man", "female"]}, + "age": {"type": "string", "description": "年龄", "enum": ["child", "adult"]}, + }}}, + "lines": {"type": "array", "items": { + "type": "object", + "properties": { + "name": {"type": "string", "description": "角色名字"}, + "tone": {"type": "string", "description": "演播情感", + "enum": ["happy", "sad", "angry", "fear", "disgust", "surprise", "neutral"]}, + "text": {"type": "string", "description": "台词"}, + } + }} + } + } + + async def execute(self, **kwargs) -> str: + logger.info(f"execute text_to_speech function: {kwargs}") + + language = kwargs.get("language") + if language is None: + language = "zh" + model = kwargs.get("model") + roles = kwargs.get("roles") + lines = kwargs.get("lines") + + audio = None + for line in lines: + name = line.get("name") + tone = line.get("tone") + text = line.get("text") + gender = None + age = None + for role in roles: + role_name = role.get("name") + if role_name == name: + gender = role.get("gender") + age = role.get("age") + break + i = 0 + while i < 3: + try: + data = await ComputeKernel.get_instance().do_text_to_speech(text, language, gender, age, name, tone, model_name=model) + if audio is None: + audio = AudioSegment.from_mp3(io.BytesIO(data)) + else: + audio = audio + AudioSegment.from_mp3(io.BytesIO(data)) + break + except Exception as e: + logger.error(f"do_text_to_speech failed: {e}") + i += 1 + continue + + if audio is not None: + path = os.path.join(self.speech_path, "{}.mp3".format(''.join(random.sample('zyxwvutsrqponmlkjihgfedcba', 10)))) + audio.export(path, format="mp3") + return "exec text_to_speech OK，speech file store at ```{}```".format(path) + else: + return "exec text_to_speech failed" + + def is_local(self) -> bool: + return True + + def is_in_zone(self) -> bool: + return True + + def is_ready_only(self) -> bool: + return False + + diff --git a/src/aios_kernel/text_to_speech_function.py b/src/aios_kernel/text_to_speech_function.py index 90dfd49..0f90ee1 100644 --- a/src/aios_kernel/text_to_speech_function.py +++ b/src/aios_kernel/text_to_speech_function.py @@ -2,19 +2,23 @@ import io import logging import os import random +from pathlib import Path from typing import Dict -from aios_kernel import ComputeKernel +from aios_kernel import ComputeKernel, AIStorage from aios_kernel.ai_function import AIFunction from pydub import AudioSegment logger = logging.getLogger(__name__) + class TextToSpeechFunction(AIFunction): def __init__(self): self.func_id = "text_to_speech" - self.description = "根据输入的剧本生成音频文件，成功时会返回音频文件路径" + self.description = "根据输入的文本生成音频文件，成功时会返回音频文件路径" + self.speech_path = os.path.join(AIStorage.get_instance().get_myai_dir(), "tts") + Path(self.speech_path).mkdir(exist_ok=True) def get_name(self) -> str: return self.func_id @@ -27,22 +31,8 @@ class TextToSpeechFunction(AIFunction): "type": "object", "properties": { "language": {"type": "string", "description": "演播语言", "enum": ["zh", "en"]}, - "roles": {"type": "array", "items": { - "type": "object", - "properties": { - "name": {"type": "string", "description": "角色名字"}, - "gender": {"type": "string", "description": "角色性别", "enum": ["man", "female"]}, - "age": {"type": "string", "description": "年龄", "enum": ["child", "adult"]}, - }}}, - "lines": {"type": "array", "items": { - "type": "object", - "properties": { - "name": {"type": "string", "description": "角色名字"}, - "tone": {"type": "string", "description": "演播情感", - "enum": ["happy", "sad", "angry", "fear", "disgust", "surprise", "neutral"]}, - "text": {"type": "string", "description": "台词"}, - } - }} + "model": {"type": "string", "description": "演播模型", "enum": ["tts-1", "tts-1-hd"]}, + "text": {"type": "string", "description": "文本内容"} } } @@ -51,41 +41,27 @@ class TextToSpeechFunction(AIFunction): language = kwargs.get("language") if language is None: - language = "zh" - roles = kwargs.get("roles") - lines = kwargs.get("lines") + language = "en" + model = kwargs.get("model") + text = kwargs.get("text") - audio = None - for line in lines: - name = line.get("name") - tone = line.get("tone") - text = line.get("text") - gender = None - age = None - for role in roles: - role_name = role.get("name") - if role_name == name: - gender = role.get("gender") - age = role.get("age") - break - i = 0 - while i < 3: - try: - data = await ComputeKernel.get_instance().do_text_to_speech(text, language, gender, age, name, tone) - if audio is None: - audio = AudioSegment.from_mp3(io.BytesIO(data)) - else: - audio = audio + AudioSegment.from_mp3(io.BytesIO(data)) - break - except Exception as e: - logger.error(f"do_text_to_speech failed: {e}") - i += 1 - continue + i = 0 + while i < 3: + try: + data = await ComputeKernel.get_instance().do_text_to_speech(text, language, None, None, None, None, + model_name=model) + if data is not None: + audio = AudioSegment.from_mp3(io.BytesIO(data)) + break + except Exception as e: + logger.error(f"do_text_to_speech failed: {e}") + i += 1 + continue if audio is not None: - path = os.path.join(os.path.realpath(os.curdir), "{}.mp3".format(''.join(random.sample('zyxwvutsrqponmlkjihgfedcba', 10)))) + path = os.path.join(self.speech_path, "{}.mp3".format(''.join(random.sample('zyxwvutsrqponmlkjihgfedcba', 10)))) audio.export(path, format="mp3") - return "exec text_to_speech OK，speech file store at {}".format(path) + return "exec text_to_speech OK，speech file store at ```{}```".format(path) else: return "exec text_to_speech failed" diff --git a/src/aios_kernel/whisper_node.py b/src/aios_kernel/whisper_node.py index b036ad5..bac0cc4 100644 --- a/src/aios_kernel/whisper_node.py +++ b/src/aios_kernel/whisper_node.py @@ -1,55 +1,77 @@ +import io +import json from asyncio import Queue import asyncio import openai import os import logging +import srt +import webvtt +from openai import AsyncOpenAI +from openai.cli._progress import BufferReader +from pydub import AudioSegment +from datetime import timedelta + +from . import AIStorage from .compute_node import ComputeNode from .compute_task import ComputeTask, ComputeTaskResult, ComputeTaskState, ComputeTaskType logger = logging.getLogger(__name__) +SECONDS_IN_HOUR = 3600 +SECONDS_IN_MINUTE = 60 +HOURS_IN_DAY = 24 +MICROSECONDS_IN_MILLISECOND = 1000 + + +def timedelta_to_vtt_timestamp(timedelta_timestamp): + hrs, secs_remainder = divmod(timedelta_timestamp.seconds, SECONDS_IN_HOUR) + hrs += timedelta_timestamp.days * HOURS_IN_DAY + mins, secs = divmod(secs_remainder, SECONDS_IN_MINUTE) + msecs = timedelta_timestamp.microseconds // MICROSECONDS_IN_MILLISECOND + return "%02d:%02d:%02d.%03d" % (hrs, mins, secs, msecs) + class WhisperComputeNode(ComputeNode): _instance = None - def __new__(cls): + @classmethod + def get_instance(cls): if cls._instance is None: - cls._instance = super().__new__(cls) - cls._instance.is_start = False + cls._instance = cls() return cls._instance def __init__(self) -> None: super().__init__() - if self.is_start is True: - logger.warn("WhisperComputeNode is already start") - return - - self.is_start = True + self.is_start = False self.node_id = "whisper_node" self.enable = True self.task_queue = Queue() - self.open_api_key = None - if self.open_api_key is None and os.getenv("OPENAI_API_KEY") is not None: - self.open_api_key = os.getenv("OPENAI_API_KEY") - - if self.open_api_key is None: - raise Exception("WhisperComputeNode open_api_key is None") + if os.getenv("OPENAI_API_KEY") is not None: + self.openai_api_key = os.getenv("OPENAI_API_KEY") + else: + self.openai_api_key = AIStorage.get_instance().get_user_config().get_value("openai_api_key") self.start() def start(self): + if self.is_start is True: + logger.warn("WhisperComputeNode is already start") + return + self.is_start = True async def _run_task_loop(): while True: task = await self.task_queue.get() try: - result = self._run_task(task) + result = await self._run_task(task) if result is not None: task.state = ComputeTaskState.DONE task.result = result except Exception as e: logger.error(f"whisper_node run task error: {e}") + logger.exception(e) task.state = ComputeTaskState.ERROR task.result = ComputeTaskResult() task.result.set_from_task(task) @@ -58,7 +80,7 @@ class WhisperComputeNode(ComputeNode): asyncio.create_task(_run_task_loop()) - def _run_task(self, task: ComputeTask): + async def _run_task(self, task: ComputeTask): task.state = ComputeTaskState.RUNNING prompt = task.params["prompt"] response_format = None @@ -72,19 +94,111 @@ class WhisperComputeNode(ComputeNode): language = task.params["language"] file = task.params["file"] - resp = openai.Audio.transcribe("whisper-1", - file, - self.open_api_key, - prompt=prompt, - response_format=response_format, - temperature=temperature, - language=language) - result = ComputeTaskResult() - result.set_from_task(task) - result.worker_id = self.node_id - result.result_str = resp["text"] - result.result = resp - return result + client = AsyncOpenAI(api_key=self.openai_api_key) + + if os.path.getsize(file) > 25 * 1024 * 1024: + audio = AudioSegment.from_file(file) + text = "" + results = [] + latest_resp = None + step = 30 * 1000 + for i in range(0, 60 * 1000, step): + chunk = audio[i:i + step] + seg_file = io.BytesIO() + chunk.export(seg_file, format="mp3") + seg_file.seek(0) + + resp = await client.audio.transcriptions.create(model="whisper-1", + file = ("test.mp3", seg_file), + language=language, + temperature=temperature, + prompt=prompt, + response_format=response_format) + if response_format == "json": + if text == "": + text = resp.text + else: + text += "," + resp.text + elif response_format == "text": + if text == "": + text = resp + else: + text += "," + resp + elif response_format == "verbose_json": + if text == "": + text = resp.text + else: + text += "," + resp.text + results.extend(resp.segments) + elif response_format == "srt": + srt_list = list(srt.parse(resp)) + for item in srt_list: + item.start += timedelta(milliseconds=i) + item.end += timedelta(milliseconds=i) + results.append(item) + elif response_format == "vtt": + vtt = webvtt.read_buffer(io.StringIO(resp)) + for caption in vtt.captions: + start = timedelta_to_vtt_timestamp( + srt.srt_timestamp_to_timedelta(caption.start) + timedelta(milliseconds=i)) + end = timedelta_to_vtt_timestamp( + srt.srt_timestamp_to_timedelta(caption.end) + timedelta(milliseconds=i)) + results.append(webvtt.Caption(start, end, caption.text)) + else: + raise Exception(f"not support response_format: {response_format}") + + latest_resp = resp + + result = ComputeTaskResult() + result.set_from_task(task) + result.worker_id = self.node_id + if response_format == "text": + result.result_str = text + result.result = text + elif response_format == "json": + result.result_str = json.dumps({"text": text}) + resp.text = text + result.result = resp + elif response_format == "verbose_json": + result.result_str = json.dumps({"text": text, "segments": results}) + latest_resp.text = text + latest_resp.segments = results + result.result = latest_resp + elif response_format == "srt": + result.result_str = srt.compose(results) + result.result = result.result_str + elif response_format == "vtt": + vtt = webvtt.WebVTT() + vtt.captions.extend(results) + f = io.StringIO() + vtt.write(f) + f.seek(0) + result.result_str = f.read() + result.result = result.result_str + return result + else: + with open(file, "rb") as file_reader: + buffer_reader = BufferReader(file_reader.read(), desc="Upload progress") + + resp = await client.audio.transcriptions.create(model="whisper-1", + file = (file, buffer_reader), + language=language, + temperature=temperature, + prompt=prompt, + response_format=response_format) + result = ComputeTaskResult() + result.set_from_task(task) + result.worker_id = self.node_id + if response_format == "json": + result.result_str = json.dumps({"text": resp.text}) + elif response_format == "verbose_json": + result.result_str = json.dumps({"text": resp.text, "segments": resp.segments}) + elif response_format == "srt" or response_format == "vtt" or response_format == "text": + result.result_str = resp + else: + raise Exception(f"not support response_format: {response_format}") + result.result = resp + return result async def push_task(self, task: ComputeTask, proiority: int = 0): logger.info(f"whisper_node push task: {task.display()}") @@ -102,9 +216,10 @@ class WhisperComputeNode(ComputeNode): def get_capacity(self): return 0 - def is_support(self, task_type: ComputeTaskType) -> bool: - if task_type == ComputeTaskType.VOICE_2_TEXT: - return True + def is_support(self, task: ComputeTask) -> bool: + if task.task_type == ComputeTaskType.VOICE_2_TEXT: + if task.params['model_name'] is None or task.params['model_name'] == 'openai-whisper': + return True return False def is_local(self) -> bool: diff --git a/src/aios_kernel/workflow_env.py b/src/aios_kernel/workflow_env.py index 9201fba..03cd128 100644 --- a/src/aios_kernel/workflow_env.py +++ b/src/aios_kernel/workflow_env.py @@ -8,7 +8,7 @@ import threading import logging from typing import Optional -from .text_to_speech_function import TextToSpeechFunction +from .script_to_speech_function import ScriptToSpeechFunction from .image_2_text_function import Image2TextFunction from .compute_kernel import ComputeKernel, ComputeTaskResultCode from .environment import Environment,EnvironmentEvent @@ -80,7 +80,7 @@ class CalenderEnvironment(Environment): "update event in calender", self._update_event,update_param)) - + #maybe this function should be in other env? paint_param = { "prompt": "A description of the content of the painting", @@ -89,18 +89,18 @@ class CalenderEnvironment(Environment): self.add_ai_function(SimpleAIFunction("paint", "Draw a picture according to the description", self._paint,paint_param)) - + self.add_ai_function(SimpleAIFunction("get_contact", "get contact info", self._get_contact,{"name":"name of contact"})) - + self.add_ai_function(SimpleAIFunction("set_contact", "set contact info", self._set_contact,{"name":"name of contact","contact_info":"A json to descrpit contact"})) - - - - + + + + #self.add_ai_function(SimpleAIFunction("user_confirm", # "user confirm", # self._user_confirm)) @@ -169,10 +169,10 @@ class CalenderEnvironment(Environment): _event["location"] = row[5] _event["details"] = row[6] result[row[0]] = _event - + if not have_result: return "No event." - + return json.dumps(result, indent=4, sort_keys=True) async def _update_event(self,event_id, new_title=None, new_participants=None, new_location=None, new_details=None ,start_time=None, end_time=None): @@ -230,7 +230,7 @@ class CalenderEnvironment(Environment): def _do_get_value(self,key:str) -> Optional[str]: return None - + async def _get_contact(self,name:str) -> str: cm = ContactManager.get_instance() contact : Contact = cm.find_contact_by_name(name) @@ -302,7 +302,7 @@ class CalenderEnvironment(Environment): formatted_time = now.strftime('%Y-%m-%d %H:%M:%S') return formatted_time - + async def _paint(self, prompt, model_name = None) -> str: result = await ComputeKernel.get_instance().do_text_2_image(prompt, model_name) if result.result_code == ComputeTaskResultCode.ERROR: @@ -344,7 +344,7 @@ class WorkflowEnvironment(Environment): self.db_file = db_file self.local = threading.local() self.table_name = "WorkflowEnv_" + env_id - self.add_ai_function(TextToSpeechFunction()) + self.add_ai_function(ScriptToSpeechFunction()) self.add_ai_function(Image2TextFunction()) @@ -418,4 +418,4 @@ class WorkflowEnvironment(Environment): logging.error(f"Error occurred while update env{self.env_id}.{key} ,error:{e}") def get_functions(self): - pass \ No newline at end of file + pass diff --git a/src/requirements.txt b/src/requirements.txt index 20eebbf..151ea86 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -139,4 +139,6 @@ stability_sdk sentence-transformers==2.2.2 tiktoken markdown -PyPDF2 \ No newline at end of file +PyPDF2 +srt==3.5.3 +webvtt-py==0.4.6 diff --git a/src/service/aios_shell/aios_shell.py b/src/service/aios_shell/aios_shell.py index 6ecf52e..f3610c2 100644 --- a/src/service/aios_shell/aios_shell.py +++ b/src/service/aios_shell/aios_shell.py @@ -20,6 +20,8 @@ from prompt_toolkit.auto_suggest import AutoSuggestFromHistory from prompt_toolkit.completion import WordCompleter from prompt_toolkit.styles import Style +from aios_kernel.openai_tts_node import OpenAITTSComputeNode + directory = os.path.dirname(__file__) sys.path.append(directory + '/../../') @@ -74,8 +76,8 @@ class AIOS_Shell: user_config.add_user_config("shell.current","last opened target and topic",True,"default@Jarvis") proxy.declare_user_config() - google_text_to_speech = GoogleTextToSpeechNode.get_instance() - google_text_to_speech.declare_user_config() + # google_text_to_speech = GoogleTextToSpeechNode.get_instance() + # google_text_to_speech.declare_user_config() Local_Stability_ComputeNode.declare_user_config() @@ -93,8 +95,8 @@ class AIOS_Shell: if a_workflow is not None: bus.register_message_handler(target_id,a_workflow._process_msg) return True - - a_contact = await ContactManager.get_instance().get_contact(target_id) + + a_contact = ContactManager.get_instance().find_contact_by_name(target_id) if a_contact is not None: bus.register_message_handler(target_id,a_contact._process_msg) return True @@ -143,6 +145,12 @@ class AIOS_Shell: return False ComputeKernel.get_instance().add_compute_node(open_ai_node) + whisper_node = WhisperComputeNode.get_instance() + ComputeKernel.get_instance().add_compute_node(whisper_node); + + openai_tts_node = OpenAITTSComputeNode.get_instance() + ComputeKernel.get_instance().add_compute_node(openai_tts_node) + llama_nodes = ComputeNodeConfig.get_instance().initial() for llama_node in llama_nodes: llama_node.start() @@ -158,41 +166,41 @@ class AIOS_Shell: await AIStorage.get_instance().set_feature_init_result("llama",False) - if await AIStorage.get_instance().is_feature_enable("aigc"): - try: - google_text_to_speech_node = GoogleTextToSpeechNode.get_instance() - google_text_to_speech_node.init() - ComputeKernel.get_instance().add_compute_node(google_text_to_speech_node) - except Exception as e: - logger.error(f"google text to speech node initial failed! {e}") - await AIStorage.get_instance.set_feature_init_result("aigc",False) + # if await AIStorage.get_instance().is_feature_enable("aigc"): + # try: + # google_text_to_speech_node = GoogleTextToSpeechNode.get_instance() + # google_text_to_speech_node.init() + # ComputeKernel.get_instance().add_compute_node(google_text_to_speech_node) + # except Exception as e: + # logger.error(f"google text to speech node initial failed! {e}") + # await AIStorage.get_instance.set_feature_init_result("aigc",False) # stability_api_node = Stability_ComputeNode() # if await stability_api_node.initial() is not True: # logger.error("stability api node initial failed!") # ComputeKernel.get_instance().add_compute_node(stability_api_node) - - + + local_st_text_compute_node = LocalSentenceTransformer_Text_ComputeNode() if local_st_text_compute_node.initial() is not True: logger.error("local sentence transformer text embedding node initial failed!") else: ComputeKernel.get_instance().add_compute_node(local_st_text_compute_node) - + local_st_image_compute_node = LocalSentenceTransformer_Image_ComputeNode() if local_st_image_compute_node.initial() is not True: logger.error("local sentence transformer image embedding node initial failed!") else: ComputeKernel.get_instance().add_compute_node(local_st_image_compute_node) - + await ComputeKernel.get_instance().start() AIBus().get_default_bus().register_unhandle_message_handler(self._handle_no_target_msg) #AIBus().get_default_bus().register_message_handler(self.username,self._user_process_msg) - - + + pipelines = KnowledgePipelineManager.initial(os.path.join(AIStorage().get_instance().get_myai_dir(), "knowledge/pipelines")) pipelines.load_dir(os.path.join(AIStorage().get_instance().get_system_app_dir(), "knowledge_pipelines")) pipelines.load_dir(os.path.join(AIStorage().get_instance().get_myai_dir(), "knowledge_pipelines")) @@ -220,7 +228,7 @@ class AIOS_Shell: async def send_msg(self,msg:str,target_id:str,topic:str,sender:str = None) -> str: if sender == self.username: AIBus().get_default_bus().register_message_handler(self.username,self._user_process_msg) - + agent_msg = AgentMsg() agent_msg.set(sender,target_id,msg) agent_msg.topic = topic @@ -235,7 +243,7 @@ class AIOS_Shell: async def _user_process_msg(self,msg:AgentMsg) -> AgentMsg: pass - + async def get_tunnel_config_from_input(self,tunnel_target,tunnel_type): tunnel_config = {} @@ -274,7 +282,7 @@ class AIOS_Shell: async def append_tunnel_config(self,tunnel_config): user_data_dir = AIStorage.get_instance().get_myai_dir() tunnels_config_path = os.path.abspath(f"{user_data_dir}/etc/tunnels.cfg.toml") - all_tunnels = None + all_tunnels = None try: all_tunnels = toml.load(tunnels_config_path) except Exception as e: @@ -327,12 +335,12 @@ class AIOS_Shell: if contact_telegram is None: return None contact.telegram = contact_telegram - + contact_email = await try_get_input(f"Input {contact_name}'s email:") if contact_email is None: return None contact.email = contact_email - + contact_phone = await try_get_input(f"Input {contact_name}'s phone (optional):") if contact_phone is not None: contact.phone = contact_phone @@ -340,13 +348,13 @@ class AIOS_Shell: contact_note = await try_get_input(f"Input {contact_name}'s note (optional):") if contact_note is not None: contact.note = contact_note - + contact.added_by = self.username if is_update: cm.set_contact(contact_name,contact) else: cm.add_contact(contact_name,contact) - + async def handle_knowledge_commands(self, args): show_text = FormattedText([("class:title", "sub command not support!\n" "/knowledge pipelines\n" @@ -393,7 +401,7 @@ class AIOS_Shell: if args[1] == "llama": if len(args) < 4: return show_text - + model_name = args[2] url = args[3] ComputeNodeConfig.get_instance().add_node("llama", url, model_name) @@ -409,7 +417,7 @@ class AIOS_Shell: if args[1] == "llama": if len(args) < 4: return show_text - + model_name = args[2] url = args[3] ComputeNodeConfig.get_instance().remove_node("llama", url, model_name) @@ -495,7 +503,7 @@ class AIOS_Shell: else: show_text = FormattedText([("class:error", "/open Need Target Agent/Workflow ID! like /open Jarvis default")]) return show_text - + if len(args) >= 2: topic = args[1] else: @@ -506,7 +514,7 @@ class AIOS_Shell: target_exist = True if await WorkflowManager.get_instance().is_exist(target_id): target_exist = True - + if target_exist is False: show_text = FormattedText([("class:error", f"Target {target_id} not exist!")]) return show_text @@ -537,11 +545,11 @@ class AIOS_Shell: else: show_text = FormattedText([("class:error", "/disable Need Feature Name! like /disable llama")]) return show_text - + if not await AIStorage.get_instance().is_feature_enable(feature): show_text = FormattedText([("class:title", f"Feature {feature} already disabled!")]) return show_text - + await AIStorage.get_instance().disable_feature(feature) show_text = FormattedText([("class:title", f"Feature {feature} disabled!")]) return show_text @@ -717,8 +725,8 @@ async def main(): logging.basicConfig(handlers=[handler], level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') - + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') + is_daemon = False logger.info(f"Check Host OS :{os.name}") if os.name != 'nt': @@ -739,7 +747,7 @@ async def main(): shell.username = AIStorage.get_instance().get_user_config().get_value("username") init_result = await shell.initial() proxy.apply_storage() - + if init_result is False: if is_daemon: logger.error("aios shell initial failed!") @@ -759,7 +767,7 @@ async def main(): '/connect $target', '/contact $name', '/knowledge pipelines', - '/knowledge journal $pipeline [$topn]', + '/knowledge journal $pipeline [$topn]', '/knowledge query $object_id', '/set_config $key', '/enable $feature', From 7564349cb7f66abb5b2d5b84d2ab13c5ddedcc80 Mon Sep 17 00:00:00 2001 From: wugren Date: Wed, 22 Nov 2023 19:05:05 +0800 Subject: [PATCH 2/2] Fix split long audio bug --- src/aios_kernel/whisper_node.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/aios_kernel/whisper_node.py b/src/aios_kernel/whisper_node.py index bac0cc4..f9dcee5 100644 --- a/src/aios_kernel/whisper_node.py +++ b/src/aios_kernel/whisper_node.py @@ -101,9 +101,12 @@ class WhisperComputeNode(ComputeNode): text = "" results = [] latest_resp = None - step = 30 * 1000 - for i in range(0, 60 * 1000, step): - chunk = audio[i:i + step] + step = 10 * 60 * 1000 + for i in range(0, len(audio), step): + if i + step < len(audio): + chunk = audio[i:i + step] + else: + chunk = audio[i:] seg_file = io.BytesIO() chunk.export(seg_file, format="mp3") seg_file.seek(0)