TTS and ASR function implemented based on openai api
This commit is contained in:
wugren
@@ -6,6 +6,7 @@ from .compute_kernel import ComputeKernel,ComputeTask,ComputeTaskResult,ComputeT
|
||||
from .compute_node import ComputeNode,LocalComputeNode
|
||||
from .open_ai_node import OpenAI_ComputeNode
|
||||
from .role import AIRole,AIRoleGroup
|
||||
from .storage import ResourceLocation,AIStorage,UserConfig,UserConfigItem
|
||||
from .workflow import Workflow
|
||||
from .bus import AIBus
|
||||
from .workflow_env import WorkflowEnvironment,CalenderEnvironment,CalenderEvent,PaintEnvironment
|
||||
@@ -15,7 +16,6 @@ from .google_text_to_speech_node import GoogleTextToSpeechNode
|
||||
from .tunnel import AgentTunnel
|
||||
from .tg_tunnel import TelegramTunnel
|
||||
from .email_tunnel import EmailTunnel
|
||||
from .storage import ResourceLocation,AIStorage,UserConfig,UserConfigItem
|
||||
from .contact_manager import ContactManager,Contact,FamilyMember
|
||||
from .text_to_speech_function import TextToSpeechFunction
|
||||
from .image_2_text_function import Image2TextFunction
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import logging
|
||||
from typing import Dict
|
||||
|
||||
from aios_kernel import ComputeKernel
|
||||
from aios_kernel.ai_function import AIFunction
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AsrFunction(AIFunction):
|
||||
def __init__(self):
|
||||
self.func_id = "speech_to_text"
|
||||
self.description = "语音识别，将语音转换为文字"
|
||||
|
||||
def get_name(self) -> str:
|
||||
return self.func_id
|
||||
|
||||
def get_description(self) -> str:
|
||||
return self.description
|
||||
|
||||
def get_parameters(self) -> Dict:
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"audio_file": {"type": "string", "description": "音频文件路径"},
|
||||
"model": {"type": "string", "description": "识别模型", "enum": ["openai-whisper"]},
|
||||
"prompt": {"type": "string", "description": "提示语句,可以为None"},
|
||||
"response_format": {"type": "string", "description": "返回格式", "enum": ["text", "json", "srt", "verbose_json", "vtt"]},
|
||||
}
|
||||
}
|
||||
|
||||
async def execute(self, **kwargs) -> str:
|
||||
logger.info(f"execute asr function: {kwargs}")
|
||||
|
||||
audio_file = kwargs.get("audio_file")
|
||||
model = kwargs.get("model")
|
||||
prompt = kwargs.get("prompt")
|
||||
response_format = kwargs.get("response_format")
|
||||
if response_format is None:
|
||||
response_format = "text"
|
||||
|
||||
result = await ComputeKernel.get_instance().do_speech_to_text(audio_file, model, prompt, response_format)
|
||||
if result is not None:
|
||||
return f"exec speech_to_text Ok. {response_format} is\n```\n{result.result_str}\n```"
|
||||
else:
|
||||
return "exec speech_to_text failed"
|
||||
|
||||
def is_local(self) -> bool:
|
||||
return True
|
||||
|
||||
def is_in_zone(self) -> bool:
|
||||
return True
|
||||
|
||||
def is_ready_only(self) -> bool:
|
||||
return False
|
||||
@@ -197,7 +197,8 @@ class ComputeKernel:
|
||||
gender: Optional[str] = None,
|
||||
age: Optional[str] = None,
|
||||
voice_name: Optional[str] = None,
|
||||
tone: Optional[str] = None):
|
||||
tone: Optional[str] = None,
|
||||
model_name: Optional[str] = None):
|
||||
task_req = ComputeTask()
|
||||
task_req.params["text"] = input
|
||||
task_req.params["language_code"] = language_code
|
||||
@@ -205,6 +206,7 @@ class ComputeKernel:
|
||||
task_req.params["age"] = age
|
||||
task_req.params["voice_name"] = voice_name
|
||||
task_req.params["tone"] = tone
|
||||
task_req.params["model_name"] = model_name
|
||||
task_req.task_type = ComputeTaskType.TEXT_2_VOICE
|
||||
self.run(task_req)
|
||||
|
||||
@@ -213,6 +215,24 @@ class ComputeKernel:
|
||||
if task_req.state == ComputeTaskState.DONE:
|
||||
return task_result.result
|
||||
|
||||
async def do_speech_to_text(self,
|
||||
audio: str,
|
||||
model: str,
|
||||
prompt: Optional[str],
|
||||
response_format: Optional[str]):
|
||||
task_req = ComputeTask()
|
||||
task_req.params["file"] = audio
|
||||
task_req.params["model_name"] = model
|
||||
task_req.params["prompt"] = prompt
|
||||
task_req.params["response_format"] = response_format
|
||||
task_req.task_type = ComputeTaskType.VOICE_2_TEXT
|
||||
|
||||
self.run(task_req)
|
||||
|
||||
task_result = await self._wait_task(task_req)
|
||||
|
||||
if task_req.state == ComputeTaskState.DONE:
|
||||
return task_result
|
||||
|
||||
def text_2_image(self, prompt:str, model_name:Optional[str] = None, negative_prompt = None):
|
||||
task = ComputeTask()
|
||||
|
||||
@@ -117,9 +117,18 @@ class GoogleTextToSpeechNode(ComputeNode):
|
||||
def _run_task(self, task: ComputeTask):
|
||||
task.state = ComputeTaskState.RUNNING
|
||||
language_code = task.params["language_code"]
|
||||
if language_code is None:
|
||||
language_code = "en"
|
||||
|
||||
text = task.params["text"]
|
||||
voice_name = task.params["voice_name"]
|
||||
if voice_name is None:
|
||||
voice_name = "default"
|
||||
|
||||
gender = task.params["gender"]
|
||||
if gender is None:
|
||||
gender = "female"
|
||||
|
||||
age = task.params["age"]
|
||||
|
||||
if language_code == "zh":
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
import asyncio
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
from asyncio import Queue
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
from aios_kernel import ComputeNode, ComputeTask, ComputeTaskState, ComputeTaskResult, ComputeTaskType, AIStorage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OpenAITTSComputeNode(ComputeNode):
|
||||
_instance = None
|
||||
|
||||
@classmethod
|
||||
def get_instance(cls):
|
||||
if cls._instance is None:
|
||||
cls._instance = cls()
|
||||
return cls._instance
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.is_start = False
|
||||
self.node_id = "openai_tts_node"
|
||||
self.task_queue = Queue()
|
||||
self.voice_list = {
|
||||
"female": ["nova", "shimmer"],
|
||||
"man": ["alloy", "echo", "fable", "onyx"]
|
||||
}
|
||||
if os.getenv("OPENAI_API_KEY") is not None:
|
||||
self.openai_api_key = os.getenv("OPENAI_API_KEY")
|
||||
else:
|
||||
self.openai_api_key = AIStorage.get_instance().get_user_config().get_value("openai_api_key")
|
||||
|
||||
self.start()
|
||||
|
||||
def start(self):
|
||||
if self.is_start is True:
|
||||
logger.warn("OpenAITTSComputeNode is already start")
|
||||
return
|
||||
self.is_start = True
|
||||
|
||||
async def _run_task_loop():
|
||||
while True:
|
||||
task = await self.task_queue.get()
|
||||
try:
|
||||
result = await self._run_task(task)
|
||||
if result is not None:
|
||||
task.state = ComputeTaskState.DONE
|
||||
task.result = result
|
||||
except Exception as e:
|
||||
logger.error(f"openai_tts_node run task error: {e}")
|
||||
task.state = ComputeTaskState.ERROR
|
||||
task.result = ComputeTaskResult()
|
||||
task.result.set_from_task(task)
|
||||
task.result.worker_id = self.node_id
|
||||
task.result.result_str = str(e)
|
||||
|
||||
asyncio.create_task(_run_task_loop())
|
||||
|
||||
async def _run_task(self,task: ComputeTask):
|
||||
task.state = ComputeTaskState.RUNNING
|
||||
text = task.params["text"]
|
||||
voice_name = task.params["voice_name"]
|
||||
if voice_name is None:
|
||||
voice_name = "default"
|
||||
gender = task.params["gender"]
|
||||
if gender is None:
|
||||
gender = "female"
|
||||
|
||||
voice_list = self.voice_list[gender]
|
||||
voice = voice_list[hash(voice_name)%len(voice_list)]
|
||||
|
||||
model_name = task.params['model_name']
|
||||
if model_name is None:
|
||||
model_name = 'tts-1'
|
||||
|
||||
client = AsyncOpenAI(api_key=self.openai_api_key)
|
||||
|
||||
response = await client.audio.speech.create(model=model_name, voice=voice, input=text)
|
||||
|
||||
cache = io.BytesIO()
|
||||
async for data in await response.aiter_bytes():
|
||||
cache.write(data)
|
||||
|
||||
cache.seek(0)
|
||||
|
||||
result = ComputeTaskResult()
|
||||
result.set_from_task(task)
|
||||
result.worker_id = self.node_id
|
||||
result.result = cache.read()
|
||||
return result
|
||||
|
||||
async def push_task(self, task: ComputeTask, proiority: int = 0):
|
||||
logger.info(f"openai_tts_node push task: {task.display()}")
|
||||
self.task_queue.put_nowait(task)
|
||||
|
||||
async def remove_task(self, task_id: str):
|
||||
pass
|
||||
|
||||
def get_task_state(self, task_id: str):
|
||||
pass
|
||||
|
||||
def display(self) -> str:
|
||||
return f"OpenAITTSComputeNode: {self.node_id}"
|
||||
|
||||
def get_capacity(self):
|
||||
return 0
|
||||
|
||||
def is_support(self, task: ComputeTask) -> bool:
|
||||
if task.task_type == ComputeTaskType.TEXT_2_VOICE:
|
||||
if task.params['model_name'] is None or task.params['model_name'] == 'tts-1' or task.params['model_name'] == 'tts-1-hd':
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_local(self) -> bool:
|
||||
return False
|
||||
@@ -0,0 +1,107 @@
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
from aios_kernel import ComputeKernel, AIStorage
|
||||
from aios_kernel.ai_function import AIFunction
|
||||
|
||||
from pydub import AudioSegment
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ScriptToSpeechFunction(AIFunction):
|
||||
def __init__(self):
|
||||
self.func_id = "script_to_speech"
|
||||
self.description = "根据输入的剧本生成音频文件，成功时会返回音频文件路径"
|
||||
self.speech_path = os.path.join(AIStorage.get_instance().get_myai_dir(), "tts")
|
||||
Path(self.speech_path).mkdir(exist_ok=True)
|
||||
|
||||
def get_name(self) -> str:
|
||||
return self.func_id
|
||||
|
||||
def get_description(self) -> str:
|
||||
return self.description
|
||||
|
||||
def get_parameters(self) -> Dict:
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"language": {"type": "string", "description": "演播语言", "enum": ["zh", "en"]},
|
||||
"model": {"type": "string", "description": "演播模型", "enum": ["tts-1", "tts-1-hd"]},
|
||||
"roles": {"type": "array", "items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string", "description": "角色名字"},
|
||||
"gender": {"type": "string", "description": "角色性别", "enum": ["man", "female"]},
|
||||
"age": {"type": "string", "description": "年龄", "enum": ["child", "adult"]},
|
||||
}}},
|
||||
"lines": {"type": "array", "items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string", "description": "角色名字"},
|
||||
"tone": {"type": "string", "description": "演播情感",
|
||||
"enum": ["happy", "sad", "angry", "fear", "disgust", "surprise", "neutral"]},
|
||||
"text": {"type": "string", "description": "台词"},
|
||||
}
|
||||
}}
|
||||
}
|
||||
}
|
||||
|
||||
async def execute(self, **kwargs) -> str:
|
||||
logger.info(f"execute text_to_speech function: {kwargs}")
|
||||
|
||||
language = kwargs.get("language")
|
||||
if language is None:
|
||||
language = "zh"
|
||||
model = kwargs.get("model")
|
||||
roles = kwargs.get("roles")
|
||||
lines = kwargs.get("lines")
|
||||
|
||||
audio = None
|
||||
for line in lines:
|
||||
name = line.get("name")
|
||||
tone = line.get("tone")
|
||||
text = line.get("text")
|
||||
gender = None
|
||||
age = None
|
||||
for role in roles:
|
||||
role_name = role.get("name")
|
||||
if role_name == name:
|
||||
gender = role.get("gender")
|
||||
age = role.get("age")
|
||||
break
|
||||
i = 0
|
||||
while i < 3:
|
||||
try:
|
||||
data = await ComputeKernel.get_instance().do_text_to_speech(text, language, gender, age, name, tone, model_name=model)
|
||||
if audio is None:
|
||||
audio = AudioSegment.from_mp3(io.BytesIO(data))
|
||||
else:
|
||||
audio = audio + AudioSegment.from_mp3(io.BytesIO(data))
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"do_text_to_speech failed: {e}")
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if audio is not None:
|
||||
path = os.path.join(self.speech_path, "{}.mp3".format(''.join(random.sample('zyxwvutsrqponmlkjihgfedcba', 10))))
|
||||
audio.export(path, format="mp3")
|
||||
return "exec text_to_speech OK，speech file store at ```{}```".format(path)
|
||||
else:
|
||||
return "exec text_to_speech failed"
|
||||
|
||||
def is_local(self) -> bool:
|
||||
return True
|
||||
|
||||
def is_in_zone(self) -> bool:
|
||||
return True
|
||||
|
||||
def is_ready_only(self) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@@ -2,19 +2,23 @@ import io
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
from aios_kernel import ComputeKernel
|
||||
from aios_kernel import ComputeKernel, AIStorage
|
||||
from aios_kernel.ai_function import AIFunction
|
||||
|
||||
from pydub import AudioSegment
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TextToSpeechFunction(AIFunction):
|
||||
def __init__(self):
|
||||
self.func_id = "text_to_speech"
|
||||
self.description = "根据输入的剧本生成音频文件，成功时会返回音频文件路径"
|
||||
self.description = "根据输入的文本生成音频文件，成功时会返回音频文件路径"
|
||||
self.speech_path = os.path.join(AIStorage.get_instance().get_myai_dir(), "tts")
|
||||
Path(self.speech_path).mkdir(exist_ok=True)
|
||||
|
||||
def get_name(self) -> str:
|
||||
return self.func_id
|
||||
@@ -27,22 +31,8 @@ class TextToSpeechFunction(AIFunction):
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"language": {"type": "string", "description": "演播语言", "enum": ["zh", "en"]},
|
||||
"roles": {"type": "array", "items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string", "description": "角色名字"},
|
||||
"gender": {"type": "string", "description": "角色性别", "enum": ["man", "female"]},
|
||||
"age": {"type": "string", "description": "年龄", "enum": ["child", "adult"]},
|
||||
}}},
|
||||
"lines": {"type": "array", "items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string", "description": "角色名字"},
|
||||
"tone": {"type": "string", "description": "演播情感",
|
||||
"enum": ["happy", "sad", "angry", "fear", "disgust", "surprise", "neutral"]},
|
||||
"text": {"type": "string", "description": "台词"},
|
||||
}
|
||||
}}
|
||||
"model": {"type": "string", "description": "演播模型", "enum": ["tts-1", "tts-1-hd"]},
|
||||
"text": {"type": "string", "description": "文本内容"}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,41 +41,27 @@ class TextToSpeechFunction(AIFunction):
|
||||
|
||||
language = kwargs.get("language")
|
||||
if language is None:
|
||||
language = "zh"
|
||||
roles = kwargs.get("roles")
|
||||
lines = kwargs.get("lines")
|
||||
language = "en"
|
||||
model = kwargs.get("model")
|
||||
text = kwargs.get("text")
|
||||
|
||||
audio = None
|
||||
for line in lines:
|
||||
name = line.get("name")
|
||||
tone = line.get("tone")
|
||||
text = line.get("text")
|
||||
gender = None
|
||||
age = None
|
||||
for role in roles:
|
||||
role_name = role.get("name")
|
||||
if role_name == name:
|
||||
gender = role.get("gender")
|
||||
age = role.get("age")
|
||||
break
|
||||
i = 0
|
||||
while i < 3:
|
||||
try:
|
||||
data = await ComputeKernel.get_instance().do_text_to_speech(text, language, gender, age, name, tone)
|
||||
if audio is None:
|
||||
audio = AudioSegment.from_mp3(io.BytesIO(data))
|
||||
else:
|
||||
audio = audio + AudioSegment.from_mp3(io.BytesIO(data))
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"do_text_to_speech failed: {e}")
|
||||
i += 1
|
||||
continue
|
||||
i = 0
|
||||
while i < 3:
|
||||
try:
|
||||
data = await ComputeKernel.get_instance().do_text_to_speech(text, language, None, None, None, None,
|
||||
model_name=model)
|
||||
if data is not None:
|
||||
audio = AudioSegment.from_mp3(io.BytesIO(data))
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"do_text_to_speech failed: {e}")
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if audio is not None:
|
||||
path = os.path.join(os.path.realpath(os.curdir), "{}.mp3".format(''.join(random.sample('zyxwvutsrqponmlkjihgfedcba', 10))))
|
||||
path = os.path.join(self.speech_path, "{}.mp3".format(''.join(random.sample('zyxwvutsrqponmlkjihgfedcba', 10))))
|
||||
audio.export(path, format="mp3")
|
||||
return "exec text_to_speech OK，speech file store at {}".format(path)
|
||||
return "exec text_to_speech OK，speech file store at ```{}```".format(path)
|
||||
else:
|
||||
return "exec text_to_speech failed"
|
||||
|
||||
|
||||
+147
-32
@@ -1,55 +1,77 @@
|
||||
import io
|
||||
import json
|
||||
from asyncio import Queue
|
||||
import asyncio
|
||||
import openai
|
||||
import os
|
||||
import logging
|
||||
import srt
|
||||
import webvtt
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
from openai.cli._progress import BufferReader
|
||||
from pydub import AudioSegment
|
||||
from datetime import timedelta
|
||||
|
||||
from . import AIStorage
|
||||
from .compute_node import ComputeNode
|
||||
from .compute_task import ComputeTask, ComputeTaskResult, ComputeTaskState, ComputeTaskType
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SECONDS_IN_HOUR = 3600
|
||||
SECONDS_IN_MINUTE = 60
|
||||
HOURS_IN_DAY = 24
|
||||
MICROSECONDS_IN_MILLISECOND = 1000
|
||||
|
||||
|
||||
def timedelta_to_vtt_timestamp(timedelta_timestamp):
|
||||
hrs, secs_remainder = divmod(timedelta_timestamp.seconds, SECONDS_IN_HOUR)
|
||||
hrs += timedelta_timestamp.days * HOURS_IN_DAY
|
||||
mins, secs = divmod(secs_remainder, SECONDS_IN_MINUTE)
|
||||
msecs = timedelta_timestamp.microseconds // MICROSECONDS_IN_MILLISECOND
|
||||
return "%02d:%02d:%02d.%03d" % (hrs, mins, secs, msecs)
|
||||
|
||||
|
||||
class WhisperComputeNode(ComputeNode):
|
||||
_instance = None
|
||||
|
||||
def __new__(cls):
|
||||
@classmethod
|
||||
def get_instance(cls):
|
||||
if cls._instance is None:
|
||||
cls._instance = super().__new__(cls)
|
||||
cls._instance.is_start = False
|
||||
cls._instance = cls()
|
||||
return cls._instance
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
if self.is_start is True:
|
||||
logger.warn("WhisperComputeNode is already start")
|
||||
return
|
||||
|
||||
self.is_start = True
|
||||
self.is_start = False
|
||||
self.node_id = "whisper_node"
|
||||
self.enable = True
|
||||
self.task_queue = Queue()
|
||||
self.open_api_key = None
|
||||
|
||||
if self.open_api_key is None and os.getenv("OPENAI_API_KEY") is not None:
|
||||
self.open_api_key = os.getenv("OPENAI_API_KEY")
|
||||
|
||||
if self.open_api_key is None:
|
||||
raise Exception("WhisperComputeNode open_api_key is None")
|
||||
if os.getenv("OPENAI_API_KEY") is not None:
|
||||
self.openai_api_key = os.getenv("OPENAI_API_KEY")
|
||||
else:
|
||||
self.openai_api_key = AIStorage.get_instance().get_user_config().get_value("openai_api_key")
|
||||
|
||||
self.start()
|
||||
|
||||
def start(self):
|
||||
if self.is_start is True:
|
||||
logger.warn("WhisperComputeNode is already start")
|
||||
return
|
||||
self.is_start = True
|
||||
async def _run_task_loop():
|
||||
while True:
|
||||
task = await self.task_queue.get()
|
||||
try:
|
||||
result = self._run_task(task)
|
||||
result = await self._run_task(task)
|
||||
if result is not None:
|
||||
task.state = ComputeTaskState.DONE
|
||||
task.result = result
|
||||
except Exception as e:
|
||||
logger.error(f"whisper_node run task error: {e}")
|
||||
logger.exception(e)
|
||||
task.state = ComputeTaskState.ERROR
|
||||
task.result = ComputeTaskResult()
|
||||
task.result.set_from_task(task)
|
||||
@@ -58,7 +80,7 @@ class WhisperComputeNode(ComputeNode):
|
||||
|
||||
asyncio.create_task(_run_task_loop())
|
||||
|
||||
def _run_task(self, task: ComputeTask):
|
||||
async def _run_task(self, task: ComputeTask):
|
||||
task.state = ComputeTaskState.RUNNING
|
||||
prompt = task.params["prompt"]
|
||||
response_format = None
|
||||
@@ -72,19 +94,111 @@ class WhisperComputeNode(ComputeNode):
|
||||
language = task.params["language"]
|
||||
file = task.params["file"]
|
||||
|
||||
resp = openai.Audio.transcribe("whisper-1",
|
||||
file,
|
||||
self.open_api_key,
|
||||
prompt=prompt,
|
||||
response_format=response_format,
|
||||
temperature=temperature,
|
||||
language=language)
|
||||
result = ComputeTaskResult()
|
||||
result.set_from_task(task)
|
||||
result.worker_id = self.node_id
|
||||
result.result_str = resp["text"]
|
||||
result.result = resp
|
||||
return result
|
||||
client = AsyncOpenAI(api_key=self.openai_api_key)
|
||||
|
||||
if os.path.getsize(file) > 25 * 1024 * 1024:
|
||||
audio = AudioSegment.from_file(file)
|
||||
text = ""
|
||||
results = []
|
||||
latest_resp = None
|
||||
step = 30 * 1000
|
||||
for i in range(0, 60 * 1000, step):
|
||||
chunk = audio[i:i + step]
|
||||
seg_file = io.BytesIO()
|
||||
chunk.export(seg_file, format="mp3")
|
||||
seg_file.seek(0)
|
||||
|
||||
resp = await client.audio.transcriptions.create(model="whisper-1",
|
||||
file = ("test.mp3", seg_file),
|
||||
language=language,
|
||||
temperature=temperature,
|
||||
prompt=prompt,
|
||||
response_format=response_format)
|
||||
if response_format == "json":
|
||||
if text == "":
|
||||
text = resp.text
|
||||
else:
|
||||
text += "," + resp.text
|
||||
elif response_format == "text":
|
||||
if text == "":
|
||||
text = resp
|
||||
else:
|
||||
text += "," + resp
|
||||
elif response_format == "verbose_json":
|
||||
if text == "":
|
||||
text = resp.text
|
||||
else:
|
||||
text += "," + resp.text
|
||||
results.extend(resp.segments)
|
||||
elif response_format == "srt":
|
||||
srt_list = list(srt.parse(resp))
|
||||
for item in srt_list:
|
||||
item.start += timedelta(milliseconds=i)
|
||||
item.end += timedelta(milliseconds=i)
|
||||
results.append(item)
|
||||
elif response_format == "vtt":
|
||||
vtt = webvtt.read_buffer(io.StringIO(resp))
|
||||
for caption in vtt.captions:
|
||||
start = timedelta_to_vtt_timestamp(
|
||||
srt.srt_timestamp_to_timedelta(caption.start) + timedelta(milliseconds=i))
|
||||
end = timedelta_to_vtt_timestamp(
|
||||
srt.srt_timestamp_to_timedelta(caption.end) + timedelta(milliseconds=i))
|
||||
results.append(webvtt.Caption(start, end, caption.text))
|
||||
else:
|
||||
raise Exception(f"not support response_format: {response_format}")
|
||||
|
||||
latest_resp = resp
|
||||
|
||||
result = ComputeTaskResult()
|
||||
result.set_from_task(task)
|
||||
result.worker_id = self.node_id
|
||||
if response_format == "text":
|
||||
result.result_str = text
|
||||
result.result = text
|
||||
elif response_format == "json":
|
||||
result.result_str = json.dumps({"text": text})
|
||||
resp.text = text
|
||||
result.result = resp
|
||||
elif response_format == "verbose_json":
|
||||
result.result_str = json.dumps({"text": text, "segments": results})
|
||||
latest_resp.text = text
|
||||
latest_resp.segments = results
|
||||
result.result = latest_resp
|
||||
elif response_format == "srt":
|
||||
result.result_str = srt.compose(results)
|
||||
result.result = result.result_str
|
||||
elif response_format == "vtt":
|
||||
vtt = webvtt.WebVTT()
|
||||
vtt.captions.extend(results)
|
||||
f = io.StringIO()
|
||||
vtt.write(f)
|
||||
f.seek(0)
|
||||
result.result_str = f.read()
|
||||
result.result = result.result_str
|
||||
return result
|
||||
else:
|
||||
with open(file, "rb") as file_reader:
|
||||
buffer_reader = BufferReader(file_reader.read(), desc="Upload progress")
|
||||
|
||||
resp = await client.audio.transcriptions.create(model="whisper-1",
|
||||
file = (file, buffer_reader),
|
||||
language=language,
|
||||
temperature=temperature,
|
||||
prompt=prompt,
|
||||
response_format=response_format)
|
||||
result = ComputeTaskResult()
|
||||
result.set_from_task(task)
|
||||
result.worker_id = self.node_id
|
||||
if response_format == "json":
|
||||
result.result_str = json.dumps({"text": resp.text})
|
||||
elif response_format == "verbose_json":
|
||||
result.result_str = json.dumps({"text": resp.text, "segments": resp.segments})
|
||||
elif response_format == "srt" or response_format == "vtt" or response_format == "text":
|
||||
result.result_str = resp
|
||||
else:
|
||||
raise Exception(f"not support response_format: {response_format}")
|
||||
result.result = resp
|
||||
return result
|
||||
|
||||
async def push_task(self, task: ComputeTask, proiority: int = 0):
|
||||
logger.info(f"whisper_node push task: {task.display()}")
|
||||
@@ -102,9 +216,10 @@ class WhisperComputeNode(ComputeNode):
|
||||
def get_capacity(self):
|
||||
return 0
|
||||
|
||||
def is_support(self, task_type: ComputeTaskType) -> bool:
|
||||
if task_type == ComputeTaskType.VOICE_2_TEXT:
|
||||
return True
|
||||
def is_support(self, task: ComputeTask) -> bool:
|
||||
if task.task_type == ComputeTaskType.VOICE_2_TEXT:
|
||||
if task.params['model_name'] is None or task.params['model_name'] == 'openai-whisper':
|
||||
return True
|
||||
return False
|
||||
|
||||
def is_local(self) -> bool:
|
||||
|
||||
@@ -8,7 +8,7 @@ import threading
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from .text_to_speech_function import TextToSpeechFunction
|
||||
from .script_to_speech_function import ScriptToSpeechFunction
|
||||
from .image_2_text_function import Image2TextFunction
|
||||
from .compute_kernel import ComputeKernel, ComputeTaskResultCode
|
||||
from .environment import Environment,EnvironmentEvent
|
||||
@@ -344,7 +344,7 @@ class WorkflowEnvironment(Environment):
|
||||
self.db_file = db_file
|
||||
self.local = threading.local()
|
||||
self.table_name = "WorkflowEnv_" + env_id
|
||||
self.add_ai_function(TextToSpeechFunction())
|
||||
self.add_ai_function(ScriptToSpeechFunction())
|
||||
self.add_ai_function(Image2TextFunction())
|
||||
|
||||
|
||||
|
||||
@@ -140,3 +140,5 @@ sentence-transformers==2.2.2
|
||||
tiktoken
|
||||
markdown
|
||||
PyPDF2
|
||||
srt==3.5.3
|
||||
webvtt-py==0.4.6
|
||||
|
||||
@@ -20,6 +20,8 @@ from prompt_toolkit.auto_suggest import AutoSuggestFromHistory
|
||||
from prompt_toolkit.completion import WordCompleter
|
||||
from prompt_toolkit.styles import Style
|
||||
|
||||
from aios_kernel.openai_tts_node import OpenAITTSComputeNode
|
||||
|
||||
directory = os.path.dirname(__file__)
|
||||
sys.path.append(directory + '/../../')
|
||||
|
||||
@@ -74,8 +76,8 @@ class AIOS_Shell:
|
||||
user_config.add_user_config("shell.current","last opened target and topic",True,"default@Jarvis")
|
||||
proxy.declare_user_config()
|
||||
|
||||
google_text_to_speech = GoogleTextToSpeechNode.get_instance()
|
||||
google_text_to_speech.declare_user_config()
|
||||
# google_text_to_speech = GoogleTextToSpeechNode.get_instance()
|
||||
# google_text_to_speech.declare_user_config()
|
||||
|
||||
Local_Stability_ComputeNode.declare_user_config()
|
||||
|
||||
@@ -94,7 +96,7 @@ class AIOS_Shell:
|
||||
bus.register_message_handler(target_id,a_workflow._process_msg)
|
||||
return True
|
||||
|
||||
a_contact = await ContactManager.get_instance().get_contact(target_id)
|
||||
a_contact = ContactManager.get_instance().find_contact_by_name(target_id)
|
||||
if a_contact is not None:
|
||||
bus.register_message_handler(target_id,a_contact._process_msg)
|
||||
return True
|
||||
@@ -143,6 +145,12 @@ class AIOS_Shell:
|
||||
return False
|
||||
ComputeKernel.get_instance().add_compute_node(open_ai_node)
|
||||
|
||||
whisper_node = WhisperComputeNode.get_instance()
|
||||
ComputeKernel.get_instance().add_compute_node(whisper_node);
|
||||
|
||||
openai_tts_node = OpenAITTSComputeNode.get_instance()
|
||||
ComputeKernel.get_instance().add_compute_node(openai_tts_node)
|
||||
|
||||
llama_nodes = ComputeNodeConfig.get_instance().initial()
|
||||
for llama_node in llama_nodes:
|
||||
llama_node.start()
|
||||
@@ -158,14 +166,14 @@ class AIOS_Shell:
|
||||
await AIStorage.get_instance().set_feature_init_result("llama",False)
|
||||
|
||||
|
||||
if await AIStorage.get_instance().is_feature_enable("aigc"):
|
||||
try:
|
||||
google_text_to_speech_node = GoogleTextToSpeechNode.get_instance()
|
||||
google_text_to_speech_node.init()
|
||||
ComputeKernel.get_instance().add_compute_node(google_text_to_speech_node)
|
||||
except Exception as e:
|
||||
logger.error(f"google text to speech node initial failed! {e}")
|
||||
await AIStorage.get_instance.set_feature_init_result("aigc",False)
|
||||
# if await AIStorage.get_instance().is_feature_enable("aigc"):
|
||||
# try:
|
||||
# google_text_to_speech_node = GoogleTextToSpeechNode.get_instance()
|
||||
# google_text_to_speech_node.init()
|
||||
# ComputeKernel.get_instance().add_compute_node(google_text_to_speech_node)
|
||||
# except Exception as e:
|
||||
# logger.error(f"google text to speech node initial failed! {e}")
|
||||
# await AIStorage.get_instance.set_feature_init_result("aigc",False)
|
||||
|
||||
# stability_api_node = Stability_ComputeNode()
|
||||
# if await stability_api_node.initial() is not True:
|
||||
|
||||
Reference in New Issue
Block a user