gyt/opendan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support multimodal input

Browse Source
This commit is contained in:
wugren
2024-02-08 17:39:46 +08:00
parent 3e9f8ac0ed
commit 42c4f97a94
8 changed files with 158 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/aios/agent/llm_process.py
+91 -63
View File
@@ -1,5 +1,7 @@
# Old name is behavior, I belive new name "llm_process" is better
# pylint:disable=E0402
import os.path
from ..utils import video_utils,image_utils
from ..proto.compute_task import LLMPrompt,LLMResult,ComputeTaskResult,ComputeTaskResultCode
@@ -31,11 +33,11 @@ class BaseLLMProcess(ABC):
self.goal:str = None #目标
self.input_example:str= None #输入样例
self.result_example:str = None #llm_result样例
self.enable_json_resp = False
#None means system default,
# TODO: support abcstract model name like: local-hight,local-low,local-medium,remote-hight,remote-low,remote-medium
self.model_name = None
self.model_name = None
self.max_token = 1000 # result_token
self.max_prompt_token = 1000 # not include input prompt
self.timeout = 1800 # 30 min
@@ -55,8 +57,8 @@ class BaseLLMProcess(ABC):
@abstractmethod
def prepare_inner_function_context_for_exec(self,inner_func_name:str,parameters:Dict):
return
return
@abstractmethod
async def post_llm_process(self,actions:List[ActionNode],input:Dict,llm_result:LLMResult) -> bool:
pass
@@ -76,14 +78,14 @@ class BaseLLMProcess(ABC):
self.max_token = config.get("max_token")
if config.get("timeout"):
self.timeout = config.get("timeout")
return True
@abstractmethod
async def initial(self,params:Dict = None) -> bool:
pass
def _format_content_by_env_value(self,content:str,env)->str:
return content.format_map(env)
@@ -120,12 +122,12 @@ class BaseLLMProcess(ABC):
task_result.result_code = ComputeTaskResultCode.ERROR
task_result.error_str = f"prompt too long,can not predict"
return task_result
if stack_limit > 0:
inner_functions=prompt.inner_functions
else:
inner_functions = None
task_result: ComputeTaskResult = await (ComputeKernel.get_instance().do_llm_completion(
prompt,
@@ -140,7 +142,7 @@ class BaseLLMProcess(ABC):
return task_result
inner_func_call_node = None
result_message : dict = task_result.result.get("message")
if result_message:
inner_func_call_node = result_message.get("function_call")
@@ -166,7 +168,7 @@ class BaseLLMProcess(ABC):
max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.model_name)
#if max_result_token < MIN_PREDICT_TOKEN_LEN:
# return LLMResult.from_error_str(f"prompt too long,can not predict")
task_result: ComputeTaskResult = await (ComputeKernel.get_instance().do_llm_completion(
prompt,
resp_mode=resp_mode,
@@ -174,12 +176,12 @@ class BaseLLMProcess(ABC):
max_token=max_result_token,
inner_functions=prompt.inner_functions, #NOTICE: inner_function in prompt can be a subset of get_inner_function
timeout=self.timeout))
if task_result.result_code != ComputeTaskResultCode.OK:
err_str = f"do_llm_completion error:{task_result.error_str}"
logger.error(err_str)
return LLMResult.from_error_str(err_str)
result_message = task_result.result.get("message")
inner_func_call_node = None
if result_message:
@@ -202,7 +204,7 @@ class BaseLLMProcess(ABC):
await self.post_llm_process(llm_result.action_list,input,llm_result)
return llm_result
class LLMAgentBaseProcess(BaseLLMProcess):
def __init__(self) -> None:
super().__init__()
@@ -211,11 +213,11 @@ class LLMAgentBaseProcess(BaseLLMProcess):
self.process_description:str = None
self.reply_format:str = None
self.context : str = None
self.workspace : AgentWorkspace = None # If Workspace is not none , enable Agent Tasklist
self.memory : AgentMemory = None
self.enable_kb : bool = False
self.kb = None
self.kb = None
async def initial(self,params:Dict = None) -> bool:
self.memory = params.get("memory")
@@ -227,23 +229,23 @@ class LLMAgentBaseProcess(BaseLLMProcess):
return True
async def load_default_config(self) -> bool:
return True
async def load_from_config(self, config: dict,is_load_default=True) -> Coroutine[Any, Any, bool]:
if is_load_default:
await self.load_default_config()
if await super().load_from_config(config) is False:
return False
self.role_description = config.get("role_desc")
if self.role_description is None:
logger.error(f"role_description not found in config")
return False
if config.get("process_description"):
self.process_description = config.get("process_description")
if config.get("reply_format"):
self.reply_format = config.get("reply_format")
@@ -282,7 +284,7 @@ class LLMAgentBaseProcess(BaseLLMProcess):
return system_prompt_dict
def prepare_inner_function_context_for_exec(self,inner_func_name:str,parameters:Dict):
parameters["_workspace"] = self.workspace
parameters["_workspace"] = self.workspace
def get_action_desc(self) -> Dict:
result = {}
@@ -290,17 +292,17 @@ class LLMAgentBaseProcess(BaseLLMProcess):
for action in actions_list:
result[action.get_name()] = action.get_description()
return result
async def get_inner_function_for_exec(self,func_name:str) -> AIFunction:
return self.llm_context.get_ai_function(func_name)
async def _execute_actions(self,actions:List[ActionNode],action_params:Dict):
for action_item in actions:
op : AIAction = self.llm_context.get_ai_action(action_item.name)
if op:
if action_item.parms is None:
action_item.parms = {}
real_parms = {**action_params,**action_item.parms}
action_item.parms["_result"] = await op.execute(real_parms)
@@ -309,17 +311,19 @@ class LLMAgentBaseProcess(BaseLLMProcess):
logger.warn(f"action {action_item.name} not found")
return False
class AgentMessageProcess(LLMAgentBaseProcess):
def __init__(self) -> None:
super().__init__()
self.mutil_model = None
self.enable_media2text = False
self.is_mutil_model = False
self.asr_model = None
self.tts_model = None
async def load_default_config(self) -> bool:
return True
async def load_from_config(self, config: dict,is_load_default=True) -> Coroutine[Any, Any, bool]:
if is_load_default:
await self.load_default_config()
@@ -331,23 +335,26 @@ class AgentMessageProcess(LLMAgentBaseProcess):
if config.get("mutil_model"):
self.mutil_model = config.get("mutil_model")
self.asr_model = config.get("asr_model")
self.tts_model = config.get("tts_model")
def get_llm_model_name(self) -> str:
if self.is_mutil_model:
return self.mutil_model
else:
return self.model_name
def check_and_to_base64(self, image_path: str) -> str:
if image_utils.is_file(image_path):
return image_utils.to_base64(image_path, (1024, 1024))
else:
return image_path
async def get_prompt_from_msg(self,msg:AgentMsg) -> LLMPrompt:
msg_prompt = LLMPrompt()
self.is_mutil_model = False
if msg.is_image_msg():
if msg.is_image_msg():
if self.enable_media2text:
logger.error(f"enable_media2text is not supported yet")
else:
@@ -358,35 +365,56 @@ class AgentMessageProcess(LLMAgentBaseProcess):
content = [{"type": "text", "text": image_prompt}]
content.extend([{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images])
msg_prompt.messages = [{"role": "user", "content": content}]
if self.mutil_model:
self.is_mutil_model = True
else:
logger.warning(f"mutil_model is not set!")
elif msg.is_video_msg():
video_prompt, video = msg.get_video_body()
frames = video_utils.extract_frames(video, (1024, 1024))
if video_prompt is None:
msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": frame}} for frame in frames]}]
if self.enable_media2text:
logger.error(f"enable_media2text is not supported yet")
else:
content = [{"type": "text", "text": video_prompt}]
video_prompt, video = msg.get_video_body()
frames = video_utils.extract_frames(video, (1024, 1024))
audio_file = os.path.splitext(video)[0] + ".mp3"
video_utils.extract_audio(video, audio_file)
voice_content = None
if self.asr_model is not None:
resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, model=self.asr_model, prompt=None, response_format="text"))
if resp.result_code == ComputeTaskResultCode.OK:
voice_content = resp.result_str
content = []
if video_prompt is not None:
content.append({"type": "text", "text": video_prompt})
if voice_content is not None and voice_content != "":
content.append({"type": "text", "text": f"Voice content in video:{voice_content}"})
content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
msg_prompt.messages = [{"role": "user", "content": content}]
if self.mutil_model:
self.is_mutil_model = True
else:
logger.warning(f"mutil_model is not set!")
elif msg.is_audio_msg():
audio_file = msg.body
resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text"))
if resp.result_code != ComputeTaskResultCode.OK:
error_resp = msg.create_error_resp(resp.error_str)
return error_resp
if self.enable_media2text:
logger.error(f"enable_media2text is not supported yet")
else:
msg.body = resp.result_str
msg_prompt.messages = [{"role":"user","content":resp.result_str}]
audio_file = msg.body
resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, model=self.asr_model, prompt=None, response_format="text"))
if resp.result_code != ComputeTaskResultCode.OK:
error_resp = msg.create_error_resp(resp.error_str)
return error_resp
else:
msg.body = resp.result_str
msg_prompt.messages = [{"role":"user","content":resp.result_str}]
else:
msg_prompt.messages = [{"role":"user","content":msg.body}]
return msg_prompt
async def sender_info(self,msg:AgentMsg)->str:
sender_id = msg.sender
#TODO Is sender an agent?
@@ -400,14 +428,14 @@ class AgentMessageProcess(LLMAgentBaseProcess):
async def get_log_summary(self,msg:AgentMsg)->str:
return None
async def get_extend_known_info(self,msg:AgentMsg,prompt:LLMPrompt)->str:
return None
async def prepare_prompt(self,input:Dict) -> LLMPrompt:
prompt = LLMPrompt()
# User Prompt
# User Prompt
## Input Msg
msg : AgentMsg = input.get("msg")
context_info = input.get("context_info")
@@ -422,8 +450,8 @@ class AgentMessageProcess(LLMAgentBaseProcess):
## 通用的角色相关的系统提示词
system_prompt_dict = self.prepare_role_system_prompt(context_info)
## 已知信息
## 已知信息
known_info = {}
#prompt.append_system_message(self.known_info_tips)
### 信息发送者资料
@@ -442,23 +470,23 @@ class AgentMessageProcess(LLMAgentBaseProcess):
known_info["summary"] = summary
#prompt.append_system_message(await self.get_log_summary(self,msg))
system_prompt_dict["known_info"] = known_info
prompt.inner_functions =LLMProcessContext.aifunctions_to_inner_functions(self.llm_context.get_all_ai_functions())
if self.workspace:
#TODO eanble workspace functions?
logger.info(f"workspace is not none,enable workspace functions")
## 给予查询KB的权限
if self.enable_kb:
## 给予查询KB的权限
if self.enable_kb:
logger.info(f"enable kb")
prompt.append_system_message(json.dumps(system_prompt_dict,ensure_ascii=False))
## 扩展已知信息 (这可能是一个LLM过程)
prompt.append_system_message(await self.get_extend_known_info(msg,prompt))
return prompt
async def post_llm_process(self,actions:List[ActionNode],input:Dict,llm_result:LLMResult) -> bool:
msg:AgentMsg = input.get("msg")
@@ -466,14 +494,14 @@ class AgentMessageProcess(LLMAgentBaseProcess):
resp_msg = msg.create_group_resp_msg(self.memory.agent_id,llm_result.resp)
else:
resp_msg = msg.create_resp_msg(llm_result.resp)
llm_result.raw_result["_resp_msg"] = resp_msg
action_params = {}
action_params["_input"] = input
action_params["_memory"] = self.memory
action_params["_workspace"] = self.workspace
action_params["_resp_msg"] = resp_msg
action_params["_resp_msg"] = resp_msg
action_params["_llm_result"] = llm_result
action_params["_agentid"] = self.memory.agent_id
action_params["_start_at"] = datetime.now()
@@ -482,7 +510,7 @@ class AgentMessageProcess(LLMAgentBaseProcess):
chatsession = self.memory.get_session_from_msg(msg)
chatsession.append(msg)
chatsession.append(resp_msg)
chatsession.append(resp_msg)
return True
@@ -567,11 +595,11 @@ class AgentSelfThinking(LLMAgentBaseProcess):
record_list = input.get("record_list")
context_info = input.get("context_info")
if record_list is None:
logger.error(f"AgentSelfThinking prepare_prompt failed! input not found")
return None
prompt.append_user_message(json.dumps(record_list,ensure_ascii=False))
system_prompt_dict = self.prepare_role_system_prompt(context_info)
@@ -594,7 +622,7 @@ class AgentSelfThinking(LLMAgentBaseProcess):
if known_experience_list:
known_info["known_experience_list"] = known_experience_list
have_known_info = True
if have_known_info:
system_prompt_dict["known_info"] = known_info
@@ -626,7 +654,7 @@ class AgentSelfLearning(BaseLLMProcess):
async def prepare_prompt(self) -> LLMPrompt:
prompt = LLMPrompt()
pass
pass
async def get_inner_function_for_exec(self,func_name:str) -> AIFunction:
pass
@@ -636,7 +664,7 @@ class AgentSelfLearning(BaseLLMProcess):
class AgentSelfImprove(BaseLLMProcess):
def __init__(self) -> None:
super().__init__()
super().__init__()
src/aios/proto/compute_task.py
+12 -12
View File
@@ -80,16 +80,16 @@ class LLMPrompt:
def append_system_message(self,content:str):
if content is None:
return
if self.system_message is None:
self.system_message = {"role":"system","content":content}
else:
self.system_message["content"] += content
def append_user_message(self,content:str):
if content is None:
return
self.messages.append({"role":"user","content":content})
def as_str(self)->str:
@@ -109,13 +109,13 @@ class LLMPrompt:
result.append(self.system_message)
result.extend(self.messages)
return result
def append(self,prompt:'LLMPrompt'):
if prompt is None:
return
if prompt.inner_functions:
if self.inner_functions is None:
self.inner_functions = copy.deepcopy(prompt.inner_functions)
@@ -164,8 +164,8 @@ class LLMResult:
@classmethod
def from_error_str(self,error_str:str) -> 'LLMResult':
r = LLMResult()
r.state = "error"
r.compute_error_str = error_str
r.state = LLMResultStates.ERROR
r.error_str = error_str
return r
@classmethod
@@ -177,7 +177,7 @@ class LLMResult:
if llm_json_str == "**IGNORE**":
r.state = LLMResultStates.IGNORE
return r
r.state = LLMResultStates.OK
llm_json = json.loads(llm_json_str)
@@ -198,7 +198,7 @@ class LLMResult:
func_name = str_list[0]
params = str_list[1:]
return func_name, params
@classmethod
def from_str(self,llm_result_str:str,valid_func:List[str]=None) -> 'LLMResult':
r = LLMResult()
@@ -226,10 +226,10 @@ class LLMResult:
target_id = action_item.args[0]
msg_content = action_item.body
new_msg.set("",target_id,msg_content)
return True
return False
src/aios/utils/video_utils.py
+6
View File
@@ -3,6 +3,7 @@ from typing import List, Tuple
import cv2
import numpy as np
import moviepy.editor as mp
def precess_image(image):
@@ -120,3 +121,8 @@ def extract_frames(video_path: str, resize: Tuple[int, int] = None, smooth=False
i += 1
vidcap.release()
return frames
def extract_audio(video_path: str, audio_path: str):
my_clip = mp.VideoFileClip(video_path)
my_clip.audio.write_audiofile(audio_path)
src/component/common_environment/local_document.py
+29 -29
View File
@@ -13,6 +13,7 @@ import PyPDF2
import datetime
from typing import Optional, List
from aios import *
from aios.environment.workspace_env import TodoListEnvironment, TodoListType
from .local_file_system import FilesystemEnvironment
logger = logging.getLogger(__name__)
@@ -21,7 +22,7 @@ class MetaDatabase:
def __init__(self,db_path:str):
self.db_path = db_path
self._get_conn()
def _get_conn(self):
""" get db connection """
local = threading.local()
@@ -43,7 +44,7 @@ class MetaDatabase:
self._create_tables(conn)
return conn
def _create_tables(self,conn):
cursor = conn.cursor()
cursor.execute('''
@@ -68,7 +69,7 @@ class MetaDatabase:
create_time TEXT
)
''')
cursor.execute('''
CREATE INDEX IF NOT EXISTS idx_documents_doc_hash
ON documents (doc_hash)
@@ -110,7 +111,7 @@ class MetaDatabase:
WHERE doc_path = ?
''', (doc_hash, doc_path))
conn.commit()
def get_docs_without_hash(self,limit:int=1024) -> List[str]:
conn = self._get_conn()
cursor = conn.cursor()
@@ -186,7 +187,7 @@ class MetaDatabase:
row = cursor.fetchone()
if row is None:
return None
# get doc path
cursor.execute('''
SELECT doc_path
@@ -197,7 +198,7 @@ class MetaDatabase:
if row2 is None:
return None
doc_path = row2[0]
return {
"full_path": doc_path,
@@ -261,7 +262,7 @@ class LearningCache:
def remove(self, key):
with self.cache_lock:
return self.cache.pop(key, None)
class LocalKnowledgeBase(CompositeEnvironment):
def __init__(self, workspace: str) -> None:
@@ -275,10 +276,10 @@ class LocalKnowledgeBase(CompositeEnvironment):
async def learn(op:dict):
full_path = op.get("original_path")
if not full_path:
return
return
meta = self.learning_cache.get(full_path)
meta.update(op)
self.add_ai_operation(SimpleAIAction(
op="learn",
description="update knowledge llm summary",
@@ -287,16 +288,16 @@ class LocalKnowledgeBase(CompositeEnvironment):
self.fs = FilesystemEnvironment(self.root_path)
self.add_env(self.fs)
async def get_knowledege_catalog(self,path:str=None,only_dir =True,max_depth:int=5)->str:
if path:
full_path = f"{self.root_path}/{path}"
else:
full_path = self.root_path
catlogs,file_count = await self.get_directory_structure(full_path,max_depth,only_dir)
return catlogs
async def get_directory_structure(self,root_dir, max_depth:int=4, only_dir=True, indent=1):
file_count = 0
structure_str = ''
@@ -315,11 +316,11 @@ class LocalKnowledgeBase(CompositeEnvironment):
if only_dir is False:
for file_name in sub_files:
structure_str = structure_str + ' ' * (indent+1) + file_name + '\n'
structure_str = structure_str + ' ' * (indent+1) + file_name + '\n'
dir_name = os.path.basename(root_dir)
dir_info = f"{dir_name} <count: {file_count}>"
structure_str = ' ' * indent + dir_info + '\n' + structure_str
@@ -328,7 +329,7 @@ class LocalKnowledgeBase(CompositeEnvironment):
else:
return structure_str, file_count
# inner_function
# inner_function
async def get_knowledge_meta(self,path:str) -> str:
full_path = f"{self.root_path}/{path}"
if os.islink(full_path):
@@ -336,9 +337,9 @@ class LocalKnowledgeBase(CompositeEnvironment):
hash = self.meta_db.get_hash_by_doc_path(org_path)
if hash:
return self.meta_db.get_knowledge(org_path)
return "not found"
async def load_knowledge_content(self,path:str,pos:int=0,length:int=None) -> str:
if path.endswith("pdf"):
logger.info("load_knowledge_content:pdf")
@@ -367,12 +368,12 @@ class ScanLocalDocument:
workspace = string.Template(config["workspace"]).substitute(myai_dir=AIStorage.get_instance().get_myai_dir())
path = string.Template(config["path"]).substitute(myai_dir=AIStorage.get_instance().get_myai_dir())
self.knowledge_base = LocalKnowledgeBase(workspace)
self.path = path
self.path = path
def _support_file(self,file_name:str) -> bool:
if file_name.startswith("."):
return False
if file_name.endswith(".pdf"):
return True
if file_name.endswith(".md"):
@@ -380,7 +381,7 @@ class ScanLocalDocument:
if file_name.endswith(".txt"):
return True
return False
async def next(self):
while True:
for root, dirs, files in os.walk(self.path):
@@ -391,10 +392,10 @@ class ScanLocalDocument:
if self.knowledge_base.meta_db.is_doc_exist(full_path):
continue
yield(full_path, full_path)
else:
else:
continue
yield(None, None)
class ParseLocalDocument:
@@ -425,7 +426,7 @@ class ParseLocalDocument:
await self.knowledge_base.fs.symlink(full_path, new_path)
logger.info(f"create soft link {full_path} -> {new_path}")
return full_path
async def _get_meta_prompt(self,meta: dict,temp_meta = None,need_catalogs = False) -> str:
kb_tree = await self.knowledge_base.get_knowledege_catalog()
@@ -473,7 +474,7 @@ class ParseLocalDocument:
full_content_len = self._token_len(full_content)
full_path = meta["original_path"]
self.knowledge_base.learning_cache.add(full_path, meta)
if full_content_len < self.token_limit:
# 短文章不用总结catalog
@@ -521,7 +522,7 @@ class ParseLocalDocument:
if item.title:
new_item = {}
new_item["page"] = item.page.idnum
new_item["title"] = item.title
new_item["title"] = item.title
my_childs = []
if item.childs:
if len(item.childs) > 0:
@@ -573,7 +574,7 @@ class ParseLocalDocument:
return {}
def _parse_md(self,doc_path:str):
metadata = {}
metadata = {}
cur_encode = "utf-8"
with open(doc_path,'rb') as f:
cur_encode = chardet.detect(f.read(1024))['encoding']
@@ -588,7 +589,7 @@ class ParseLocalDocument:
toc = md.toc
if toc:
metadata['catalogs'] = toc
return metadata
def _parse_document(self,doc_path:str):
@@ -614,5 +615,4 @@ class ParseLocalDocument:
meta_data["title"] = title
logger.info("parse document %s!",doc_path)
return hash_result, meta_data
src/component/openai_node/open_ai_node.py
+6 -4
View File
@@ -216,14 +216,16 @@ class OpenAI_ComputeNode(ComputeNode):
client = AsyncOpenAI(api_key=self.openai_api_key)
try:
if llm_inner_functions is None or len(llm_inner_functions) == 0:
logger.info(f"call openai {mode_name} prompts: {prompts}")
if mode_name != "gpt-4-vision-preview":
logger.info(f"call openai {mode_name} prompts: {prompts}")
resp = await client.chat.completions.create(model=mode_name,
messages=prompts,
response_format = response_format,
max_tokens=result_token,
)
else:
logger.info(f"call openai {mode_name} prompts: \n\t {prompts} \nfunctions: \n\t{json.dumps(llm_inner_functions,ensure_ascii=False)}")
if mode_name != "gpt-4-vision-preview":
logger.info(f"call openai {mode_name} prompts: \n\t {prompts} \nfunctions: \n\t{json.dumps(llm_inner_functions,ensure_ascii=False)}")
resp = await client.chat.completions.create(model=mode_name,
messages=prompts,
response_format = response_format,
@@ -239,7 +241,7 @@ class OpenAI_ComputeNode(ComputeNode):
#logger.info(f"openai response: {resp}")
#TODO: gpt-4v api is image_2_text ?
if mode_name == "gpt-4-vision-preview":
if mode_name == "gpt-4-vision-preview":
status_code = resp.choices[0].finish_reason
if status_code is None:
status_code = resp.choices[0].finish_details['type']
@@ -267,7 +269,7 @@ class OpenAI_ComputeNode(ComputeNode):
if token_usage:
result.result_refers["token_usage"] = token_usage
logger.info(f"openai success response: {result.result_str}")
return result
case _:
src/component/slack_tunnel.py
+1
View File
@@ -119,6 +119,7 @@ class SlackTunnel(AgentTunnel):
continue
await download_file(file_info["file"]["url_private_download"], file_path, self.token)
mime_type = file["mimetype"]
if file["mimetype"].startswith("image/"):
if file_type is None:
file_type = "image"
src/requirements.txt
+1
View File
@@ -156,3 +156,4 @@ opencv-python
discord.py
slack_bolt
wget
moviepy