diff --git a/rootfs/agents/Jarvis/agent.toml b/rootfs/agents/Jarvis/agent.toml index cb0e1c7..129d7f3 100644 --- a/rootfs/agents/Jarvis/agent.toml +++ b/rootfs/agents/Jarvis/agent.toml @@ -16,6 +16,8 @@ Only clearly specifying the task you completed can be completed independently. type="AgentMessageProcess" # TODO: 是否应该自动记录 inner function和action的执行细节 mutil_model="gpt-4-vision-preview" +asr_model="openai-whisper" +tts_model="tts-1" process_description=""" 1. Based on your role and the existing information, please think and then make a brief and efficient reply. @@ -30,7 +32,7 @@ The Response must be directly parsed by `python json.loads`. Here is an example: { think:'$think step-by-step to be sure you have the right reply.' resp: '$What you want to reply', - actions: [{ + actions: [{ name: '$action_name', $param_name: '$parm' #Optional, fill in only if the action has parameters. }] @@ -63,7 +65,7 @@ The Response must be directly parsed by `python json.loads`. Here is an example: { think:'$think step-by-step to be sure you can triage tasks well.' resp : '$determine, summary what you do', - actions: [{ + actions: [{ name: '$action_name', $param_name: '$parm' #Optional, fill in only if the action has parameters. }] @@ -89,7 +91,7 @@ The Response must be directly parsed by `python json.loads`. Here is an example: { think:'$thinking step by step to ensure the accurate and efficient processing task.', resp:'$determine, summary what you do' - actions: [{ + actions: [{ name: '$action_name', $param_name: '$parm' #Optional, fill in only if the action has parameters. }] @@ -114,7 +116,7 @@ The Response must be directly parsed by `python json.loads`. Here is an example: { think:'$think step-by-step to be sure you have the right result.', resp : '$determine, summary what you will do', - actions: [{ + actions: [{ name: '$action_name', $param_name: '$parm' #Optional, fill in only if the action has parameters. }] @@ -124,11 +126,11 @@ The Response must be directly parsed by `python json.loads`. Here is an example: llm_context.actions.enable = ["agent.workspace.cancel_task","agent.workspace.update_task"] context="Your Principal now in {location}, time: {now}, weather: {weather}." -[behavior.do] +[behavior.do] # do TODO type="AgentDo" process_description=""" -The input is a TODO comes from a Task. +The input is a TODO comes from a Task. 1. Your task is to combine your role definition, tools on hand, known information, and complete a certain Todo.After completing the Todo, you will get a tip of $ 200. 2. 8000 word limit for short term memory. Your short term memory is short, so immediately save important information to files. 3. In the process of completing Todo, you should think first and then execute. During the execution, you can use functions to access the results of the front steps. @@ -141,10 +143,10 @@ The Response must be directly parsed by `python json.loads`. Here is an example: { think:'$think step by step, how to complete the todo', resp: '$simport report about what you do', - actions: [{ + actions: [{ name: '$action1_name', $param_name: '$parm' #Optional, fill in only if the action has parameters. - }, ... + }, ... ] } """ @@ -168,7 +170,7 @@ The Response must be directly parsed by `python json.loads`. Here is an example: resp:'$think step by step, how to check the todo', name: '$action1_name', $param_name: '$parm' #Optional, fill in only if the action has parameters. - }, ... + }, ... ] } """ @@ -197,7 +199,7 @@ The Response must be directly parsed by `python json.loads`. Here is an example: resp:'$Summary in one sentence', name: '$action1_name', $param_name: '$parm' #Optional, fill in only if the action has parameters. - }, ... + }, ... ] } """ diff --git a/src/aios/agent/llm_process.py b/src/aios/agent/llm_process.py index 34fdfe7..e90c7ec 100644 --- a/src/aios/agent/llm_process.py +++ b/src/aios/agent/llm_process.py @@ -1,5 +1,7 @@ # Old name is behavior, I belive new name "llm_process" is better # pylint:disable=E0402 +import os.path + from ..utils import video_utils,image_utils from ..proto.compute_task import LLMPrompt,LLMResult,ComputeTaskResult,ComputeTaskResultCode @@ -31,11 +33,11 @@ class BaseLLMProcess(ABC): self.goal:str = None #目标 self.input_example:str= None #输入样例 self.result_example:str = None #llm_result样例 - + self.enable_json_resp = False #None means system default, # TODO: support abcstract model name like: local-hight,local-low,local-medium,remote-hight,remote-low,remote-medium - self.model_name = None + self.model_name = None self.max_token = 1000 # result_token self.max_prompt_token = 1000 # not include input prompt self.timeout = 1800 # 30 min @@ -55,8 +57,8 @@ class BaseLLMProcess(ABC): @abstractmethod def prepare_inner_function_context_for_exec(self,inner_func_name:str,parameters:Dict): - return - + return + @abstractmethod async def post_llm_process(self,actions:List[ActionNode],input:Dict,llm_result:LLMResult) -> bool: pass @@ -76,14 +78,14 @@ class BaseLLMProcess(ABC): self.max_token = config.get("max_token") if config.get("timeout"): self.timeout = config.get("timeout") - + return True - + @abstractmethod async def initial(self,params:Dict = None) -> bool: pass - + def _format_content_by_env_value(self,content:str,env)->str: return content.format_map(env) @@ -120,12 +122,12 @@ class BaseLLMProcess(ABC): task_result.result_code = ComputeTaskResultCode.ERROR task_result.error_str = f"prompt too long,can not predict" return task_result - + if stack_limit > 0: inner_functions=prompt.inner_functions else: inner_functions = None - + task_result: ComputeTaskResult = await (ComputeKernel.get_instance().do_llm_completion( prompt, @@ -140,7 +142,7 @@ class BaseLLMProcess(ABC): return task_result inner_func_call_node = None - + result_message : dict = task_result.result.get("message") if result_message: inner_func_call_node = result_message.get("function_call") @@ -166,7 +168,7 @@ class BaseLLMProcess(ABC): max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.model_name) #if max_result_token < MIN_PREDICT_TOKEN_LEN: # return LLMResult.from_error_str(f"prompt too long,can not predict") - + task_result: ComputeTaskResult = await (ComputeKernel.get_instance().do_llm_completion( prompt, resp_mode=resp_mode, @@ -174,12 +176,12 @@ class BaseLLMProcess(ABC): max_token=max_result_token, inner_functions=prompt.inner_functions, #NOTICE: inner_function in prompt can be a subset of get_inner_function timeout=self.timeout)) - + if task_result.result_code != ComputeTaskResultCode.OK: err_str = f"do_llm_completion error:{task_result.error_str}" logger.error(err_str) return LLMResult.from_error_str(err_str) - + result_message = task_result.result.get("message") inner_func_call_node = None if result_message: @@ -202,7 +204,7 @@ class BaseLLMProcess(ABC): await self.post_llm_process(llm_result.action_list,input,llm_result) return llm_result - + class LLMAgentBaseProcess(BaseLLMProcess): def __init__(self) -> None: super().__init__() @@ -211,11 +213,11 @@ class LLMAgentBaseProcess(BaseLLMProcess): self.process_description:str = None self.reply_format:str = None self.context : str = None - + self.workspace : AgentWorkspace = None # If Workspace is not none , enable Agent Tasklist self.memory : AgentMemory = None self.enable_kb : bool = False - self.kb = None + self.kb = None async def initial(self,params:Dict = None) -> bool: self.memory = params.get("memory") @@ -227,23 +229,23 @@ class LLMAgentBaseProcess(BaseLLMProcess): return True async def load_default_config(self) -> bool: return True - - + + async def load_from_config(self, config: dict,is_load_default=True) -> Coroutine[Any, Any, bool]: if is_load_default: await self.load_default_config() if await super().load_from_config(config) is False: return False - + self.role_description = config.get("role_desc") if self.role_description is None: logger.error(f"role_description not found in config") return False - + if config.get("process_description"): self.process_description = config.get("process_description") - + if config.get("reply_format"): self.reply_format = config.get("reply_format") @@ -282,7 +284,7 @@ class LLMAgentBaseProcess(BaseLLMProcess): return system_prompt_dict def prepare_inner_function_context_for_exec(self,inner_func_name:str,parameters:Dict): - parameters["_workspace"] = self.workspace + parameters["_workspace"] = self.workspace def get_action_desc(self) -> Dict: result = {} @@ -290,17 +292,17 @@ class LLMAgentBaseProcess(BaseLLMProcess): for action in actions_list: result[action.get_name()] = action.get_description() return result - + async def get_inner_function_for_exec(self,func_name:str) -> AIFunction: return self.llm_context.get_ai_function(func_name) - + async def _execute_actions(self,actions:List[ActionNode],action_params:Dict): for action_item in actions: op : AIAction = self.llm_context.get_ai_action(action_item.name) if op: if action_item.parms is None: action_item.parms = {} - + real_parms = {**action_params,**action_item.parms} action_item.parms["_result"] = await op.execute(real_parms) @@ -309,17 +311,19 @@ class LLMAgentBaseProcess(BaseLLMProcess): logger.warn(f"action {action_item.name} not found") return False - + class AgentMessageProcess(LLMAgentBaseProcess): def __init__(self) -> None: super().__init__() self.mutil_model = None self.enable_media2text = False self.is_mutil_model = False + self.asr_model = None + self.tts_model = None async def load_default_config(self) -> bool: return True - + async def load_from_config(self, config: dict,is_load_default=True) -> Coroutine[Any, Any, bool]: if is_load_default: await self.load_default_config() @@ -331,23 +335,26 @@ class AgentMessageProcess(LLMAgentBaseProcess): if config.get("mutil_model"): self.mutil_model = config.get("mutil_model") - + + self.asr_model = config.get("asr_model") + self.tts_model = config.get("tts_model") + def get_llm_model_name(self) -> str: if self.is_mutil_model: return self.mutil_model else: return self.model_name - + def check_and_to_base64(self, image_path: str) -> str: if image_utils.is_file(image_path): return image_utils.to_base64(image_path, (1024, 1024)) else: return image_path - + async def get_prompt_from_msg(self,msg:AgentMsg) -> LLMPrompt: msg_prompt = LLMPrompt() self.is_mutil_model = False - if msg.is_image_msg(): + if msg.is_image_msg(): if self.enable_media2text: logger.error(f"enable_media2text is not supported yet") else: @@ -358,35 +365,56 @@ class AgentMessageProcess(LLMAgentBaseProcess): content = [{"type": "text", "text": image_prompt}] content.extend([{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images]) msg_prompt.messages = [{"role": "user", "content": content}] - + if self.mutil_model: self.is_mutil_model = True else: logger.warning(f"mutil_model is not set!") - + elif msg.is_video_msg(): - video_prompt, video = msg.get_video_body() - frames = video_utils.extract_frames(video, (1024, 1024)) - if video_prompt is None: - msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": frame}} for frame in frames]}] + if self.enable_media2text: + logger.error(f"enable_media2text is not supported yet") else: - content = [{"type": "text", "text": video_prompt}] + video_prompt, video = msg.get_video_body() + frames = video_utils.extract_frames(video, (1024, 1024)) + audio_file = os.path.splitext(video)[0] + ".mp3" + video_utils.extract_audio(video, audio_file) + + voice_content = None + if self.asr_model is not None: + resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, model=self.asr_model, prompt=None, response_format="text")) + if resp.result_code == ComputeTaskResultCode.OK: + voice_content = resp.result_str + + content = [] + if video_prompt is not None: + content.append({"type": "text", "text": video_prompt}) + if voice_content is not None and voice_content != "": + content.append({"type": "text", "text": f"Voice content in video:{voice_content}"}) + content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames]) msg_prompt.messages = [{"role": "user", "content": content}] + if self.mutil_model: + self.is_mutil_model = True + else: + logger.warning(f"mutil_model is not set!") elif msg.is_audio_msg(): - audio_file = msg.body - resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text")) - if resp.result_code != ComputeTaskResultCode.OK: - error_resp = msg.create_error_resp(resp.error_str) - return error_resp + if self.enable_media2text: + logger.error(f"enable_media2text is not supported yet") else: - msg.body = resp.result_str - msg_prompt.messages = [{"role":"user","content":resp.result_str}] + audio_file = msg.body + resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, model=self.asr_model, prompt=None, response_format="text")) + if resp.result_code != ComputeTaskResultCode.OK: + error_resp = msg.create_error_resp(resp.error_str) + return error_resp + else: + msg.body = resp.result_str + msg_prompt.messages = [{"role":"user","content":resp.result_str}] else: msg_prompt.messages = [{"role":"user","content":msg.body}] return msg_prompt - + async def sender_info(self,msg:AgentMsg)->str: sender_id = msg.sender #TODO Is sender an agent? @@ -400,14 +428,14 @@ class AgentMessageProcess(LLMAgentBaseProcess): async def get_log_summary(self,msg:AgentMsg)->str: return None - + async def get_extend_known_info(self,msg:AgentMsg,prompt:LLMPrompt)->str: return None async def prepare_prompt(self,input:Dict) -> LLMPrompt: prompt = LLMPrompt() - # User Prompt + # User Prompt ## Input Msg msg : AgentMsg = input.get("msg") context_info = input.get("context_info") @@ -422,8 +450,8 @@ class AgentMessageProcess(LLMAgentBaseProcess): ## 通用的角色相关的系统提示词 system_prompt_dict = self.prepare_role_system_prompt(context_info) - - ## 已知信息 + + ## 已知信息 known_info = {} #prompt.append_system_message(self.known_info_tips) ### 信息发送者资料 @@ -442,23 +470,23 @@ class AgentMessageProcess(LLMAgentBaseProcess): known_info["summary"] = summary #prompt.append_system_message(await self.get_log_summary(self,msg)) system_prompt_dict["known_info"] = known_info - + prompt.inner_functions =LLMProcessContext.aifunctions_to_inner_functions(self.llm_context.get_all_ai_functions()) if self.workspace: #TODO eanble workspace functions? logger.info(f"workspace is not none,enable workspace functions") - ## 给予查询KB的权限 - if self.enable_kb: + ## 给予查询KB的权限 + if self.enable_kb: logger.info(f"enable kb") - + prompt.append_system_message(json.dumps(system_prompt_dict,ensure_ascii=False)) ## 扩展已知信息 (这可能是一个LLM过程) prompt.append_system_message(await self.get_extend_known_info(msg,prompt)) return prompt - + async def post_llm_process(self,actions:List[ActionNode],input:Dict,llm_result:LLMResult) -> bool: msg:AgentMsg = input.get("msg") @@ -466,14 +494,14 @@ class AgentMessageProcess(LLMAgentBaseProcess): resp_msg = msg.create_group_resp_msg(self.memory.agent_id,llm_result.resp) else: resp_msg = msg.create_resp_msg(llm_result.resp) - + llm_result.raw_result["_resp_msg"] = resp_msg action_params = {} action_params["_input"] = input action_params["_memory"] = self.memory action_params["_workspace"] = self.workspace - action_params["_resp_msg"] = resp_msg + action_params["_resp_msg"] = resp_msg action_params["_llm_result"] = llm_result action_params["_agentid"] = self.memory.agent_id action_params["_start_at"] = datetime.now() @@ -482,7 +510,7 @@ class AgentMessageProcess(LLMAgentBaseProcess): chatsession = self.memory.get_session_from_msg(msg) chatsession.append(msg) - chatsession.append(resp_msg) + chatsession.append(resp_msg) return True @@ -567,11 +595,11 @@ class AgentSelfThinking(LLMAgentBaseProcess): record_list = input.get("record_list") context_info = input.get("context_info") - + if record_list is None: logger.error(f"AgentSelfThinking prepare_prompt failed! input not found") return None - + prompt.append_user_message(json.dumps(record_list,ensure_ascii=False)) system_prompt_dict = self.prepare_role_system_prompt(context_info) @@ -594,7 +622,7 @@ class AgentSelfThinking(LLMAgentBaseProcess): if known_experience_list: known_info["known_experience_list"] = known_experience_list have_known_info = True - + if have_known_info: system_prompt_dict["known_info"] = known_info @@ -626,7 +654,7 @@ class AgentSelfLearning(BaseLLMProcess): async def prepare_prompt(self) -> LLMPrompt: prompt = LLMPrompt() - pass + pass async def get_inner_function_for_exec(self,func_name:str) -> AIFunction: pass @@ -636,7 +664,7 @@ class AgentSelfLearning(BaseLLMProcess): class AgentSelfImprove(BaseLLMProcess): def __init__(self) -> None: - super().__init__() + super().__init__() diff --git a/src/aios/proto/compute_task.py b/src/aios/proto/compute_task.py index 3f241f0..e483d0e 100644 --- a/src/aios/proto/compute_task.py +++ b/src/aios/proto/compute_task.py @@ -80,16 +80,16 @@ class LLMPrompt: def append_system_message(self,content:str): if content is None: return - + if self.system_message is None: self.system_message = {"role":"system","content":content} else: self.system_message["content"] += content - + def append_user_message(self,content:str): if content is None: return - + self.messages.append({"role":"user","content":content}) def as_str(self)->str: @@ -109,13 +109,13 @@ class LLMPrompt: result.append(self.system_message) result.extend(self.messages) return result - - + + def append(self,prompt:'LLMPrompt'): if prompt is None: return - + if prompt.inner_functions: if self.inner_functions is None: self.inner_functions = copy.deepcopy(prompt.inner_functions) @@ -164,8 +164,8 @@ class LLMResult: @classmethod def from_error_str(self,error_str:str) -> 'LLMResult': r = LLMResult() - r.state = "error" - r.compute_error_str = error_str + r.state = LLMResultStates.ERROR + r.error_str = error_str return r @classmethod @@ -177,7 +177,7 @@ class LLMResult: if llm_json_str == "**IGNORE**": r.state = LLMResultStates.IGNORE return r - + r.state = LLMResultStates.OK llm_json = json.loads(llm_json_str) @@ -198,7 +198,7 @@ class LLMResult: func_name = str_list[0] params = str_list[1:] return func_name, params - + @classmethod def from_str(self,llm_result_str:str,valid_func:List[str]=None) -> 'LLMResult': r = LLMResult() @@ -226,10 +226,10 @@ class LLMResult: target_id = action_item.args[0] msg_content = action_item.body new_msg.set("",target_id,msg_content) - + return True - + return False diff --git a/src/aios/utils/video_utils.py b/src/aios/utils/video_utils.py index dd687cf..15cbb4e 100644 --- a/src/aios/utils/video_utils.py +++ b/src/aios/utils/video_utils.py @@ -3,6 +3,7 @@ from typing import List, Tuple import cv2 import numpy as np +import moviepy.editor as mp def precess_image(image): @@ -120,3 +121,8 @@ def extract_frames(video_path: str, resize: Tuple[int, int] = None, smooth=False i += 1 vidcap.release() return frames + + +def extract_audio(video_path: str, audio_path: str): + my_clip = mp.VideoFileClip(video_path) + my_clip.audio.write_audiofile(audio_path) diff --git a/src/component/common_environment/local_document.py b/src/component/common_environment/local_document.py index bfb3f27..8929d94 100644 --- a/src/component/common_environment/local_document.py +++ b/src/component/common_environment/local_document.py @@ -13,6 +13,7 @@ import PyPDF2 import datetime from typing import Optional, List from aios import * +from aios.environment.workspace_env import TodoListEnvironment, TodoListType from .local_file_system import FilesystemEnvironment logger = logging.getLogger(__name__) @@ -21,7 +22,7 @@ class MetaDatabase: def __init__(self,db_path:str): self.db_path = db_path self._get_conn() - + def _get_conn(self): """ get db connection """ local = threading.local() @@ -43,7 +44,7 @@ class MetaDatabase: self._create_tables(conn) return conn - + def _create_tables(self,conn): cursor = conn.cursor() cursor.execute(''' @@ -68,7 +69,7 @@ class MetaDatabase: create_time TEXT ) ''') - + cursor.execute(''' CREATE INDEX IF NOT EXISTS idx_documents_doc_hash ON documents (doc_hash) @@ -110,7 +111,7 @@ class MetaDatabase: WHERE doc_path = ? ''', (doc_hash, doc_path)) conn.commit() - + def get_docs_without_hash(self,limit:int=1024) -> List[str]: conn = self._get_conn() cursor = conn.cursor() @@ -186,7 +187,7 @@ class MetaDatabase: row = cursor.fetchone() if row is None: return None - + # get doc path cursor.execute(''' SELECT doc_path @@ -197,7 +198,7 @@ class MetaDatabase: if row2 is None: return None doc_path = row2[0] - + return { "full_path": doc_path, @@ -261,7 +262,7 @@ class LearningCache: def remove(self, key): with self.cache_lock: return self.cache.pop(key, None) - + class LocalKnowledgeBase(CompositeEnvironment): def __init__(self, workspace: str) -> None: @@ -275,10 +276,10 @@ class LocalKnowledgeBase(CompositeEnvironment): async def learn(op:dict): full_path = op.get("original_path") if not full_path: - return + return meta = self.learning_cache.get(full_path) meta.update(op) - + self.add_ai_operation(SimpleAIAction( op="learn", description="update knowledge llm summary", @@ -287,16 +288,16 @@ class LocalKnowledgeBase(CompositeEnvironment): self.fs = FilesystemEnvironment(self.root_path) self.add_env(self.fs) - + async def get_knowledege_catalog(self,path:str=None,only_dir =True,max_depth:int=5)->str: if path: full_path = f"{self.root_path}/{path}" else: full_path = self.root_path - + catlogs,file_count = await self.get_directory_structure(full_path,max_depth,only_dir) return catlogs - + async def get_directory_structure(self,root_dir, max_depth:int=4, only_dir=True, indent=1): file_count = 0 structure_str = '' @@ -315,11 +316,11 @@ class LocalKnowledgeBase(CompositeEnvironment): if only_dir is False: for file_name in sub_files: - structure_str = structure_str + ' ' * (indent+1) + file_name + '\n' + structure_str = structure_str + ' ' * (indent+1) + file_name + '\n' dir_name = os.path.basename(root_dir) dir_info = f"{dir_name} " - + structure_str = ' ' * indent + dir_info + '\n' + structure_str @@ -328,7 +329,7 @@ class LocalKnowledgeBase(CompositeEnvironment): else: return structure_str, file_count - # inner_function + # inner_function async def get_knowledge_meta(self,path:str) -> str: full_path = f"{self.root_path}/{path}" if os.islink(full_path): @@ -336,9 +337,9 @@ class LocalKnowledgeBase(CompositeEnvironment): hash = self.meta_db.get_hash_by_doc_path(org_path) if hash: return self.meta_db.get_knowledge(org_path) - + return "not found" - + async def load_knowledge_content(self,path:str,pos:int=0,length:int=None) -> str: if path.endswith("pdf"): logger.info("load_knowledge_content:pdf") @@ -367,12 +368,12 @@ class ScanLocalDocument: workspace = string.Template(config["workspace"]).substitute(myai_dir=AIStorage.get_instance().get_myai_dir()) path = string.Template(config["path"]).substitute(myai_dir=AIStorage.get_instance().get_myai_dir()) self.knowledge_base = LocalKnowledgeBase(workspace) - self.path = path + self.path = path def _support_file(self,file_name:str) -> bool: if file_name.startswith("."): return False - + if file_name.endswith(".pdf"): return True if file_name.endswith(".md"): @@ -380,7 +381,7 @@ class ScanLocalDocument: if file_name.endswith(".txt"): return True return False - + async def next(self): while True: for root, dirs, files in os.walk(self.path): @@ -391,10 +392,10 @@ class ScanLocalDocument: if self.knowledge_base.meta_db.is_doc_exist(full_path): continue yield(full_path, full_path) - else: + else: continue yield(None, None) - + class ParseLocalDocument: @@ -425,7 +426,7 @@ class ParseLocalDocument: await self.knowledge_base.fs.symlink(full_path, new_path) logger.info(f"create soft link {full_path} -> {new_path}") return full_path - + async def _get_meta_prompt(self,meta: dict,temp_meta = None,need_catalogs = False) -> str: kb_tree = await self.knowledge_base.get_knowledege_catalog() @@ -473,7 +474,7 @@ class ParseLocalDocument: full_content_len = self._token_len(full_content) full_path = meta["original_path"] self.knowledge_base.learning_cache.add(full_path, meta) - + if full_content_len < self.token_limit: # 短文章不用总结catalog @@ -521,7 +522,7 @@ class ParseLocalDocument: if item.title: new_item = {} new_item["page"] = item.page.idnum - new_item["title"] = item.title + new_item["title"] = item.title my_childs = [] if item.childs: if len(item.childs) > 0: @@ -573,7 +574,7 @@ class ParseLocalDocument: return {} def _parse_md(self,doc_path:str): - metadata = {} + metadata = {} cur_encode = "utf-8" with open(doc_path,'rb') as f: cur_encode = chardet.detect(f.read(1024))['encoding'] @@ -588,7 +589,7 @@ class ParseLocalDocument: toc = md.toc if toc: metadata['catalogs'] = toc - + return metadata def _parse_document(self,doc_path:str): @@ -614,5 +615,4 @@ class ParseLocalDocument: meta_data["title"] = title logger.info("parse document %s!",doc_path) return hash_result, meta_data - - \ No newline at end of file + diff --git a/src/component/openai_node/open_ai_node.py b/src/component/openai_node/open_ai_node.py index 3695696..63849a1 100644 --- a/src/component/openai_node/open_ai_node.py +++ b/src/component/openai_node/open_ai_node.py @@ -216,14 +216,16 @@ class OpenAI_ComputeNode(ComputeNode): client = AsyncOpenAI(api_key=self.openai_api_key) try: if llm_inner_functions is None or len(llm_inner_functions) == 0: - logger.info(f"call openai {mode_name} prompts: {prompts}") + if mode_name != "gpt-4-vision-preview": + logger.info(f"call openai {mode_name} prompts: {prompts}") resp = await client.chat.completions.create(model=mode_name, messages=prompts, response_format = response_format, max_tokens=result_token, ) else: - logger.info(f"call openai {mode_name} prompts: \n\t {prompts} \nfunctions: \n\t{json.dumps(llm_inner_functions,ensure_ascii=False)}") + if mode_name != "gpt-4-vision-preview": + logger.info(f"call openai {mode_name} prompts: \n\t {prompts} \nfunctions: \n\t{json.dumps(llm_inner_functions,ensure_ascii=False)}") resp = await client.chat.completions.create(model=mode_name, messages=prompts, response_format = response_format, @@ -239,7 +241,7 @@ class OpenAI_ComputeNode(ComputeNode): #logger.info(f"openai response: {resp}") #TODO: gpt-4v api is image_2_text ? - if mode_name == "gpt-4-vision-preview": + if mode_name == "gpt-4-vision-preview": status_code = resp.choices[0].finish_reason if status_code is None: status_code = resp.choices[0].finish_details['type'] @@ -267,7 +269,7 @@ class OpenAI_ComputeNode(ComputeNode): if token_usage: result.result_refers["token_usage"] = token_usage - + logger.info(f"openai success response: {result.result_str}") return result case _: diff --git a/src/component/slack_tunnel.py b/src/component/slack_tunnel.py index 7deda23..52a5b63 100644 --- a/src/component/slack_tunnel.py +++ b/src/component/slack_tunnel.py @@ -119,6 +119,7 @@ class SlackTunnel(AgentTunnel): continue await download_file(file_info["file"]["url_private_download"], file_path, self.token) + mime_type = file["mimetype"] if file["mimetype"].startswith("image/"): if file_type is None: file_type = "image" diff --git a/src/requirements.txt b/src/requirements.txt index ed743c4..656343e 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -156,3 +156,4 @@ opencv-python discord.py slack_bolt wget +moviepy