gyt/opendan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mime

Browse Source
This commit is contained in:
wugren
2023-12-01 14:22:34 +08:00
parent eb67980537
commit 9cf4613d31
11 changed files with 206 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/aios/agent/agent.py
+49 -2
View File
@@ -27,6 +27,7 @@ from ..environment.workspace_env import WorkspaceEnvironment
from ..storage.storage import AIStorage
from ..knowledge import *
from . import video_utils, image_utils
logger = logging.getLogger(__name__)
@@ -423,11 +424,39 @@ class AIAgent(BaseAIAgent):
async def _create_openai_thread(self) -> str:
return None
def check_and_to_base64(self, image_path: str) -> str:
if image_utils.is_file(image_path):
return image_utils.image_to_base64(image_path)
else:
return image_path
async def _process_msg(self,msg:AgentMsg,workspace = None) -> AgentMsg:
msg_prompt = AgentPrompt()
if msg.msg_type == AgentMsgType.TYPE_GROUPMSG:
need_process = False
msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{msg.body}"}]
if msg.is_image_msg():
image_prompt, images = msg.get_image_body()
if image_prompt is None:
content = [[{"type": "text", "text": f"{msg.sender}'s message"}]]
content.extend([{"type": "image_url", "url": self.check_and_to_base64(image)} for image in images])
msg_prompt.messages = [{"role": "user", "content": content}]
else:
content = [{"type": "text", "text": f"{msg.sender}:{image_prompt}"}]
content.extend([{"type": "image_url", "url": self.check_and_to_base64(image)} for image in images])
msg_prompt.messages = [{"role": "user", "content": content}]
elif msg.is_video_msg():
video_prompt, video = msg.get_video_body()
frames = video_utils.extract_frames(video)
if video_prompt is None:
content = [{"type": "text", "text": f"{msg.sender}'s message"}]
content.extend([{"type": "image_url", "url": frame} for frame in frames])
msg_prompt.messages = [{"role": "user", "content": content}]
else:
content = [{"type": "text", "text": f"{msg.sender}:{video_prompt}"}]
content.extend([{"type": "image_url", "url": frame} for frame in frames])
msg_prompt.messages = [{"role": "user", "content": content}]
else:
msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{msg.body}"}]
session_topic = msg.target + "#" + msg.topic
chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.chat_db)
@@ -441,7 +470,25 @@ class AIAgent(BaseAIAgent):
resp_msg = msg.create_group_resp_msg(self.agent_id,"")
return resp_msg
else:
msg_prompt.messages = [{"role":"user","content":msg.body}]
if msg.is_image_msg():
image_prompt, images = msg.get_image_body()
if image_prompt is None:
msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "url": image} for image in images]}]
else:
content = [{"type": "text", "text": image_prompt}]
content.extend([{"type": "image_url", "url": image} for image in images])
msg_prompt.messages = [{"role": "user", "content": content}]
elif msg.is_video_msg():
video_prompt, video = msg.get_video_body()
frames = video_utils.extract_frames(video)
if video_prompt is None:
msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "url": frame} for frame in frames]}]
else:
content = [{"type": "text", "text": video_prompt}]
content.extend([{"type": "image_url", "url": frame} for frame in frames])
msg_prompt.messages = [{"role": "user", "content": content}]
else:
msg_prompt.messages = [{"role":"user","content":msg.body}]
session_topic = msg.get_sender() + "#" + msg.topic
chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.chat_db)
if self.enable_thread:
src/aios/agent/agent_base.py
+26 -3
View File
@@ -9,7 +9,7 @@ import time
import re
import shlex
import json
from typing import List
from typing import List, Tuple
from .ai_function import FunctionItem, AIFunction
from ..proto.agent_msg import AgentMsg, AgentMsgType
@@ -410,6 +410,10 @@ class BaseAIAgent(abc.ABC):
def get_max_token_size(self) -> int:
pass
@abstractmethod
async def _process_msg(self,msg:AgentMsg,workspace = None) -> AgentMsg:
pass
@classmethod
def get_inner_functions(cls, env:Environment) -> (dict,int):
if env is None:
@@ -445,10 +449,29 @@ class BaseAIAgent(abc.ABC):
#logger.debug(f"Agent {self.agent_id} do llm token static system:{system_prompt_len},function:{function_token_len},history:{history_token_len},input:{input_len}, totoal prompt:{system_prompt_len + function_token_len + history_token_len} ")
if inner_functions is None and env is not None:
inner_functions,_ = BaseAIAgent.get_inner_functions(env)
model_name = self.get_llm_model_name()
if org_msg.is_video_msg() or org_msg.is_image_msg():
if model_name.startswith("gpt4"):
model_name = "gpt-4-vision-preview"
if is_json_resp:
task_result:ComputeTaskResult = await ComputeKernel.get_instance().do_llm_completion(prompt,resp_mode="json",mode_name=self.get_llm_model_name(),max_token=self.get_max_token_size(),inner_functions=inner_functions,timeout=None)
task_result: ComputeTaskResult = await (ComputeKernel.get_instance()
.do_llm_completion(
prompt,
resp_mode="json",
mode_name=model_name,
max_token=self.get_max_token_size(),
inner_functions=inner_functions,
timeout=None))
else:
task_result:ComputeTaskResult = await ComputeKernel.get_instance().do_llm_completion(prompt,resp_mode="text",mode_name=self.get_llm_model_name(),max_token=self.get_max_token_size(),inner_functions=inner_functions,timeout=None)
task_result: ComputeTaskResult = await (ComputeKernel.get_instance()
.do_llm_completion(
prompt,
resp_mode="text",
mode_name=model_name,
max_token=self.get_max_token_size(),
inner_functions=inner_functions,
timeout=None))
if task_result.result_code != ComputeTaskResultCode.OK:
logger.error(f"_do_llm_complection llm compute error:{task_result.error_str}")
#error_resp = msg.create_error_resp(task_result.error_str)
src/aios/knowledge/data/writer.py
+19 -19
View File
@@ -19,10 +19,10 @@ def _join_docs(docs: List[str], separator: str) -> Optional[str]:
return text
def _merge_splits(
splits: Iterable[str],
separator: str,
chunk_size: int,
chunk_overlap: int,
splits: Iterable[str],
separator: str,
chunk_size: int,
chunk_overlap: int,
length_function: Callable[[str], int]
) -> List[str]:
# We now want to combine these smaller pieces into medium size
@@ -86,11 +86,11 @@ def _split_text_with_regex(
return [s for s in splits if s != ""]
def _split_text(
text: str,
separators: List[str],
chunk_size: int,
chunk_overlap: int,
def split_text(
text: str,
separators: List[str],
chunk_size: int,
chunk_overlap: int,
length_function: Callable[[str], int]
) -> List[str]:
@@ -127,7 +127,7 @@ def _split_text(
if not new_separators:
final_chunks.append(s)
else:
other_info = _split_text(s, new_separators, chunk_size, chunk_overlap, length_function)
other_info = split_text(s, new_separators, chunk_size, chunk_overlap, length_function)
final_chunks.extend(other_info)
if _good_splits:
merged_text = _merge_splits(_good_splits, _separator, chunk_size, chunk_overlap, length_function)
@@ -153,7 +153,7 @@ class ChunkListWriter:
chunk = file.read(chunk_size)
if not chunk:
break
chunk_len = len(chunk)
chunk_id = ChunkID.hash_data(chunk)
chunk_list.append(chunk_id)
@@ -176,14 +176,14 @@ class ChunkListWriter:
file_hash = HashValue(hash_obj.digest())
# print(f"calc file hash: {file_path}, {file_hash}")
return ChunkList(chunk_list, file_hash)
def create_chunk_list_from_text(
self,
text: str,
chunk_size: int = 4000,
chunk_overlap: int = 200,
self,
text: str,
chunk_size: int = 4000,
chunk_overlap: int = 200,
separators: str = ["\n\n", "\n", " ", ""]
) -> ChunkList:
enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
@@ -196,8 +196,8 @@ class ChunkListWriter:
disallowed_special="all",
)
)
text_list = _split_text(text, separators, chunk_size, chunk_overlap, length_function)
text_list = split_text(text, separators, chunk_size, chunk_overlap, length_function)
chunk_list = []
hash_obj = hashlib.sha256()
@@ -211,4 +211,4 @@ class ChunkListWriter:
self.chunk_store.put_chunk(chunk_id, chunk_bytes)
hash = HashValue(hash_obj.digest())
return ChunkList(chunk_list, hash)
return ChunkList(chunk_list, hash)