gyt/opendan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #108 from wugren/MVP

Browse Source 
AgentMsg support image and video
This commit is contained in:
Liu Zhicong
2023-12-04 11:29:31 -08:00
committed by GitHub
parent 49c75c9d66 1e987cc9d3
commit 66e407add5
18 changed files with 558 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/component/agent_manager/agent_manager.py
+3 -6
View File
@@ -130,14 +130,11 @@ class AgentManager:
logger.error(f"read agent.toml cfg from {agent_media} failed! unexpected error occurred: {str(e)}")
return None
agent_name = os.path.split(agent_media.full_path)[1]
spec = importlib.util.spec_from_file_location(agent_name, custom_agent)
the_api = importlib.util.module_from_spec(spec)
spec.loader.exec_module(the_api)
if not hasattr(the_api,"Agent"):
agent = runpy.run_path(custom_agent)
if "init" not in agent:
logger.error(f"read agent.toml cfg from {agent_media} failed! unexpected error occurred: {str(e)}")
return None
return the_api.Agent()
return agent["init"]()
src/component/mail_environment/issue.py
+38 -30
View File
@@ -19,7 +19,7 @@ class IssueUpdateHistory:
"source": self.source,
"changes": self.changes,
}
@classmethod
def from_json_dict(cls, json_dict: dict) -> "IssueUpdateHistory":
return IssueUpdateHistory(json_dict["source"], json_dict["changes"])
@@ -40,7 +40,7 @@ class Issue:
json_dict = {
"id": self.id,
"summary": self.summary,
"state": self.state.name,
"state": self.state.name,
"create_time": self.create_time,
"deadline": self.deadline,
"source": self.source,
@@ -54,7 +54,7 @@ class Issue:
json_dict["update_history"] = []
for history in self.update_history:
json_dict["update_history"].append(history.to_json_dict())
return json_dict
@classmethod
@@ -78,26 +78,26 @@ class Issue:
history = IssueUpdateHistory.from_json_dict(history_json_dict)
issue.update_history.append(history)
return issue
@classmethod
def object_type(cls) -> ObjectType:
return ObjectType.from_user_def_type_code(0)
def __to_desc(self, desc_list:[], recursion=None):
desc = {
"id": self.id,
"summary": self.summary,
"state": self.state.name,
"state": self.state.name,
"deadline": self.deadline,
}
desc_list.append(desc)
if not recursion or not self.parent:
return
return
else:
parent = recursion.get_issue_by_id(self.parent)
parent.__to_desc(desc_list, recursion)
def to_prompt(self, recursion=None) -> str:
desc_list = []
self.__to_desc(desc_list, recursion)
@@ -107,8 +107,8 @@ class Issue:
root["child"] = child
root = child
return json.dumps(root)
@classmethod
def prompt_desc(cls) -> str:
return '''a issue contains following fileds: {
@@ -119,7 +119,7 @@ class Issue:
children: child issues of this issue
}
'''
def calculate_id(self) -> str:
desc = {
"summary": self.summary,
@@ -183,7 +183,7 @@ class IssueStorage:
return self.root
this_mail = mail_storage.get_mail_by_id(this_mail.reply_to)
def add_issue(self, source_id: str, parent_id: str, summary: str):
parent_issue = self.get_issue_by_id(parent_id)
issue = Issue()
@@ -204,11 +204,19 @@ class IssueStorage:
"new": value,
}
issue.__dict__[key] = value
issue.update_history.append(IssueUpdateHistory(source_id, changes))
issue.update_history.append(IssueUpdateHistory(source_id, changes))
self.__flush()
return issue
class IssueAgent(CustomAIAgent):
async def _process_msg(self, msg: AgentMsg, workspace=None) -> AgentMsg:
pass
def __init__(self, agent_id: str, llm_model_name: str, max_token_size: int) -> None:
super().__init__(agent_id, llm_model_name, max_token_size)
class IssueParserEnvironment(Environment):
def __init__(self, env_id: str, storage: IssueStorage) -> None:
@@ -217,30 +225,30 @@ class IssueParserEnvironment(Environment):
create_description = '''create a new issue'''
create_param = {
"mail_id": "new issue with which email object id",
"mail_id": "new issue with which email object id",
"issue_id": '''new issue's parent issue id''',
"summary": '''new issue's summary''',
}
self.add_ai_function(SimpleAIFunction("create_issue",
self.add_ai_function(SimpleAIFunction("create_issue",
create_description,
self._create,
self._create,
create_param))
update_description = '''update an existing issue'''
update_param = {
"mail_id": "update issue with which email object id",
"mail_id": "update issue with which email object id",
"issue_id": '''update issue's id''',
"summary": '''issue's new summary''',
}
self.add_ai_function(SimpleAIFunction("update_issue",
self.add_ai_function(SimpleAIFunction("update_issue",
update_description,
self._update,
self._update,
update_param))
async def _create(self, mail_id: str, issue_id: str, summary: str):
issue = self.storage.add_issue(mail_id, issue_id, summary)
return issue.id
async def _update(self, mail_id: str, issue_id: str, summary: str):
update = {}
update["summary"] = summary
@@ -253,7 +261,7 @@ class IssueParser:
mail_path = string.Template(config["mail_path"]).substitute(myai_dir=AIStorage.get_instance().get_myai_dir())
issue_path = string.Template(config["issue_path"]).substitute(myai_dir=AIStorage.get_instance().get_myai_dir())
config["path"] = issue_path
self.env = env
self.config = config
self.mail_storage = MailStorage(mail_path)
@@ -268,7 +276,7 @@ class IssueParser:
self.llm_env = IssueParserEnvironment("issue_parser", self.issue_storage)
@classmethod
def __load_issue_config(cls, issue_config: dict) -> Issue:
def __load_issue_config(cls, issue_config: dict) -> Issue:
issue = Issue()
issue.summary = issue_config["summary"]
if "children" in issue_config:
@@ -276,15 +284,15 @@ class IssueParser:
child_issue = cls.__load_issue_config(child_config)
issue.children.append(child_issue)
return issue
@classmethod
def __calac_issue_id(cls, issue: Issue):
issue_id = issue.calculate_id()
for child in issue.children:
child.parent = issue_id
cls.__calac_issue_id(child)
def get_path(self) -> str:
return self.config["path"]
@@ -304,8 +312,8 @@ class IssueParser:
and a issue in json format, {issue_desc}. Read mail's fileds and issue's fileds, and decide if you should update the issue or create a new issue with this mail.
Then call the function create_issue or update_issue.
if this mail is not associated with issue, you should ignore this mail.'''}
prompt.append(AgentPrompt(f'''Mail is {mail_str}, issue is {issue_str}. Answer me the function's return value or None if igonred.
prompt.append(IssueAgent(f'''Mail is {mail_str}, issue is {issue_str}. Answer me the function's return value or None if igonred.
'''))
llm_result = await CustomAIAgent("issue parser", "gpt-4-1106-preview", 4000).do_llm_complection(prompt, env=self.llm_env)
src/component/openai_node/open_ai_node.py
+29 -9
View File
@@ -8,8 +8,10 @@ import json
import aiohttp
import base64
import requests
from openai._types import NOT_GIVEN
from aios import ComputeTask, ComputeTaskResult, ComputeTaskState, ComputeTaskType,ComputeTaskResultCode,ComputeNode,AIStorage,UserConfig
from aios import image_utils
logger = logging.getLogger(__name__)
@@ -92,15 +94,19 @@ class OpenAI_ComputeNode(ComputeNode):
def _image_2_text(self, task: ComputeTask):
logger.info('openai image_2_text')
# 本地图片处理
def encode_image(image_path):
with open(image_path, "rb") as image_file:
return base64.b64encode(image_file.read()).decode('utf-8')
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {self.openai_api_key }"
}
model_name = task.params["model_name"]
base64_image = encode_image(task.params["image_path"])
image_path = task.params["image_path"]
if image_utils.is_file(image_path):
url = image_utils.to_base64(image_path, (1024, 1024))
else:
url = image_path
payload = {
"model": model_name,
"messages": [
@@ -114,7 +120,7 @@ class OpenAI_ComputeNode(ComputeNode):
{
"type": "image_url",
"image_url": {
"url": f"data:image/jpeg;base64,{base64_image}"
"url": url
}
}
]
@@ -196,7 +202,16 @@ class OpenAI_ComputeNode(ComputeNode):
if max_token_size is None:
max_token_size = 4000
result_token = max_token_size
if mode_name == "gpt-4-vision-preview":
response_format = NOT_GIVEN
llm_inner_functions = None
if max_token_size > 4096:
result_token = 4096
else:
result_token = max_token_size
else:
result_token = NOT_GIVEN
client = AsyncOpenAI(api_key=self.openai_api_key)
try:
if llm_inner_functions is None:
@@ -204,7 +219,7 @@ class OpenAI_ComputeNode(ComputeNode):
resp = await client.chat.completions.create(model=mode_name,
messages=prompts,
response_format = response_format,
#max_tokens=result_token,
max_tokens=result_token,
)
else:
logger.info(f"call openai {mode_name} prompts: \n\t {prompts} \nfunctions: \n\t{json.dumps(llm_inner_functions)}")
@@ -212,7 +227,7 @@ class OpenAI_ComputeNode(ComputeNode):
messages=prompts,
response_format = response_format,
functions=llm_inner_functions,
# max_tokens=result_token,
max_tokens=result_token,
) # TODO: add temperature to task params?
except Exception as e:
logger.error(f"openai run LLM_COMPLETION task error: {e}")
@@ -222,7 +237,12 @@ class OpenAI_ComputeNode(ComputeNode):
return result
logger.info(f"openai response: {resp}")
status_code = resp.choices[0].finish_reason
if mode_name == "gpt-4-vision-preview":
status_code = resp.choices[0].finish_reason
if status_code is None:
status_code = resp.choices[0].finish_details['type']
else:
status_code = resp.choices[0].finish_reason
token_usage = resp.usage
match status_code:
src/component/tg_tunnel.py
+42 -12
View File
@@ -1,4 +1,6 @@
import datetime
import logging
import os.path
import threading
import asyncio
import uuid
@@ -51,6 +53,9 @@ class TelegramTunnel(AgentTunnel):
self.allow_group = "contact"
self.in_process_tg_msg = {}
self.chatid_record = {}
self.telegram_cache = os.path.join(AIStorage.get_instance().get_myai_dir(), "telegram")
if not os.path.exists(self.telegram_cache):
os.makedirs(self.telegram_cache)
async def _do_process_raw_message(self,bot: Bot, update_id: int) -> int:
# Request updates after the last update_id
@@ -58,7 +63,7 @@ class TelegramTunnel(AgentTunnel):
for update in updates:
next_update_id = update.update_id + 1
if update.message and update.message.text:
if update.message and (update.message.text or (update.message.photo and len(update.message.photo) > 0) or update.message.video):
await self.on_message(bot,update)
return next_update_id
@@ -96,9 +101,10 @@ class TelegramTunnel(AgentTunnel):
update_id += 1
except Exception as e:
logger.error(f"tg_tunnel error:{e}")
logger.exception(e)
await asyncio.sleep(1)
asyncio.create_task(_run_app())
logger.info(f"tunnel {self.tunnel_id} started.")
@@ -120,7 +126,7 @@ class TelegramTunnel(AgentTunnel):
# if chatid is None:
# logger.warning(f"tg_tunnel process message {msg.msg_id} from agent {msg.sender} to human {msg.target} failed! chatid not found!")
# return None
# if bot is None:
# logger.warning(f"tg_tunnel process message {msg.msg_id} from agent {msg.sender} to human {msg.target} failed! bot not found!")
# return None
@@ -130,7 +136,7 @@ class TelegramTunnel(AgentTunnel):
# await bot.send_message(chat_id=chatid,text=msg.body)
# logging.info(f"tg_tunnel send message {msg.msg_id} from agent {msg.sender} to human {msg.target} @ chatid:{chatid}success!")
# return None
# logger.warning(f"tg_tunnel process message {msg.msg_id} from agent {msg.sender} to human {msg.target} failed! contact not found!")
# return None
@@ -143,13 +149,37 @@ class TelegramTunnel(AgentTunnel):
else:
logger.warning(f"tg_tunnel process message {msg.msg_id} from agent {msg.sender} to human {msg.target} failed! chatid not found!")
def get_cache_path(self) -> str:
today = datetime.datetime.today()
path = os.path.join(self.telegram_cache, str(today.year), str(today.month))
if not os.path.exists(path):
os.makedirs(path)
return path
async def conver_tg_msg_to_agent_msg(self,message:Message) -> AgentMsg:
agent_msg = AgentMsg()
agent_msg.topic = "_telegram"
agent_msg.msg_id = "tg_msg#" + str(message.message_id) + "#" + uuid.uuid4().hex
agent_msg.target = self.target_id
agent_msg.body = message.text
if message.text is not None:
agent_msg.body = message.text
elif message.photo is not None and len(message.photo) > 0:
photo_files = []
photo_file = await message.photo[-1].get_file()
ext = photo_file.file_path.rsplit(".")[-1]
file_path = os.path.join(self.get_cache_path(), photo_file.file_id + f".{ext}")
await photo_file.download_to_drive(file_path)
photo_files.append(file_path)
agent_msg.body = agent_msg.create_image_body(photo_files, message.caption)
agent_msg.body_mime = f"image/{ext}"
elif message.video is not None:
video_file = await message.video.get_file()
ext = video_file.file_path.rsplit(".")[-1]
file_path = os.path.join(self.get_cache_path(), video_file.file_id + f".{ext}")
await video_file.download_to_drive(file_path)
agent_msg.body = agent_msg.create_video_body(file_path, message.caption)
agent_msg.body_mime = f"video/{ext}"
agent_msg.create_time = time.time()
messag_type = message.chat.type
if messag_type == "supergroup" or messag_type == "group":
@@ -168,7 +198,7 @@ class TelegramTunnel(AgentTunnel):
agent_msg.mentions.append(self.target_id)
else:
agent_msg.mentions.append(mention)
if message.caption_entities:
for entity in message.caption_entities:
if entity.type == 'mention':
@@ -203,11 +233,11 @@ class TelegramTunnel(AgentTunnel):
if update.effective_user.is_bot:
logger.warning(f"ignore message from telegram bot {update.effective_user.id}")
return None
if self.in_process_tg_msg.get(update.message.message_id) is not None:
logger.warning(f"ignore message from telegram bot {update.effective_user.id}")
return None
self.in_process_tg_msg[update.message.message_id] = True
agent_msg = await self.conver_tg_msg_to_agent_msg(message)
@@ -226,7 +256,7 @@ class TelegramTunnel(AgentTunnel):
if self.allow_group != "contact" and self.allow_group !="guest":
await update.message.reply_text(f"You're not supposed to talk to me! Please contact my father~")
return
else:
if self.allow_group != "guest":
await update.message.reply_text(f"The current Telegram account is not in the contact list. If you want to receive a reply, you can add the configuration in the contacts.toml file or switch tunnel to guest mode.")
@@ -246,7 +276,7 @@ class TelegramTunnel(AgentTunnel):
if contact is not None:
contact.set_active_tunnel(self.target_id,self)
self.chatid_record[reomte_user_name] = update.effective_chat.id
self.ai_bus.register_message_handler(reomte_user_name,contact._process_msg)
self.ai_bus.register_message_handler(reomte_user_name,contact._process_msg)
agent_msg.sender = reomte_user_name
logger.info(f"process message {agent_msg.msg_id} from {agent_msg.sender} to {agent_msg.target}")
@@ -266,11 +296,11 @@ class TelegramTunnel(AgentTunnel):
if resp_msg.body_mime is None:
if resp_msg.body is None:
return
if len(resp_msg.body) < 1:
await update.message.reply_text("")
return
knowledge_object = KnowledgeStore().parse_object_in_message(resp_msg.body)
if knowledge_object is not None:
if knowledge_object.get_object_type() == ObjectType.Image: