gyt/opendan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Supports telegram’s picture and video messages

Browse Source
This commit is contained in:
wugren
2023-12-04 19:05:41 +08:00
parent 1cdf6f73c0
commit 1e987cc9d3
2 changed files with 45 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rootfs/agents/Vision/agent.toml
+3 -3
View File
@@ -1,8 +1,8 @@
instance_id = "Vision"
fullname = "Vision"
llm_model_name = "gpt-4-vision-preview"
llm_model_name = "gpt-4-1106-preview"
[[prompt]]
role = "system"
content = """你的工作对用户输入的图片和视频做分析，并根据用户的意图做出回应。
如果用户请求的是视频时，你接受到的是视频的关键帧，请根据关键帧内容回复用户问题。"""
content = """Your job is to analyze user input images and videos and respond based on user intent.
If the user requests a video and you receive key frames of the video, please reply to the user's question based on the key frame content."""
src/component/tg_tunnel.py
+42 -12
View File
@@ -1,4 +1,6 @@
import datetime
import logging
import os.path
import threading
import asyncio
import uuid
@@ -51,6 +53,9 @@ class TelegramTunnel(AgentTunnel):
self.allow_group = "contact"
self.in_process_tg_msg = {}
self.chatid_record = {}
self.telegram_cache = os.path.join(AIStorage.get_instance().get_myai_dir(), "telegram")
if not os.path.exists(self.telegram_cache):
os.makedirs(self.telegram_cache)
async def _do_process_raw_message(self,bot: Bot, update_id: int) -> int:
# Request updates after the last update_id
@@ -58,7 +63,7 @@ class TelegramTunnel(AgentTunnel):
for update in updates:
next_update_id = update.update_id + 1
if update.message and update.message.text:
if update.message and (update.message.text or (update.message.photo and len(update.message.photo) > 0) or update.message.video):
await self.on_message(bot,update)
return next_update_id
@@ -96,9 +101,10 @@ class TelegramTunnel(AgentTunnel):
update_id += 1
except Exception as e:
logger.error(f"tg_tunnel error:{e}")
logger.exception(e)
await asyncio.sleep(1)
asyncio.create_task(_run_app())
logger.info(f"tunnel {self.tunnel_id} started.")
@@ -120,7 +126,7 @@ class TelegramTunnel(AgentTunnel):
# if chatid is None:
# logger.warning(f"tg_tunnel process message {msg.msg_id} from agent {msg.sender} to human {msg.target} failed! chatid not found!")
# return None
# if bot is None:
# logger.warning(f"tg_tunnel process message {msg.msg_id} from agent {msg.sender} to human {msg.target} failed! bot not found!")
# return None
@@ -130,7 +136,7 @@ class TelegramTunnel(AgentTunnel):
# await bot.send_message(chat_id=chatid,text=msg.body)
# logging.info(f"tg_tunnel send message {msg.msg_id} from agent {msg.sender} to human {msg.target} @ chatid:{chatid}success!")
# return None
# logger.warning(f"tg_tunnel process message {msg.msg_id} from agent {msg.sender} to human {msg.target} failed! contact not found!")
# return None
@@ -143,13 +149,37 @@ class TelegramTunnel(AgentTunnel):
else:
logger.warning(f"tg_tunnel process message {msg.msg_id} from agent {msg.sender} to human {msg.target} failed! chatid not found!")
def get_cache_path(self) -> str:
today = datetime.datetime.today()
path = os.path.join(self.telegram_cache, str(today.year), str(today.month))
if not os.path.exists(path):
os.makedirs(path)
return path
async def conver_tg_msg_to_agent_msg(self,message:Message) -> AgentMsg:
agent_msg = AgentMsg()
agent_msg.topic = "_telegram"
agent_msg.msg_id = "tg_msg#" + str(message.message_id) + "#" + uuid.uuid4().hex
agent_msg.target = self.target_id
agent_msg.body = message.text
if message.text is not None:
agent_msg.body = message.text
elif message.photo is not None and len(message.photo) > 0:
photo_files = []
photo_file = await message.photo[-1].get_file()
ext = photo_file.file_path.rsplit(".")[-1]
file_path = os.path.join(self.get_cache_path(), photo_file.file_id + f".{ext}")
await photo_file.download_to_drive(file_path)
photo_files.append(file_path)
agent_msg.body = agent_msg.create_image_body(photo_files, message.caption)
agent_msg.body_mime = f"image/{ext}"
elif message.video is not None:
video_file = await message.video.get_file()
ext = video_file.file_path.rsplit(".")[-1]
file_path = os.path.join(self.get_cache_path(), video_file.file_id + f".{ext}")
await video_file.download_to_drive(file_path)
agent_msg.body = agent_msg.create_video_body(file_path, message.caption)
agent_msg.body_mime = f"video/{ext}"
agent_msg.create_time = time.time()
messag_type = message.chat.type
if messag_type == "supergroup" or messag_type == "group":
@@ -168,7 +198,7 @@ class TelegramTunnel(AgentTunnel):
agent_msg.mentions.append(self.target_id)
else:
agent_msg.mentions.append(mention)
if message.caption_entities:
for entity in message.caption_entities:
if entity.type == 'mention':
@@ -203,11 +233,11 @@ class TelegramTunnel(AgentTunnel):
if update.effective_user.is_bot:
logger.warning(f"ignore message from telegram bot {update.effective_user.id}")
return None
if self.in_process_tg_msg.get(update.message.message_id) is not None:
logger.warning(f"ignore message from telegram bot {update.effective_user.id}")
return None
self.in_process_tg_msg[update.message.message_id] = True
agent_msg = await self.conver_tg_msg_to_agent_msg(message)
@@ -226,7 +256,7 @@ class TelegramTunnel(AgentTunnel):
if self.allow_group != "contact" and self.allow_group !="guest":
await update.message.reply_text(f"You're not supposed to talk to me! Please contact my father~")
return
else:
if self.allow_group != "guest":
await update.message.reply_text(f"The current Telegram account is not in the contact list. If you want to receive a reply, you can add the configuration in the contacts.toml file or switch tunnel to guest mode.")
@@ -246,7 +276,7 @@ class TelegramTunnel(AgentTunnel):
if contact is not None:
contact.set_active_tunnel(self.target_id,self)
self.chatid_record[reomte_user_name] = update.effective_chat.id
self.ai_bus.register_message_handler(reomte_user_name,contact._process_msg)
self.ai_bus.register_message_handler(reomte_user_name,contact._process_msg)
agent_msg.sender = reomte_user_name
logger.info(f"process message {agent_msg.msg_id} from {agent_msg.sender} to {agent_msg.target}")
@@ -266,11 +296,11 @@ class TelegramTunnel(AgentTunnel):
if resp_msg.body_mime is None:
if resp_msg.body is None:
return
if len(resp_msg.body) < 1:
await update.message.reply_text("")
return
knowledge_object = KnowledgeStore().parse_object_in_message(resp_msg.body)
if knowledge_object is not None:
if knowledge_object.get_object_type() == ObjectType.Image: