Supports telegram’s picture and video messages

rootfs/agents/Vision/agent.toml

@@ -1,8 +1,8 @@
 instance_id = "Vision"
 fullname = "Vision"
-llm_model_name = "gpt-4-vision-preview"
+llm_model_name = "gpt-4-1106-preview"
 
 [[prompt]]
 role = "system"
-content = """你的工作对用户输入的图片和视频做分析，并根据用户的意图做出回应。
-如果用户请求的是视频时，你接受到的是视频的关键帧，请根据关键帧内容回复用户问题。"""
+content = """Your job is to analyze user input images and videos and respond based on user intent.
+If the user requests a video and you receive key frames of the video, please reply to the user's question based on the key frame content."""

src/component/tg_tunnel.py

@@ -1,4 +1,6 @@
+import datetime
 import logging
+import os.path
 import threading
 import asyncio
 import uuid
@@ -51,6 +53,9 @@ class TelegramTunnel(AgentTunnel):
         self.allow_group = "contact"
         self.in_process_tg_msg = {}
         self.chatid_record = {}
+        self.telegram_cache = os.path.join(AIStorage.get_instance().get_myai_dir(), "telegram")
+        if not os.path.exists(self.telegram_cache):
+            os.makedirs(self.telegram_cache)
 
     async def _do_process_raw_message(self,bot: Bot, update_id: int) -> int:
         # Request updates after the last update_id
@@ -58,7 +63,7 @@ class TelegramTunnel(AgentTunnel):
         for update in updates:
             next_update_id = update.update_id + 1
 
-            if update.message and update.message.text:
+            if update.message and (update.message.text or (update.message.photo and len(update.message.photo) > 0) or update.message.video):
 
                 await self.on_message(bot,update)
             return next_update_id
@@ -96,6 +101,7 @@ class TelegramTunnel(AgentTunnel):
                     update_id += 1
                 except  Exception as e:
                     logger.error(f"tg_tunnel error:{e}")
+                    logger.exception(e)
                     await asyncio.sleep(1)
 
 
@@ -143,13 +149,37 @@ class TelegramTunnel(AgentTunnel):
         else:
             logger.warning(f"tg_tunnel process message {msg.msg_id} from agent {msg.sender} to human {msg.target} failed! chatid not found!")
 
+    def get_cache_path(self) -> str:
+        today = datetime.datetime.today()
+        path = os.path.join(self.telegram_cache, str(today.year), str(today.month))
+        if not os.path.exists(path):
+            os.makedirs(path)
+        return path
 
     async def conver_tg_msg_to_agent_msg(self,message:Message) -> AgentMsg:
         agent_msg = AgentMsg()
         agent_msg.topic = "_telegram"
         agent_msg.msg_id = "tg_msg#" + str(message.message_id) + "#" + uuid.uuid4().hex
         agent_msg.target = self.target_id
-        agent_msg.body = message.text
+        if message.text is not None:
+            agent_msg.body = message.text
+        elif message.photo is not None and len(message.photo) > 0:
+            photo_files = []
+            photo_file = await message.photo[-1].get_file()
+            ext = photo_file.file_path.rsplit(".")[-1]
+            file_path = os.path.join(self.get_cache_path(), photo_file.file_id + f".{ext}")
+            await photo_file.download_to_drive(file_path)
+            photo_files.append(file_path)
+            agent_msg.body = agent_msg.create_image_body(photo_files, message.caption)
+            agent_msg.body_mime = f"image/{ext}"
+        elif message.video is not None:
+            video_file = await message.video.get_file()
+            ext = video_file.file_path.rsplit(".")[-1]
+            file_path = os.path.join(self.get_cache_path(), video_file.file_id + f".{ext}")
+            await video_file.download_to_drive(file_path)
+            agent_msg.body = agent_msg.create_video_body(file_path, message.caption)
+            agent_msg.body_mime = f"video/{ext}"
+
         agent_msg.create_time = time.time()
         messag_type = message.chat.type
         if messag_type == "supergroup" or messag_type == "group":