AgentMsg support image and video

rootfs/agents/AgentAssistant/environment.py

@@ -3,9 +3,10 @@ from typing import Optional
 
 import toml
 
-from aios_kernel import Environment, SimpleAIFunction
 import os
 
+from aios.agent.ai_function import SimpleAIFunction
+from aios.environment.environment import Environment
 
 local_path = os.path.split(os.path.realpath(__file__))[0]
 

rootfs/agents/DBQueryer/environment.py

@@ -1,7 +1,7 @@
 from typing import Optional
 
-from aios_kernel import Environment
-from aios_kernel.sql_database_function import GetTableInfosFunction, ExecuteSqlFunction
+from aios.environment.environment import Environment
+from aios.environment.sql_database_function import GetTableInfosFunction, ExecuteSqlFunction
 
 
 class DBQuerierEnvironment(Environment):

rootfs/agents/TextSummary/agent.py

@@ -1,8 +1,9 @@
 import copy
 
-from aios_kernel import CustomAIAgent, AgentMsg, AgentMsgType, AgentPrompt
-from aios_kernel.compute_task import ComputeTaskResultCode
-from knowledge.data.writer import split_text
+from aios.agent.agent_base import CustomAIAgent, AgentPrompt
+from aios.knowledge.data.writer import split_text
+from aios.proto.agent_msg import AgentMsg, AgentMsgType
+from aios.proto.compute_task import ComputeTaskResultCode
 
 
 class TextSummaryAgent(CustomAIAgent):

rootfs/agents/Vision/agent.toml

@@ -4,4 +4,5 @@ llm_model_name = "gpt-4-vision-preview"
 
 [[prompt]]
 role = "system"
-content = """你的工作对用户输入的图片和视频做分析，并根据用户的意图做出回应。"""
+content = """你的工作对用户输入的图片和视频做分析，并根据用户的意图做出回应。
+如果用户请求的是视频时，你接受到的是视频的关键帧，请根据关键帧内容回复用户问题。"""

src/aios/__init__.py

@@ -27,6 +27,7 @@ from .storage.storage import ResourceLocation,AIStorage,UserConfig,UserConfigIte
 from .net import *
 from .knowledge import *
 from .package_manager import *
+from .utils import *
 
 
-AIOS_Version = "0.5.2, build 2023-11-30"
+AIOS_Version = "0.5.2, build 2023-11-30"

src/aios/agent/agent.py

@@ -27,7 +27,7 @@ from ..environment.workspace_env import WorkspaceEnvironment
 from ..storage.storage import AIStorage
 
 from ..knowledge import *
-from . import video_utils, image_utils
+from ..utils import video_utils, image_utils
 
 logger = logging.getLogger(__name__)
 
@@ -426,7 +426,7 @@ class AIAgent(BaseAIAgent):
 
     def check_and_to_base64(self, image_path: str) -> str:
         if image_utils.is_file(image_path):
-            return image_utils.image_to_base64(image_path)
+            return image_utils.to_base64(image_path, (1024, 1024))
         else:
             return image_path
 
@@ -438,22 +438,22 @@ class AIAgent(BaseAIAgent):
                 image_prompt, images = msg.get_image_body()
                 if image_prompt is None:
                     content = [[{"type": "text", "text": f"{msg.sender}'s message"}]]
-                    content.extend([{"type": "image_url", "url": self.check_and_to_base64(image)} for image in images])
+                    content.extend([{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images])
                     msg_prompt.messages = [{"role": "user", "content": content}]
                 else:
                     content = [{"type": "text", "text": f"{msg.sender}:{image_prompt}"}]
-                    content.extend([{"type": "image_url", "url": self.check_and_to_base64(image)} for image in images])
+                    content.extend([{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images])
                     msg_prompt.messages = [{"role": "user", "content": content}]
             elif msg.is_video_msg():
                 video_prompt, video = msg.get_video_body()
-                frames = video_utils.extract_frames(video)
+                frames = video_utils.extract_frames(video, (1024, 1024))
                 if video_prompt is None:
                     content = [{"type": "text", "text": f"{msg.sender}'s message"}]
-                    content.extend([{"type": "image_url", "url": frame} for frame in frames])
+                    content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
                     msg_prompt.messages = [{"role": "user", "content": content}]
                 else:
                     content = [{"type": "text", "text": f"{msg.sender}:{video_prompt}"}]
-                    content.extend([{"type": "image_url", "url": frame} for frame in frames])
+                    content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
                     msg_prompt.messages = [{"role": "user", "content": content}]
             else:
                 msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{msg.body}"}]
@@ -473,19 +473,19 @@ class AIAgent(BaseAIAgent):
             if msg.is_image_msg():
                 image_prompt, images = msg.get_image_body()
                 if image_prompt is None:
-                    msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "url": image} for image in images]}]
+                    msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images]}]
                 else:
                     content = [{"type": "text", "text": image_prompt}]
-                    content.extend([{"type": "image_url", "url": image} for image in images])
+                    content.extend([{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images])
                     msg_prompt.messages = [{"role": "user", "content": content}]
             elif msg.is_video_msg():
                 video_prompt, video = msg.get_video_body()
-                frames = video_utils.extract_frames(video)
+                frames = video_utils.extract_frames(video, (1024, 1024))
                 if video_prompt is None:
-                    msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "url": frame} for frame in frames]}]
+                    msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": frame}} for frame in frames]}]
                 else:
                     content = [{"type": "text", "text": video_prompt}]
-                    content.extend([{"type": "image_url", "url": frame} for frame in frames])
+                    content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
                     msg_prompt.messages = [{"role": "user", "content": content}]
             else:
                 msg_prompt.messages = [{"role":"user","content":msg.body}]

src/aios/agent/agent_base.py

@@ -452,7 +452,7 @@ class BaseAIAgent(abc.ABC):
 
         model_name = self.get_llm_model_name()
         if org_msg.is_video_msg() or org_msg.is_image_msg():
-            if model_name.startswith("gpt4"):
+            if model_name.startswith("gpt-4"):
                 model_name = "gpt-4-vision-preview"
         if is_json_resp:
             task_result: ComputeTaskResult = await (ComputeKernel.get_instance()
@@ -501,7 +501,6 @@ class BaseAIAgent(abc.ABC):
         stack_limit = 5
     ) -> ComputeTaskResult:
         from ..frame.compute_kernel import ComputeKernel
-        
         arguments = None
         try:
             func_name = inner_func_call_node.get("name")

src/aios/proto/agent_msg.py

@@ -1,7 +1,10 @@
+import json
 import logging
+import shlex
 import uuid
 from enum import Enum
 import time
+from typing import Tuple, List
 
 logger = logging.getLogger(__name__)
 
@@ -35,8 +38,6 @@ class AgentMsgStatus(Enum):
 # 逻辑上的同一个Message在同一个session中看到的msgid相同
 #       在不同的session中看到的msgid不同
 
-
-
 class AgentMsg:
     def __init__(self,msg_type=AgentMsgType.TYPE_MSG) -> None:
         self.msg_id = "msg#" + uuid.uuid4().hex
@@ -136,14 +137,79 @@ class AgentMsg:
 
         return resp_msg
 
-    def set(self,sender:str,target:str,body:str,topic:str=None) -> None:
+    def set(self,sender:str,target:str,body:str,topic:str=None,body_mime:str=None) -> None:
         self.sender = sender
         self.target = target
         self.body = body
+        self.body_mime = body_mime
         self.create_time = time.time()
         if topic:
             self.topic = topic
 
+    @staticmethod
+    def create_image_body(images: [str], prompt: str = None):
+        return json.dumps({"images": images, "prompt": prompt})
+
+    @staticmethod
+    def parse_image_body(image_body: str) -> Tuple[str, List[str]]:
+        body = json.loads(image_body)
+        return body.get("prompt"), body.get("images")
+
+    @staticmethod
+    def create_video_body(video: str, prompt: str = None):
+        return json.dumps({"video": video, "prompt": prompt})
+
+    @staticmethod
+    def parse_video_body(video_body: str) -> Tuple[str, str]:
+        body = json.loads(video_body)
+        return body.get("prompt"), body.get("video")
+
+    def set_image(self, sender: str, target: str, image_format: str, images: [str], prompt: str = None, topic: str = None):
+        self.sender = sender
+        self.target = target
+        self.create_time = time.time()
+        self.body_mime = f"image/{image_format}"
+        self.body = self.create_image_body(images, prompt)
+        if topic:
+            self.topic = topic
+
+    def is_image_msg(self) -> bool:
+        if self.body_mime is None:
+            return False
+        if self.body_mime.startswith("image/"):
+            return True
+        return False
+
+    def get_image_body(self) -> Tuple[str, List[str]]:
+        if self.body_mime is None:
+            return None
+        if self.body_mime.startswith("image/"):
+            return self.parse_image_body(self.body)
+        return None
+
+    def set_video(self, sender: str, target: str, video_format: str, video: str, prompt: str = None, topic: str = None):
+        self.sender = sender
+        self.target = target
+        self.create_time = time.time()
+        self.body_mime = f"video/{video_format}"
+        self.body = self.create_video_body(video, prompt)
+        if topic:
+            self.topic = topic
+
+    def get_video_body(self) -> Tuple[str, str]:
+        if self.body_mime is None:
+            return None
+        if self.body_mime.startswith("video/"):
+            return self.parse_video_body(self.body)
+        return None
+
+    def is_video_msg(self) -> bool:
+        if self.body_mime is None:
+            return False
+        if self.body_mime.startswith("video/"):
+            return True
+        return False
+
     def get_msg_id(self) -> str:
         return self.msg_id
 
@@ -164,4 +230,4 @@ class AgentMsg:
         str_list = shlex.split(func_string)
         func_name = str_list[0]
         params = str_list[1:]
-        return func_name, params
+        return func_name, params

src/aios/utils/__init__.py

@@ -0,0 +1,2 @@
+from . import image_utils
+from . import video_utils

src/aios/utils/image_utils.py

@@ -0,0 +1,40 @@
+import base64
+import os.path
+from typing import Tuple
+
+import cv2
+
+
+def to_base64(image_path: str, resize: Tuple[int, int] = None) -> str:
+    """Convert image to base64."""
+    ext = os.path.splitext(image_path)[1][1:]
+    if resize is None:
+        with open(image_path, "rb") as image_file:
+            base64_image = base64.b64encode(image_file.read()).decode("utf-8")
+            return f"data:image/{ext};base64,{base64_image}"
+    else:
+        dest_width, dest_height = resize
+        img = cv2.imread(image_path)
+        width, height = img.shape[:2]
+        if width > dest_width or height > dest_height:
+            width_rate = dest_width / width
+            height_rate = dest_height / height
+            rate = min(width_rate, height_rate)
+            dest_width = int(width * rate)
+            dest_height = int(height * rate)
+            img = cv2.resize(img, (dest_width, dest_height), interpolation=cv2.INTER_AREA)
+        _, buf = cv2.imencode(f".{ext}", img)
+        base64_image = base64.b64encode(buf).decode("utf-8")
+        return f"data:image/{ext};base64,{base64_image}"
+
+
+def is_file(image_path: str) -> bool:
+    return os.path.isfile(image_path)
+
+
+def is_base64(image_path: str) -> bool:
+    return image_path.startswith("data:image/")
+
+
+def is_url(image_path: str) -> bool:
+    return image_path.startswith("http://") or image_path.startswith("https://")

src/aios/utils/video_utils.py

@@ -0,0 +1,122 @@
+import base64
+from typing import List, Tuple
+
+import cv2
+import numpy as np
+
+
+def precess_image(image):
+    '''
+    Graying and GaussianBlur
+    :param image: The image matrix,np.array
+    :return: The processed image matrix,np.array
+    '''
+    gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+    gray_image = cv2.GaussianBlur(gray_image, (3, 3), 0)
+    return gray_image
+
+
+def abs_diff(pre_image, curr_image):
+    '''
+    Calculate absolute difference between pre_image and curr_image
+    :param pre_image:The image in past frame,np.array
+    :param curr_image:The image in current frame,np.array
+    :return:
+    '''
+    gray_pre_image = precess_image(pre_image)
+    gray_curr_image = precess_image(curr_image)
+    diff = cv2.absdiff(gray_pre_image, gray_curr_image)
+    res, diff = cv2.threshold(diff, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)
+    cnt_diff = np.sum(np.sum(diff))
+    return cnt_diff
+
+
+def exponential_smoothing(alpha, s):
+    '''
+    Primary exponential smoothing
+    :param alpha:  Smoothing factor,num
+    :param s:      List of data,list
+    :return:       List of data after smoothing,list
+    '''
+    s_temp = [s[0]]
+    print(s_temp)
+    for i in range(1, len(s), 1):
+        s_temp.append(alpha * s[i - 1] + (1 - alpha) * s_temp[i - 1])
+    return s_temp
+
+
+def extract_frames(video_path: str, resize: Tuple[int, int] = None, smooth=False, alpha=0.07, window=25) -> List[str]:
+    """Extract frames from video."""
+    frames = []
+    vidcap = cv2.VideoCapture(video_path)
+    diff = []
+    frm = 0
+    pre_image = np.array([])
+    cur_image = np.array([])
+
+    while True:
+        frm = frm + 1
+        success, image = vidcap.read()
+        if not success:
+            break
+
+        if frm == 1:
+            pre_image = image
+            cur_image = image
+        else:
+            pre_image = cur_image
+            cur_image = image
+
+        diff.append(abs_diff(pre_image, cur_image))
+
+    if smooth:
+        diff = exponential_smoothing(alpha, diff)
+
+    diff = np.array(diff)
+    mean = np.mean(diff)
+    dev = np.std(diff)
+    diff = (diff - mean) / dev
+
+    idx = []
+    for i, d in enumerate(diff):
+        ub = len(diff) - 1
+        lb = 0
+        if not i - window // 2 < lb:
+            lb = i - window // 2
+        if not i + window // 2 > ub:
+            ub = i + window // 2
+
+        comp_window = diff[lb: ub]
+        if d >= max(comp_window):
+            idx.append(i)
+
+    tmp = np.array(idx)
+    tmp = tmp + 1
+    idx = set(tmp.tolist())
+    vidcap.release()
+
+    vidcap = cv2.VideoCapture(video_path)
+    i = 0
+    frm = 0
+    while vidcap.isOpened() and i < 10:
+        frm = frm + 1
+        success, image = vidcap.read()
+        if not success:
+            break
+        if frm not in idx:
+            continue
+        if resize is not None:
+            dest_width, dest_height = resize
+            width, height = image.shape[:2]
+            if width > dest_width or height > dest_height:
+                width_rate = dest_width / width
+                height_rate = dest_height / height
+                rate = min(width_rate, height_rate)
+                dest_width = int(width * rate)
+                dest_height = int(height * rate)
+                image = cv2.resize(image, (dest_width, dest_height), interpolation=cv2.INTER_AREA)
+        _, buffer = cv2.imencode(".jpg", image)
+        frames.append(f"data:image/jpg;base64,{base64.b64encode(buffer).decode('utf-8')}")
+        i += 1
+    vidcap.release()
+    return frames

src/aios_kernel/image_utils.py

@@ -1,22 +0,0 @@
-import base64
-import os.path
-
-
-def to_base64(image_path: str) -> str:
-    """Convert image to base64."""
-    ext = os.path.splitext(image_path)[1]
-    with open(image_path, "rb") as image_file:
-        base64_image = base64.b64encode(image_file.read()).decode("utf-8")
-        return f"data:image/{ext};base64,{base64_image}"
-
-
-def is_file(image_path: str) -> bool:
-    return os.path.isfile(image_path)
-
-
-def is_base64(image_path: str) -> bool:
-    return image_path.startswith("data:image/")
-
-
-def is_url(image_path: str) -> bool:
-    return image_path.startswith("http://") or image_path.startswith("https://")

src/aios_kernel/video_utils.py

@@ -1,18 +0,0 @@
-import base64
-from typing import List
-
-import cv2
-
-
-def extract_frames(video_path: str) -> List[str]:
-    """Extract frames from video."""
-    frames = []
-    vidcap = cv2.VideoCapture(video_path)
-    while vidcap.isOpened():
-        success, image = vidcap.read()
-        if not success:
-            break
-        _, buffer = cv2.imencode(".jpg", image)
-        frames.append(base64.b64encode(buffer).decode("utf-8"))
-    vidcap.release()
-    return frames

src/component/openai_node/open_ai_node.py

@@ -8,9 +8,10 @@ import json
 import aiohttp
 import base64
 import requests
+from openai._types import NOT_GIVEN
 
 from aios import ComputeTask, ComputeTaskResult, ComputeTaskState, ComputeTaskType,ComputeTaskResultCode,ComputeNode,AIStorage,UserConfig
-
+from aios import image_utils
 
 logger = logging.getLogger(__name__)
 
@@ -102,7 +103,7 @@ class OpenAI_ComputeNode(ComputeNode):
         image_path = task.params["image_path"]
 
         if image_utils.is_file(image_path):
-            url = image_utils.to_base64(image_path)
+            url = image_utils.to_base64(image_path, (1024, 1024))
         else:
             url = image_path
 
@@ -201,7 +202,16 @@ class OpenAI_ComputeNode(ComputeNode):
                 if max_token_size is None:
                     max_token_size = 4000
 
-                result_token = max_token_size
+                if mode_name == "gpt-4-vision-preview":
+                    response_format = NOT_GIVEN
+                    llm_inner_functions = None
+                    if max_token_size > 4096:
+                        result_token = 4096
+                    else:
+                        result_token = max_token_size
+                else:
+                    result_token = NOT_GIVEN
+
                 client = AsyncOpenAI(api_key=self.openai_api_key)
                 try:
                     if llm_inner_functions is None:
@@ -209,7 +219,7 @@ class OpenAI_ComputeNode(ComputeNode):
                         resp = await client.chat.completions.create(model=mode_name,
                                                         messages=prompts,
                                                         response_format = response_format,
-                                                        #max_tokens=result_token,
+                                                        max_tokens=result_token,
                                                         )
                     else:
                         logger.info(f"call openai {mode_name} prompts: \n\t {prompts} \nfunctions: \n\t{json.dumps(llm_inner_functions)}")
@@ -217,7 +227,7 @@ class OpenAI_ComputeNode(ComputeNode):
                                                             messages=prompts,
                                                             response_format = response_format,
                                                             functions=llm_inner_functions,
-                                                            # max_tokens=result_token,
+                                                            max_tokens=result_token,
                                                             ) # TODO: add temperature to task params?
                 except Exception as e:
                     logger.error(f"openai run LLM_COMPLETION task error: {e}")
@@ -227,7 +237,12 @@ class OpenAI_ComputeNode(ComputeNode):
                     return result
 
                 logger.info(f"openai response: {resp}")
-                status_code = resp.choices[0].finish_reason
+                if mode_name == "gpt-4-vision-preview":
+                    status_code = resp.choices[0].finish_reason
+                    if status_code is None:
+                        status_code = resp.choices[0].finish_details['type']
+                else:
+                    status_code = resp.choices[0].finish_reason
                 token_usage = resp.usage
 
                 match status_code:

src/service/aios_shell/aios_shell.py

@@ -31,7 +31,7 @@ from aios import *
 
 sys.path.append(directory + '/../../component/')
 
-from google_node import * 
+from google_node import *
 from llama_node import *
 from openai_node import *
 from sd_node import *
@@ -460,6 +460,54 @@ class AIOS_Shell:
                 show_text = FormattedText([("class:title", f"{self.current_topic}@{self.current_target} >>> "),
                                         ("class:content", resp)])
                 return show_text
+            case 'send_img':
+                sender = None
+                if len(args) == 4:
+                    target_id = args[0]
+                    msg_content = args[1]
+                    image_path = args[2]
+                    topic = args[3]
+                    sender = self.username
+                elif len(args) == 5:
+                    target_id = args[0]
+                    msg_content = args[1]
+                    image_path = args[2]
+                    topic = args[3]
+                    sender = args[4]
+
+                ext = os.path.splitext(image_path)[1][1:]
+                resp = await self.send_msg(AgentMsg.create_image_body([image_path], msg_content),
+                                           target_id,
+                                           topic,
+                                           sender,
+                                           f"image/{ext}")
+                show_text = FormattedText([("class:title", f"{self.current_topic}@{self.current_target} >>> "),
+                                           ("class:content", resp)])
+                return show_text
+            case 'send_video':
+                sender = None
+                if len(args) == 4:
+                    target_id = args[0]
+                    msg_content = args[1]
+                    video_path = args[2]
+                    topic = args[3]
+                    sender = self.username
+                elif len(args) == 5:
+                    target_id = args[0]
+                    msg_content = args[1]
+                    video_path = args[2]
+                    topic = args[3]
+                    sender = args[4]
+
+                ext = os.path.splitext(video_path)[1][1:]
+                resp = await self.send_msg(AgentMsg.create_video_body(video_path, msg_content),
+                                           target_id,
+                                           topic,
+                                           sender,
+                                           f"video/{ext}")
+                show_text = FormattedText([("class:title", f"{self.current_topic}@{self.current_target} >>> "),
+                                           ("class:content", resp)])
+                return show_text
             case 'set_config':
                 show_text = FormattedText([("class:error", f"set config args error,/set_config $config_item! ")])
                 if len(args) == 1:
@@ -775,6 +823,8 @@ async def main():
         return await main_daemon_loop(shell)
 
     completer = WordCompleter(['/send $target $msg $topic',
+                               '/send_img $target $msg $img_path $topic',
+                               '/send_video $target &msg &video_path $topic',
                                '/open $target $topic',
                                '/history $num $offset',
                                '/connect $target',