From fde73894436bd95b0ccb7439da2098bdbcbeb7be Mon Sep 17 00:00:00 2001 From: wugren Date: Sat, 2 Dec 2023 22:02:07 +0800 Subject: [PATCH] AgentMsg support image and video --- rootfs/agents/AgentAssistant/environment.py | 3 +- rootfs/agents/DBQueryer/environment.py | 4 +- rootfs/agents/TextSummary/agent.py | 7 +- rootfs/agents/Vision/agent.toml | 3 +- src/aios/__init__.py | 3 +- src/aios/agent/agent.py | 24 ++-- src/aios/agent/agent_base.py | 3 +- src/aios/proto/agent_msg.py | 74 +++++++++++- src/aios/utils/__init__.py | 2 + src/aios/utils/image_utils.py | 40 +++++++ src/aios/utils/video_utils.py | 122 ++++++++++++++++++++ src/aios_kernel/image_utils.py | 22 ---- src/aios_kernel/video_utils.py | 18 --- src/component/openai_node/open_ai_node.py | 27 ++++- src/service/aios_shell/aios_shell.py | 52 ++++++++- 15 files changed, 331 insertions(+), 73 deletions(-) create mode 100644 src/aios/utils/__init__.py create mode 100644 src/aios/utils/image_utils.py create mode 100644 src/aios/utils/video_utils.py delete mode 100644 src/aios_kernel/image_utils.py delete mode 100644 src/aios_kernel/video_utils.py diff --git a/rootfs/agents/AgentAssistant/environment.py b/rootfs/agents/AgentAssistant/environment.py index 35fa35b..e5ce961 100644 --- a/rootfs/agents/AgentAssistant/environment.py +++ b/rootfs/agents/AgentAssistant/environment.py @@ -3,9 +3,10 @@ from typing import Optional import toml -from aios_kernel import Environment, SimpleAIFunction import os +from aios.agent.ai_function import SimpleAIFunction +from aios.environment.environment import Environment local_path = os.path.split(os.path.realpath(__file__))[0] diff --git a/rootfs/agents/DBQueryer/environment.py b/rootfs/agents/DBQueryer/environment.py index e733c45..8c8ec02 100644 --- a/rootfs/agents/DBQueryer/environment.py +++ b/rootfs/agents/DBQueryer/environment.py @@ -1,7 +1,7 @@ from typing import Optional -from aios_kernel import Environment -from aios_kernel.sql_database_function import GetTableInfosFunction, ExecuteSqlFunction +from aios.environment.environment import Environment +from aios.environment.sql_database_function import GetTableInfosFunction, ExecuteSqlFunction class DBQuerierEnvironment(Environment): diff --git a/rootfs/agents/TextSummary/agent.py b/rootfs/agents/TextSummary/agent.py index ba6f505..537f520 100644 --- a/rootfs/agents/TextSummary/agent.py +++ b/rootfs/agents/TextSummary/agent.py @@ -1,8 +1,9 @@ import copy -from aios_kernel import CustomAIAgent, AgentMsg, AgentMsgType, AgentPrompt -from aios_kernel.compute_task import ComputeTaskResultCode -from knowledge.data.writer import split_text +from aios.agent.agent_base import CustomAIAgent, AgentPrompt +from aios.knowledge.data.writer import split_text +from aios.proto.agent_msg import AgentMsg, AgentMsgType +from aios.proto.compute_task import ComputeTaskResultCode class TextSummaryAgent(CustomAIAgent): diff --git a/rootfs/agents/Vision/agent.toml b/rootfs/agents/Vision/agent.toml index f3bf29a..52b50b6 100644 --- a/rootfs/agents/Vision/agent.toml +++ b/rootfs/agents/Vision/agent.toml @@ -4,4 +4,5 @@ llm_model_name = "gpt-4-vision-preview" [[prompt]] role = "system" -content = """你的工作对用户输入的图片和视频做分析，并根据用户的意图做出回应。""" +content = """你的工作对用户输入的图片和视频做分析，并根据用户的意图做出回应。 +如果用户请求的是视频时，你接受到的是视频的关键帧，请根据关键帧内容回复用户问题。""" diff --git a/src/aios/__init__.py b/src/aios/__init__.py index 1a0c512..ce866cf 100644 --- a/src/aios/__init__.py +++ b/src/aios/__init__.py @@ -27,6 +27,7 @@ from .storage.storage import ResourceLocation,AIStorage,UserConfig,UserConfigIte from .net import * from .knowledge import * from .package_manager import * +from .utils import * -AIOS_Version = "0.5.2, build 2023-11-30" \ No newline at end of file +AIOS_Version = "0.5.2, build 2023-11-30" diff --git a/src/aios/agent/agent.py b/src/aios/agent/agent.py index f9e728d..2b97326 100644 --- a/src/aios/agent/agent.py +++ b/src/aios/agent/agent.py @@ -27,7 +27,7 @@ from ..environment.workspace_env import WorkspaceEnvironment from ..storage.storage import AIStorage from ..knowledge import * -from . import video_utils, image_utils +from ..utils import video_utils, image_utils logger = logging.getLogger(__name__) @@ -426,7 +426,7 @@ class AIAgent(BaseAIAgent): def check_and_to_base64(self, image_path: str) -> str: if image_utils.is_file(image_path): - return image_utils.image_to_base64(image_path) + return image_utils.to_base64(image_path, (1024, 1024)) else: return image_path @@ -438,22 +438,22 @@ class AIAgent(BaseAIAgent): image_prompt, images = msg.get_image_body() if image_prompt is None: content = [[{"type": "text", "text": f"{msg.sender}'s message"}]] - content.extend([{"type": "image_url", "url": self.check_and_to_base64(image)} for image in images]) + content.extend([{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images]) msg_prompt.messages = [{"role": "user", "content": content}] else: content = [{"type": "text", "text": f"{msg.sender}:{image_prompt}"}] - content.extend([{"type": "image_url", "url": self.check_and_to_base64(image)} for image in images]) + content.extend([{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images]) msg_prompt.messages = [{"role": "user", "content": content}] elif msg.is_video_msg(): video_prompt, video = msg.get_video_body() - frames = video_utils.extract_frames(video) + frames = video_utils.extract_frames(video, (1024, 1024)) if video_prompt is None: content = [{"type": "text", "text": f"{msg.sender}'s message"}] - content.extend([{"type": "image_url", "url": frame} for frame in frames]) + content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames]) msg_prompt.messages = [{"role": "user", "content": content}] else: content = [{"type": "text", "text": f"{msg.sender}:{video_prompt}"}] - content.extend([{"type": "image_url", "url": frame} for frame in frames]) + content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames]) msg_prompt.messages = [{"role": "user", "content": content}] else: msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{msg.body}"}] @@ -473,19 +473,19 @@ class AIAgent(BaseAIAgent): if msg.is_image_msg(): image_prompt, images = msg.get_image_body() if image_prompt is None: - msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "url": image} for image in images]}] + msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images]}] else: content = [{"type": "text", "text": image_prompt}] - content.extend([{"type": "image_url", "url": image} for image in images]) + content.extend([{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images]) msg_prompt.messages = [{"role": "user", "content": content}] elif msg.is_video_msg(): video_prompt, video = msg.get_video_body() - frames = video_utils.extract_frames(video) + frames = video_utils.extract_frames(video, (1024, 1024)) if video_prompt is None: - msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "url": frame} for frame in frames]}] + msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": frame}} for frame in frames]}] else: content = [{"type": "text", "text": video_prompt}] - content.extend([{"type": "image_url", "url": frame} for frame in frames]) + content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames]) msg_prompt.messages = [{"role": "user", "content": content}] else: msg_prompt.messages = [{"role":"user","content":msg.body}] diff --git a/src/aios/agent/agent_base.py b/src/aios/agent/agent_base.py index 78e431d..4094c36 100644 --- a/src/aios/agent/agent_base.py +++ b/src/aios/agent/agent_base.py @@ -452,7 +452,7 @@ class BaseAIAgent(abc.ABC): model_name = self.get_llm_model_name() if org_msg.is_video_msg() or org_msg.is_image_msg(): - if model_name.startswith("gpt4"): + if model_name.startswith("gpt-4"): model_name = "gpt-4-vision-preview" if is_json_resp: task_result: ComputeTaskResult = await (ComputeKernel.get_instance() @@ -501,7 +501,6 @@ class BaseAIAgent(abc.ABC): stack_limit = 5 ) -> ComputeTaskResult: from ..frame.compute_kernel import ComputeKernel - arguments = None try: func_name = inner_func_call_node.get("name") diff --git a/src/aios/proto/agent_msg.py b/src/aios/proto/agent_msg.py index 3c1e2a0..c7209bf 100644 --- a/src/aios/proto/agent_msg.py +++ b/src/aios/proto/agent_msg.py @@ -1,7 +1,10 @@ +import json import logging +import shlex import uuid from enum import Enum import time +from typing import Tuple, List logger = logging.getLogger(__name__) @@ -35,8 +38,6 @@ class AgentMsgStatus(Enum): # 逻辑上的同一个Message在同一个session中看到的msgid相同 # 在不同的session中看到的msgid不同 - - class AgentMsg: def __init__(self,msg_type=AgentMsgType.TYPE_MSG) -> None: self.msg_id = "msg#" + uuid.uuid4().hex @@ -136,14 +137,79 @@ class AgentMsg: return resp_msg - def set(self,sender:str,target:str,body:str,topic:str=None) -> None: + def set(self,sender:str,target:str,body:str,topic:str=None,body_mime:str=None) -> None: self.sender = sender self.target = target self.body = body + self.body_mime = body_mime self.create_time = time.time() if topic: self.topic = topic + @staticmethod + def create_image_body(images: [str], prompt: str = None): + return json.dumps({"images": images, "prompt": prompt}) + + @staticmethod + def parse_image_body(image_body: str) -> Tuple[str, List[str]]: + body = json.loads(image_body) + return body.get("prompt"), body.get("images") + + @staticmethod + def create_video_body(video: str, prompt: str = None): + return json.dumps({"video": video, "prompt": prompt}) + + @staticmethod + def parse_video_body(video_body: str) -> Tuple[str, str]: + body = json.loads(video_body) + return body.get("prompt"), body.get("video") + + def set_image(self, sender: str, target: str, image_format: str, images: [str], prompt: str = None, topic: str = None): + self.sender = sender + self.target = target + self.create_time = time.time() + self.body_mime = f"image/{image_format}" + self.body = self.create_image_body(images, prompt) + if topic: + self.topic = topic + + def is_image_msg(self) -> bool: + if self.body_mime is None: + return False + if self.body_mime.startswith("image/"): + return True + return False + + def get_image_body(self) -> Tuple[str, List[str]]: + if self.body_mime is None: + return None + if self.body_mime.startswith("image/"): + return self.parse_image_body(self.body) + return None + + def set_video(self, sender: str, target: str, video_format: str, video: str, prompt: str = None, topic: str = None): + self.sender = sender + self.target = target + self.create_time = time.time() + self.body_mime = f"video/{video_format}" + self.body = self.create_video_body(video, prompt) + if topic: + self.topic = topic + + def get_video_body(self) -> Tuple[str, str]: + if self.body_mime is None: + return None + if self.body_mime.startswith("video/"): + return self.parse_video_body(self.body) + return None + + def is_video_msg(self) -> bool: + if self.body_mime is None: + return False + if self.body_mime.startswith("video/"): + return True + return False + def get_msg_id(self) -> str: return self.msg_id @@ -164,4 +230,4 @@ class AgentMsg: str_list = shlex.split(func_string) func_name = str_list[0] params = str_list[1:] - return func_name, params \ No newline at end of file + return func_name, params diff --git a/src/aios/utils/__init__.py b/src/aios/utils/__init__.py new file mode 100644 index 0000000..dbaab83 --- /dev/null +++ b/src/aios/utils/__init__.py @@ -0,0 +1,2 @@ +from . import image_utils +from . import video_utils diff --git a/src/aios/utils/image_utils.py b/src/aios/utils/image_utils.py new file mode 100644 index 0000000..3d07b13 --- /dev/null +++ b/src/aios/utils/image_utils.py @@ -0,0 +1,40 @@ +import base64 +import os.path +from typing import Tuple + +import cv2 + + +def to_base64(image_path: str, resize: Tuple[int, int] = None) -> str: + """Convert image to base64.""" + ext = os.path.splitext(image_path)[1][1:] + if resize is None: + with open(image_path, "rb") as image_file: + base64_image = base64.b64encode(image_file.read()).decode("utf-8") + return f"data:image/{ext};base64,{base64_image}" + else: + dest_width, dest_height = resize + img = cv2.imread(image_path) + width, height = img.shape[:2] + if width > dest_width or height > dest_height: + width_rate = dest_width / width + height_rate = dest_height / height + rate = min(width_rate, height_rate) + dest_width = int(width * rate) + dest_height = int(height * rate) + img = cv2.resize(img, (dest_width, dest_height), interpolation=cv2.INTER_AREA) + _, buf = cv2.imencode(f".{ext}", img) + base64_image = base64.b64encode(buf).decode("utf-8") + return f"data:image/{ext};base64,{base64_image}" + + +def is_file(image_path: str) -> bool: + return os.path.isfile(image_path) + + +def is_base64(image_path: str) -> bool: + return image_path.startswith("data:image/") + + +def is_url(image_path: str) -> bool: + return image_path.startswith("http://") or image_path.startswith("https://") diff --git a/src/aios/utils/video_utils.py b/src/aios/utils/video_utils.py new file mode 100644 index 0000000..dd687cf --- /dev/null +++ b/src/aios/utils/video_utils.py @@ -0,0 +1,122 @@ +import base64 +from typing import List, Tuple + +import cv2 +import numpy as np + + +def precess_image(image): + ''' + Graying and GaussianBlur + :param image: The image matrix,np.array + :return: The processed image matrix,np.array + ''' + gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + gray_image = cv2.GaussianBlur(gray_image, (3, 3), 0) + return gray_image + + +def abs_diff(pre_image, curr_image): + ''' + Calculate absolute difference between pre_image and curr_image + :param pre_image:The image in past frame,np.array + :param curr_image:The image in current frame,np.array + :return: + ''' + gray_pre_image = precess_image(pre_image) + gray_curr_image = precess_image(curr_image) + diff = cv2.absdiff(gray_pre_image, gray_curr_image) + res, diff = cv2.threshold(diff, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU) + cnt_diff = np.sum(np.sum(diff)) + return cnt_diff + + +def exponential_smoothing(alpha, s): + ''' + Primary exponential smoothing + :param alpha: Smoothing factor,num + :param s: List of data,list + :return: List of data after smoothing,list + ''' + s_temp = [s[0]] + print(s_temp) + for i in range(1, len(s), 1): + s_temp.append(alpha * s[i - 1] + (1 - alpha) * s_temp[i - 1]) + return s_temp + + +def extract_frames(video_path: str, resize: Tuple[int, int] = None, smooth=False, alpha=0.07, window=25) -> List[str]: + """Extract frames from video.""" + frames = [] + vidcap = cv2.VideoCapture(video_path) + diff = [] + frm = 0 + pre_image = np.array([]) + cur_image = np.array([]) + + while True: + frm = frm + 1 + success, image = vidcap.read() + if not success: + break + + if frm == 1: + pre_image = image + cur_image = image + else: + pre_image = cur_image + cur_image = image + + diff.append(abs_diff(pre_image, cur_image)) + + if smooth: + diff = exponential_smoothing(alpha, diff) + + diff = np.array(diff) + mean = np.mean(diff) + dev = np.std(diff) + diff = (diff - mean) / dev + + idx = [] + for i, d in enumerate(diff): + ub = len(diff) - 1 + lb = 0 + if not i - window // 2 < lb: + lb = i - window // 2 + if not i + window // 2 > ub: + ub = i + window // 2 + + comp_window = diff[lb: ub] + if d >= max(comp_window): + idx.append(i) + + tmp = np.array(idx) + tmp = tmp + 1 + idx = set(tmp.tolist()) + vidcap.release() + + vidcap = cv2.VideoCapture(video_path) + i = 0 + frm = 0 + while vidcap.isOpened() and i < 10: + frm = frm + 1 + success, image = vidcap.read() + if not success: + break + if frm not in idx: + continue + if resize is not None: + dest_width, dest_height = resize + width, height = image.shape[:2] + if width > dest_width or height > dest_height: + width_rate = dest_width / width + height_rate = dest_height / height + rate = min(width_rate, height_rate) + dest_width = int(width * rate) + dest_height = int(height * rate) + image = cv2.resize(image, (dest_width, dest_height), interpolation=cv2.INTER_AREA) + _, buffer = cv2.imencode(".jpg", image) + frames.append(f"data:image/jpg;base64,{base64.b64encode(buffer).decode('utf-8')}") + i += 1 + vidcap.release() + return frames diff --git a/src/aios_kernel/image_utils.py b/src/aios_kernel/image_utils.py deleted file mode 100644 index dd706bc..0000000 --- a/src/aios_kernel/image_utils.py +++ /dev/null @@ -1,22 +0,0 @@ -import base64 -import os.path - - -def to_base64(image_path: str) -> str: - """Convert image to base64.""" - ext = os.path.splitext(image_path)[1] - with open(image_path, "rb") as image_file: - base64_image = base64.b64encode(image_file.read()).decode("utf-8") - return f"data:image/{ext};base64,{base64_image}" - - -def is_file(image_path: str) -> bool: - return os.path.isfile(image_path) - - -def is_base64(image_path: str) -> bool: - return image_path.startswith("data:image/") - - -def is_url(image_path: str) -> bool: - return image_path.startswith("http://") or image_path.startswith("https://") diff --git a/src/aios_kernel/video_utils.py b/src/aios_kernel/video_utils.py deleted file mode 100644 index 444fab8..0000000 --- a/src/aios_kernel/video_utils.py +++ /dev/null @@ -1,18 +0,0 @@ -import base64 -from typing import List - -import cv2 - - -def extract_frames(video_path: str) -> List[str]: - """Extract frames from video.""" - frames = [] - vidcap = cv2.VideoCapture(video_path) - while vidcap.isOpened(): - success, image = vidcap.read() - if not success: - break - _, buffer = cv2.imencode(".jpg", image) - frames.append(base64.b64encode(buffer).decode("utf-8")) - vidcap.release() - return frames diff --git a/src/component/openai_node/open_ai_node.py b/src/component/openai_node/open_ai_node.py index 1918b2d..e45dd95 100644 --- a/src/component/openai_node/open_ai_node.py +++ b/src/component/openai_node/open_ai_node.py @@ -8,9 +8,10 @@ import json import aiohttp import base64 import requests +from openai._types import NOT_GIVEN from aios import ComputeTask, ComputeTaskResult, ComputeTaskState, ComputeTaskType,ComputeTaskResultCode,ComputeNode,AIStorage,UserConfig - +from aios import image_utils logger = logging.getLogger(__name__) @@ -102,7 +103,7 @@ class OpenAI_ComputeNode(ComputeNode): image_path = task.params["image_path"] if image_utils.is_file(image_path): - url = image_utils.to_base64(image_path) + url = image_utils.to_base64(image_path, (1024, 1024)) else: url = image_path @@ -201,7 +202,16 @@ class OpenAI_ComputeNode(ComputeNode): if max_token_size is None: max_token_size = 4000 - result_token = max_token_size + if mode_name == "gpt-4-vision-preview": + response_format = NOT_GIVEN + llm_inner_functions = None + if max_token_size > 4096: + result_token = 4096 + else: + result_token = max_token_size + else: + result_token = NOT_GIVEN + client = AsyncOpenAI(api_key=self.openai_api_key) try: if llm_inner_functions is None: @@ -209,7 +219,7 @@ class OpenAI_ComputeNode(ComputeNode): resp = await client.chat.completions.create(model=mode_name, messages=prompts, response_format = response_format, - #max_tokens=result_token, + max_tokens=result_token, ) else: logger.info(f"call openai {mode_name} prompts: \n\t {prompts} \nfunctions: \n\t{json.dumps(llm_inner_functions)}") @@ -217,7 +227,7 @@ class OpenAI_ComputeNode(ComputeNode): messages=prompts, response_format = response_format, functions=llm_inner_functions, - # max_tokens=result_token, + max_tokens=result_token, ) # TODO: add temperature to task params? except Exception as e: logger.error(f"openai run LLM_COMPLETION task error: {e}") @@ -227,7 +237,12 @@ class OpenAI_ComputeNode(ComputeNode): return result logger.info(f"openai response: {resp}") - status_code = resp.choices[0].finish_reason + if mode_name == "gpt-4-vision-preview": + status_code = resp.choices[0].finish_reason + if status_code is None: + status_code = resp.choices[0].finish_details['type'] + else: + status_code = resp.choices[0].finish_reason token_usage = resp.usage match status_code: diff --git a/src/service/aios_shell/aios_shell.py b/src/service/aios_shell/aios_shell.py index 16de836..f299d76 100644 --- a/src/service/aios_shell/aios_shell.py +++ b/src/service/aios_shell/aios_shell.py @@ -31,7 +31,7 @@ from aios import * sys.path.append(directory + '/../../component/') -from google_node import * +from google_node import * from llama_node import * from openai_node import * from sd_node import * @@ -460,6 +460,54 @@ class AIOS_Shell: show_text = FormattedText([("class:title", f"{self.current_topic}@{self.current_target} >>> "), ("class:content", resp)]) return show_text + case 'send_img': + sender = None + if len(args) == 4: + target_id = args[0] + msg_content = args[1] + image_path = args[2] + topic = args[3] + sender = self.username + elif len(args) == 5: + target_id = args[0] + msg_content = args[1] + image_path = args[2] + topic = args[3] + sender = args[4] + + ext = os.path.splitext(image_path)[1][1:] + resp = await self.send_msg(AgentMsg.create_image_body([image_path], msg_content), + target_id, + topic, + sender, + f"image/{ext}") + show_text = FormattedText([("class:title", f"{self.current_topic}@{self.current_target} >>> "), + ("class:content", resp)]) + return show_text + case 'send_video': + sender = None + if len(args) == 4: + target_id = args[0] + msg_content = args[1] + video_path = args[2] + topic = args[3] + sender = self.username + elif len(args) == 5: + target_id = args[0] + msg_content = args[1] + video_path = args[2] + topic = args[3] + sender = args[4] + + ext = os.path.splitext(video_path)[1][1:] + resp = await self.send_msg(AgentMsg.create_video_body(video_path, msg_content), + target_id, + topic, + sender, + f"video/{ext}") + show_text = FormattedText([("class:title", f"{self.current_topic}@{self.current_target} >>> "), + ("class:content", resp)]) + return show_text case 'set_config': show_text = FormattedText([("class:error", f"set config args error,/set_config $config_item! ")]) if len(args) == 1: @@ -775,6 +823,8 @@ async def main(): return await main_daemon_loop(shell) completer = WordCompleter(['/send $target $msg $topic', + '/send_img $target $msg $img_path $topic', + '/send_video $target &msg &video_path $topic', '/open $target $topic', '/history $num $offset', '/connect $target',