From dfcc5efaa08a6475bf533556ecda4aa5456f2421 Mon Sep 17 00:00:00 2001 From: Liu Zhicong Date: Mon, 19 Aug 2024 12:20:56 -0700 Subject: [PATCH] disable vector-base knowledge base. --- .../JarvisPlus/pipeline.toml | 8 - .../knowledge_pipelines/Mail/Issue/local.py | 9 - .../knowledge_pipelines/Mail/Issue/parser.py | 10 - .../Mail/Issue/pipeline.toml | 13 -- rootfs/knowledge_pipelines/Mail/Sync/input.py | 10 - .../Mail/Sync/pipeline.toml | 14 -- rootfs/knowledge_pipelines/Mia/input.py | 211 ----------------- rootfs/knowledge_pipelines/Mia/parser.py | 101 --------- rootfs/knowledge_pipelines/Mia/pipeline.toml | 6 - rootfs/knowledge_pipelines/Mia/query.py | 96 -------- rootfs/knowledge_pipelines/pipelines.toml | 3 - src/aios/agent/agent_memory.py | 12 +- src/aios/knowledge/__init__.py | 4 +- src/aios/knowledge/pipeline.py | 133 ----------- src/aios/knowledge/vector/__init__.py | 2 - src/aios/knowledge/vector/chroma_store.py | 51 ----- src/aios/knowledge/vector/vector_base.py | 16 -- src/aios/proto/agent_msg.py | 2 +- src/component/common_environment/__init__.py | 2 +- src/component/discord_tunnel.py | 42 ++-- src/component/slack_tunnel.py | 42 ++-- src/component/st_node/__init__.py | 1 - .../st_node/local_st_compute_node.py | 212 ------------------ src/component/tg_tunnel.py | 124 +++++----- src/requirements.txt | Bin 2530 -> 7112 bytes src/requirements.txt.bak | 159 +++++++++++++ src/service/aios_shell/aios_shell.py | 38 ++-- 27 files changed, 297 insertions(+), 1024 deletions(-) delete mode 100644 rootfs/knowledge_pipelines/JarvisPlus/pipeline.toml delete mode 100644 rootfs/knowledge_pipelines/Mail/Issue/local.py delete mode 100644 rootfs/knowledge_pipelines/Mail/Issue/parser.py delete mode 100644 rootfs/knowledge_pipelines/Mail/Issue/pipeline.toml delete mode 100644 rootfs/knowledge_pipelines/Mail/Sync/input.py delete mode 100644 rootfs/knowledge_pipelines/Mail/Sync/pipeline.toml delete mode 100644 rootfs/knowledge_pipelines/Mia/input.py delete mode 100644 rootfs/knowledge_pipelines/Mia/parser.py delete mode 100644 rootfs/knowledge_pipelines/Mia/pipeline.toml delete mode 100644 rootfs/knowledge_pipelines/Mia/query.py delete mode 100644 rootfs/knowledge_pipelines/pipelines.toml delete mode 100644 src/aios/knowledge/pipeline.py delete mode 100644 src/aios/knowledge/vector/__init__.py delete mode 100644 src/aios/knowledge/vector/chroma_store.py delete mode 100644 src/aios/knowledge/vector/vector_base.py delete mode 100644 src/component/st_node/__init__.py delete mode 100644 src/component/st_node/local_st_compute_node.py create mode 100644 src/requirements.txt.bak diff --git a/rootfs/knowledge_pipelines/JarvisPlus/pipeline.toml b/rootfs/knowledge_pipelines/JarvisPlus/pipeline.toml deleted file mode 100644 index ff947ad..0000000 --- a/rootfs/knowledge_pipelines/JarvisPlus/pipeline.toml +++ /dev/null @@ -1,8 +0,0 @@ -name = "JarvisPlus" -input.module = "scan_local" -input.params.workspace = "${myai_dir}/workspace/JarvisPlus" -input.params.path = "${myai_dir}/data" -parser.module = "parse_local" -parser.params.workspace = "${myai_dir}/workspace/JarvisPlus" -parser.params.assign_to = "JarvisPlus" - diff --git a/rootfs/knowledge_pipelines/Mail/Issue/local.py b/rootfs/knowledge_pipelines/Mail/Issue/local.py deleted file mode 100644 index 7c08bfd..0000000 --- a/rootfs/knowledge_pipelines/Mail/Issue/local.py +++ /dev/null @@ -1,9 +0,0 @@ -import sys -import os -from aios import KnowledgePipelineEnvironment -directory = os.path.dirname(__file__) -sys.path.append(directory + '/../../../../src/component/') - -from mail_environment import LocalEmail -def init(env: KnowledgePipelineEnvironment, params: dict): - return LocalEmail(env, params) \ No newline at end of file diff --git a/rootfs/knowledge_pipelines/Mail/Issue/parser.py b/rootfs/knowledge_pipelines/Mail/Issue/parser.py deleted file mode 100644 index fd0779e..0000000 --- a/rootfs/knowledge_pipelines/Mail/Issue/parser.py +++ /dev/null @@ -1,10 +0,0 @@ -import sys -import os -from aios import * -directory = os.path.dirname(__file__) - -sys.path.append(directory + '/../../../../src/component/') -from mail_environment import IssueParser - -def init(env: KnowledgePipelineEnvironment, params: dict): - return IssueParser(env, params) \ No newline at end of file diff --git a/rootfs/knowledge_pipelines/Mail/Issue/pipeline.toml b/rootfs/knowledge_pipelines/Mail/Issue/pipeline.toml deleted file mode 100644 index 4cb377d..0000000 --- a/rootfs/knowledge_pipelines/Mail/Issue/pipeline.toml +++ /dev/null @@ -1,13 +0,0 @@ -name = "Mail.Issue" -input.module = "local.py" -input.params.path = "${myai_dir}/mail" -input.params.watch = true -parser.module = "parser.py" -parser.params.mail_path = "${myai_dir}/mail" -parser.params.issue_path = "${myai_dir}/mail/issue.json" -[parser.params.root_issue] -summary = "巴克云公司推进中的项目" -[[parser.params.root_issue.children]] -summary = "去中心存储项目DMC" - - diff --git a/rootfs/knowledge_pipelines/Mail/Sync/input.py b/rootfs/knowledge_pipelines/Mail/Sync/input.py deleted file mode 100644 index d5e6958..0000000 --- a/rootfs/knowledge_pipelines/Mail/Sync/input.py +++ /dev/null @@ -1,10 +0,0 @@ -import sys -import os -from knowledge import KnowledgePipelineEnvironment -directory = os.path.dirname(__file__) -sys.path.append(directory + '/../../../../src/component/') - -from mail_environment import EmailSpider - -def init(env: KnowledgePipelineEnvironment, params: dict): - return EmailSpider(env, params) \ No newline at end of file diff --git a/rootfs/knowledge_pipelines/Mail/Sync/pipeline.toml b/rootfs/knowledge_pipelines/Mail/Sync/pipeline.toml deleted file mode 100644 index 17fa1e2..0000000 --- a/rootfs/knowledge_pipelines/Mail/Sync/pipeline.toml +++ /dev/null @@ -1,14 +0,0 @@ -name = "Mail.Sync" -input.module = "input.py" -[input.params] -path = "${myai_dir}/mail" -imap_server = "imap.qq.com" -imap_port = 993 -address = "115620204@qq.com" -password = "zbbjpbukeonqbjja" -[input.params.fields] -from = "from" -to = "to" -subject = "subject" - - diff --git a/rootfs/knowledge_pipelines/Mia/input.py b/rootfs/knowledge_pipelines/Mia/input.py deleted file mode 100644 index 4c0aade..0000000 --- a/rootfs/knowledge_pipelines/Mia/input.py +++ /dev/null @@ -1,211 +0,0 @@ -import copy -import os -from typing import List - -import aiofiles -import chardet -import logging -import string -import docx2txt -from PyPDF2 import PdfReader - -from aios import KnowledgePipelineEnvironment, ImageObjectBuilder, DocumentObjectBuilder, KnowledgeStore, RichTextObject -from aios.agent.agent_base import AgentPrompt -from aios.frame.compute_kernel import ComputeKernel -from aios.knowledge.data.writer import split_text -from aios.proto.compute_task import ComputeTaskResult, ComputeTaskResultCode -from aios.storage.storage import AIStorage -from aios.utils import video_utils, image_utils - - -class KnowledgeDirSource: - def __init__(self, env: KnowledgePipelineEnvironment, config): - self.env = env - path = string.Template(config["path"]).substitute(myai_dir=AIStorage.get_instance().get_myai_dir()) - config["path"] = path - self.config = config - - # @classmethod - # def user_config_items(cls): - # return [("path", "local dir path")] - - def path(self): - return self.config["path"] - - @staticmethod - async def read_txt_file(file_path:str)->str: - cur_encode = "utf-8" - async with aiofiles.open(file_path,'rb') as f: - cur_encode = chardet.detect(await f.read())['encoding'] - - async with aiofiles.open(file_path,'r',encoding=cur_encode) as f: - return await f.read() - - async def next(self): - while True: - journals = self.env.journal.latest_journals(1) - from_time = 0 - if len(journals) == 1: - latest_journal = journals[0] - if latest_journal.is_finish(): - yield None - continue - from_time = os.path.getctime(latest_journal.get_input()) - if os.path.getmtime(self.path()) <= from_time: - yield (None, None) - continue - - file_pathes = sorted(os.listdir(self.path()), key=lambda x: os.path.getctime(os.path.join(self.path(), x))) - for rel_path in file_pathes: - file_path = os.path.join(self.path(), rel_path) - timestamp = os.path.getctime(file_path) - if timestamp <= from_time: - continue - ext = os.path.splitext(file_path)[1].lower() - if ext in ['.jpg', '.jpeg', '.png', '.gif', '.bmp']: - logging.info(f"knowledge dir source found image file {file_path}") - image = ImageObjectBuilder({}, {}, file_path).build(self.env.get_knowledge_store()) - await self.env.get_knowledge_store().insert_object(image) - yield (image.calculate_id(), file_path) - if ext in ['.txt']: - logging.info(f"knowledge dir source found text file {file_path}") - text = await self.read_txt_file(file_path) - document = DocumentObjectBuilder({}, {}, text).build(self.env.get_knowledge_store()) - await self.env.get_knowledge_store().insert_object(document) - yield (document.calculate_id(), file_path) - yield (None, None) - - -def init(env: KnowledgePipelineEnvironment, params: dict) -> KnowledgeDirSource: - return KnowledgeDirSource(env, params) - - -async def image_to_text(images: List[str]) -> str: - msg_prompt = AgentPrompt() - image_prompt = "What's in this image?" - content = [{"type": "text", "text": image_prompt}] - content.extend([{"type": "image_url", "image_url": {"url": image_utils.to_base64(image)}} for image in images]) - msg_prompt.messages = [{"role": "user", "content": content}] - - resp: ComputeTaskResult = await (ComputeKernel.get_instance() - .do_llm_completion(prompt=msg_prompt, - resp_mode="text", - mode_name="gpt-4-vision-preview", - max_token=4000, - inner_functions=None, - timeout=None)) - if resp.result_code != ComputeTaskResultCode.OK: - raise Exception(f"image_to_text error: {resp.result_code} msg:{resp.error_str}") - return resp.result_str - - -async def video_to_text(video: str) -> str: - prompt = "These pictures are key frames extracted from the video. Please describe the content of the video based on these key frames." - frames = video_utils.extract_frames(video, (1024, 1024)) - msg_prompt = AgentPrompt() - content = [{"type": "text", "text": prompt}] - content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames]) - msg_prompt.messages = [{"role": "user", "content": content}] - resp: ComputeTaskResult = await (ComputeKernel.get_instance() - .do_llm_completion(prompt=msg_prompt, - resp_mode="text", - mode_name="gpt-4-vision-preview", - max_token=4000, - inner_functions=None, - timeout=None)) - if resp.result_code != ComputeTaskResultCode.OK: - raise Exception(f"video_to_text error: {resp.result_code} msg:{resp.error_str}") - return resp.result_str - - -async def summary_document(text: str, separators: List[str]=["\n\n", "\n"]) -> str: - chunks = split_text(text, separators=separators, chunk_size=4000, chunk_overlap=200, length_function=len) - - prompt = AgentPrompt() - prompt.system_message = {"role":"system","content":"Your job is to generate a summary based on the input."} - if len(chunks) == 1: - prompt.append(AgentPrompt(chunks[0])) - resp = await (ComputeKernel.get_instance() - .do_llm_completion(prompt=prompt, - resp_mode="text", - mode_name="gpt-4-1106-preview", - max_token=4000, - inner_functions=None, - timeout=None)) - if resp.result_code != ComputeTaskResultCode.OK: - raise Exception(f"summary_document error: {resp.result_code} msg:{resp.error_str}") - return resp.result_str - - segments = [] - for i, chunk in enumerate(chunks): - seg_prompt = copy.deepcopy(prompt) - seg_prompt.append(AgentPrompt(chunk)) - resp = await (ComputeKernel.get_instance() - .do_llm_completion(prompt=seg_prompt, - resp_mode="text", - mode_name="gpt-4-1106-preview", - max_token=4000, - inner_functions=None, - timeout=None)) - if resp.result_code != ComputeTaskResultCode.OK: - raise Exception(f"summary_document error: {resp.result_code} msg:{resp.error_str}") - segments.append(resp.result_str) - - segments_str = "\n".join(segments) - prompt.append(AgentPrompt(f"Please combine the summaries of the following paragraphs into one complete summary:\n{segments_str}")) - resp = await (ComputeKernel.get_instance() - .do_llm_completion(prompt=prompt, - resp_mode="text", - mode_name="gpt-4-1106-preview", - max_token=4000, - inner_functions=None, - timeout=None)) - if resp.result_code != ComputeTaskResultCode.OK: - raise Exception(f"summary_document error: {resp.result_code} msg:{resp.error_str}") - return resp.result_str - - - -def pdf_to_rich_text_object(pdf: str, store: KnowledgeStore) -> RichTextObject: - base_name = os.path.basename(pdf) - cache_path = os.path.join(AIStorage.get_instance().get_myai_dir(), "knowledge", "doc_cache", base_name) - if not os.path.exists(cache_path): - os.makedirs(cache_path) - - reader = PdfReader(pdf) - rich_text = RichTextObject() - page_texts = [] - image_count = 0 - for page in reader.pages: - text = page.extract_text() - page_texts.append(text) - for image in page.images: - image_path = os.path.join(cache_path, f"{image_count}_{image.name}") - with open(image_path, "wb") as f: - f.write(image.data) - image_object = ImageObjectBuilder({}, {}, image_path).build(store) - rich_text.add_image(image_object) - - document = DocumentObjectBuilder({}, {}, "".join(page_texts)).build(store) - rich_text.add_document(document) - - return rich_text - - -def doc_to_rich_text_object(doc: str, store: KnowledgeStore) -> RichTextObject: - base_name = os.path.basename(doc) - cache_path = os.path.join(AIStorage.get_instance().get_myai_dir(), "knowledge", "doc_cache", base_name) - if not os.path.exists(cache_path): - os.makedirs(cache_path) - text = docx2txt.process(doc, cache_path) - - rich_text = RichTextObject() - for image in os.listdir(cache_path): - image_path = os.path.join(cache_path, image) - image_object = ImageObjectBuilder({}, {}, image_path).build(store) - rich_text.add_image(image_object) - - document = DocumentObjectBuilder({}, {}, text).build(store) - rich_text.add_document(document) - - return rich_text diff --git a/rootfs/knowledge_pipelines/Mia/parser.py b/rootfs/knowledge_pipelines/Mia/parser.py deleted file mode 100644 index 59c2284..0000000 --- a/rootfs/knowledge_pipelines/Mia/parser.py +++ /dev/null @@ -1,101 +0,0 @@ -# define a knowledge base class -import json -import string -from aios import * - - -class EmbeddingParser: - def __init__(self, env: KnowledgePipelineEnvironment, config: dict): - self._default_text_model = "all-MiniLM-L6-v2" - self._default_image_model = "clip-ViT-B-32" - - path = string.Template(config["path"]).substitute(myai_dir=AIStorage.get_instance().get_myai_dir()) - if not os.path.exists(path): - os.makedirs(path) - config["path"] = path - - self.env = env - self.config = config - - def get_path(self) -> str: - return self.config["path"] - - def __get_vector_store(self, model_name: str) -> ChromaVectorStore: - return ChromaVectorStore(self.get_path(), model_name) - - async def __embedding_document(self, document: DocumentObject): - for chunk_id in document.get_chunk_list(): - chunk = self.env.get_knowledge_store().get_chunk_reader().get_chunk(chunk_id) - if chunk is None: - raise ValueError(f"text chunk not found: {chunk_id}") - - text = chunk.read().decode("utf-8") - vector = await ComputeKernel.get_instance().do_text_embedding(text, self._default_text_model) - if vector: - await self.__get_vector_store(self._default_text_model).insert(vector, chunk_id) - - async def __embedding_image(self, image: ImageObject): - # desc = {} - # if not not image.get_meta(): - # desc["meta"] = image.get_meta() - # if not not image.get_exif(): - # desc["exif"] = image.get_exif() - # if not not image.get_tags(): - # desc["tags"] = image.get_tags() - # vector = await self.compute_kernel.do_text_embedding(json.dumps(desc), self._default_text_model) - vector = await ComputeKernel.get_instance().do_image_embedding(image.calculate_id(), self._default_image_model) - if vector: - await self.__get_vector_store(self._default_image_model).insert(vector, image.calculate_id()) - - async def __embedding_video(self, vedio: VideoObject): - desc = {} - if not not vedio.get_meta(): - desc["meta"] = vedio.get_meta() - if not not vedio.get_info(): - desc["info"] = vedio.get_info() - if not not vedio.get_tags(): - desc["tags"] = vedio.get_tags() - vector = await ComputeKernel.get_instance().do_text_embedding(json.dumps(desc), self._default_text_model) - await self.__get_vector_store(self._default_text_model).insert(vector, vedio.calculate_id()) - - async def __embedding_rich_text(self, rich_text: RichTextObject): - for document_id in rich_text.get_documents().values(): - document = DocumentObject.decode(self.env.get_knowledge_store().get_object_store().get_object(document_id)) - await self.__embedding_document(document) - for image_id in rich_text.get_images().values(): - image = ImageObject.decode(self.env.get_knowledge_store().get_object_store().get_object(image_id)) - await self.__embedding_image(image) - for video_id in rich_text.get_videos().values(): - video = VideoObject.decode(self.env.get_knowledge_store().get_object_store().get_object(video_id)) - await self.__embedding_video(video) - for rich_text_id in rich_text.get_rich_texts().values(): - rich_text = RichTextObject.decode(self.env.get_knowledge_store().get_object_store().get_object(rich_text_id)) - await self.__embedding_rich_text(rich_text) - - async def __embedding_email(self, email: EmailObject): - vector = await ComputeKernel.get_instance().do_text_embedding(json.dumps(email.get_desc()), self._default_text_model) - await self.__get_vector_store(self._default_text_model).insert(vector, email.calculate_id()) - await self.__embedding_rich_text(email.get_rich_text()) - - - async def __do_embedding(self, object: KnowledgeObject): - if object.get_object_type() == ObjectType.Document: - await self.__embedding_document(object) - if object.get_object_type() == ObjectType.Image: - await self.__embedding_image(object) - if object.get_object_type() == ObjectType.Video: - await self.__embedding_video(object) - if object.get_object_type() == ObjectType.RichText: - await self.__embedding_rich_text(object) - if object.get_object_type() == ObjectType.Email: - await self.__embedding_email(object) - else: - pass - - async def parse(self, object: ObjectID) -> str: - obj = self.env.get_knowledge_store().load_object(object) - await self.__do_embedding(obj) - return str(object) - -def init(env: KnowledgePipelineEnvironment, params: dict) -> EmbeddingParser: - return EmbeddingParser(env, params) \ No newline at end of file diff --git a/rootfs/knowledge_pipelines/Mia/pipeline.toml b/rootfs/knowledge_pipelines/Mia/pipeline.toml deleted file mode 100644 index 9b9e062..0000000 --- a/rootfs/knowledge_pipelines/Mia/pipeline.toml +++ /dev/null @@ -1,6 +0,0 @@ -name = "Mia" -input.module = "input.py" -input.params.path = "${myai_dir}/data" -parser.module = "parser.py" -parser.params.path = "${myai_dir}/knowledge/indices/embedding" - diff --git a/rootfs/knowledge_pipelines/Mia/query.py b/rootfs/knowledge_pipelines/Mia/query.py deleted file mode 100644 index 2c0c801..0000000 --- a/rootfs/knowledge_pipelines/Mia/query.py +++ /dev/null @@ -1,96 +0,0 @@ -import os -import logging -import json -from aios import * - -class EmbeddingEnvironment(SimpleEnvironment): - def __init__(self, workspace: str) -> None: - super().__init__(workspace) - self.path = os.path.join(AIStorage.get_instance().get_myai_dir(), "knowledge/indices/embedding") - self._default_text_model = "all-MiniLM-L6-v2" - self._default_image_model = "clip-ViT-B-32" - - query_param = { - "tokens": "key words to query", - "types": "prefered knowledge types, one or more of [text, image]", - "limit": "index of query result" - } - self.add_ai_function(SimpleAIFunction("query_knowledge", - "vector query content from local knowledge base", - self._query, - query_param)) - - def __get_vector_store(self, model_name: str) -> ChromaVectorStore: - return ChromaVectorStore(self.path, model_name) - - async def query_objects(self, tokens: str, types: list[str], topk: int) -> [ObjectID]: - texts = [] - if "text" in types: - vector = await ComputeKernel.get_instance().do_text_embedding(tokens, self._default_text_model) - texts = await self.__get_vector_store(self._default_text_model).query(vector, topk) - images = [] - if "image" in types: - vector = await ComputeKernel.get_instance().do_text_embedding(tokens, self._default_image_model) - images = await self.__get_vector_store(self._default_image_model).query(vector, topk) - return texts + images - - - def tokens_from_objects(self, object_ids: [ObjectID]) -> list[str]: - results = dict() - for object_id in object_ids: - parents = KnowledgeStore().get_relation_store().get_related_root_objects(object_id) - # last parent is the root object - root_object_id = parents[0] if parents else object_id - logging.info(f"object_id: {str(object_id)} root_object_id: {str(root_object_id)}") - if str(root_object_id) in results: - results[str(root_object_id)].append(object_id) - else: - results[str(root_object_id)] = [root_object_id, object_id] - content = "" - result_desc = [] - for result in results.values(): - # first element in result is the root object - root_object_id = result[0] - if root_object_id.get_object_type() == ObjectType.Email: - email = KnowledgeStore().load_object(root_object_id) - desc = email.get_desc() - desc["type"] = "email" - desc["contents"] = [] - result_desc.append(desc) - upper_list = desc["contents"] - result = result[1:] - else: - upper_list = result_desc - - for object_id in result: - if object_id.get_object_type() == ObjectType.Chunk: - upper_list.append({"type": "text", "content": KnowledgeStore().get_chunk_reader().get_chunk(object_id).read().decode("utf-8")}) - if object_id.get_object_type() == ObjectType.Image: - # image = self.load_object(object_id) - desc = dict() - desc["id"] = str(object_id) - desc["type"] = "image" - upper_list.append(desc) - if object_id.get_object_type() == ObjectType.Video: - video = KnowledgeStore().load_object(object_id) - desc = video.get_desc() - desc["type"] = "video" - upper_list.append(desc) - else: - pass - content += json.dumps(result_desc) - content += ".\n" - - return content - - async def _query(self, tokens: str, types: list[str] = ["text"], index: str=0): - index = int(index) - object_ids = await self.query_objects(tokens, types, 4) - if len(object_ids) <= index: - return "*** I have no more information for your reference.\n" - else: - content = "*** I have provided the following known information for your reference with json format:\n" - return content + self.tokens_from_objects(object_ids[index:index+1]) - -def init(workspace: str) -> EmbeddingEnvironment: - return EmbeddingEnvironment(workspace) \ No newline at end of file diff --git a/rootfs/knowledge_pipelines/pipelines.toml b/rootfs/knowledge_pipelines/pipelines.toml deleted file mode 100644 index a1eaa37..0000000 --- a/rootfs/knowledge_pipelines/pipelines.toml +++ /dev/null @@ -1,3 +0,0 @@ -pipelines = [ - "JarvisPlus" -] \ No newline at end of file diff --git a/src/aios/agent/agent_memory.py b/src/aios/agent/agent_memory.py index a3307e2..c0c1e07 100644 --- a/src/aios/agent/agent_memory.py +++ b/src/aios/agent/agent_memory.py @@ -36,7 +36,7 @@ logger = logging.getLogger(__name__) class AgentMemory: - def __init__(self,agent_id:str,base_dir:str,enable_knowledge_graph = True) -> None: + def __init__(self,agent_id:str,base_dir:str,enable_knowledge_graph = False) -> None: self.agent_memory_base_dir = base_dir self.agent_id:str= agent_id @@ -52,15 +52,17 @@ class AgentMemory: self.last_think_time : float = 0.0 self.enable_knowledge_graph : bool = enable_knowledge_graph if self.enable_knowledge_graph: - kb_desc = """The Knowledgegraph is used to store important information obtained by Agent in the conversation.Use the following ways to store information: - /contacts/$name:Related information of the contact - /relations/$obj1/$obj2:The relationship between obj2 and obj1 - /summary/$topic:Based on topic summary + kb_desc = """ +The Knowledgegraph is used to store important information obtained by Agent in the conversation.Use the following ways to store information: +- /contacts/$name:Related information of the contact +- /relations/$obj1/$obj2:The relationship between obj2 and obj1 +- /summary/$topic:Based on topic summary """ self.knowledge_graph = ObjFSKnowledgeGrpah(f"{self.agent_id}.memory",self.memory_db,kb_desc) BaseKnowledgeGraph.add_kb(self.knowledge_graph) self.simple_memory_sentences = None + else: self.knowledge_graph = None self.simple_memory_sentences : List[str] = [] diff --git a/src/aios/knowledge/__init__.py b/src/aios/knowledge/__init__.py index e0d5dfd..19d4601 100644 --- a/src/aios/knowledge/__init__.py +++ b/src/aios/knowledge/__init__.py @@ -1,7 +1,7 @@ from .object import * -from .vector import * +#from .vector import * from .data import * from .store import KnowledgeStore from .core_object import * -from .pipeline import * +#from .pipeline import * from .knowledge_base import * \ No newline at end of file diff --git a/src/aios/knowledge/pipeline.py b/src/aios/knowledge/pipeline.py deleted file mode 100644 index a9a2b58..0000000 --- a/src/aios/knowledge/pipeline.py +++ /dev/null @@ -1,133 +0,0 @@ -import datetime -import sqlite3 -import os -import logging -from . import ObjectID, KnowledgeStore -from enum import Enum - -class KnowledgePipelineJournal: - def __init__(self, time: datetime.datetime, input: str, parser: str): - self.time = time - self.input = input - self.parser = parser - - def is_finish(self) -> bool: - return self.input is None - - def get_input(self) -> str: - return self.input - - def get_parser(self) -> str: - return self.parser - - def __str__(self) -> str: - if self.is_finish(): - return f"{self.time}: finished)" - else: - return f"{self.time}: input:{self.input}, parser:{self.parser})" - -# init sqlite3 client -class KnowledgePipelineJournalClient: - def __init__(self, pipeline_path: str = None): - if not os.path.exists(pipeline_path): - os.makedirs(pipeline_path) - self.journal_path = os.path.join(pipeline_path, "journal.db") - - conn = sqlite3.connect(self.journal_path) - conn.execute( - '''CREATE TABLE IF NOT EXISTS journal ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - time DATETIME DEFAULT CURRENT_TIMESTAMP, - input TEXT, - parser TEXT)''' - ) - conn.commit() - - def insert(self, input: str, parser: str, timestamp: datetime.datetime = None): - timestamp = datetime.datetime.now() if timestamp is None else timestamp - conn = sqlite3.connect(self.journal_path) - conn.execute( - "INSERT INTO journal (time, input, parser) VALUES (?, ?, ?)", - (timestamp, input, parser), - ) - conn.commit() - - def latest_journals(self, topn) -> [KnowledgePipelineJournal]: - conn = sqlite3.connect(self.journal_path) - cursor = conn.cursor() - cursor.execute("SELECT * FROM journal ORDER BY id DESC LIMIT ?", (topn,)) - return [KnowledgePipelineJournal(time, input, parser) for (_, time, input, parser) in cursor.fetchall()] - -class KnowledgePipelineEnvironment: - def __init__(self, pipeline_path: str): - self.knowledge_store = KnowledgeStore() - if not os.path.exists(pipeline_path): - os.makedirs(pipeline_path) - self.pipeline_path = pipeline_path - self.journal = KnowledgePipelineJournalClient(pipeline_path) - self.logger = logging.getLogger() - - def get_journal(self) -> KnowledgePipelineJournalClient: - return self.journal - - def get_knowledge_store(self) -> KnowledgeStore: - return self.knowledge_store - - def get_logger(self) -> logging.Logger: - return self.logger - -class KnowledgePipelineState(Enum): - INIT = 0 - RUNNING = 1 - STOPPED = 2 - FINISHED = 3 - -class NullParser: - async def parse(self, object_id): - return "" - -class KnowledgePipeline: - def __init__(self, name: str, env: KnowledgePipelineEnvironment, input_init, input_params=None, parser_init=None, parser_params=None): - self.name = name - self.state = KnowledgePipelineState.INIT - self.input_init = input_init - self.input_params = input_params - self.parser_init = parser_init - self.parser_params = parser_params - self.env = env - self.input = None - self.parser = None - - def get_name(self): - return self.name - - def get_journal(self) -> KnowledgePipelineJournalClient: - return self.env.journal - - async def run(self): - if self.state == KnowledgePipelineState.INIT: - self.input = self.input_init(self.env, self.input_params) - if self.parser_init is None: - self.parser = NullParser() - else: - self.parser = self.parser_init(self.env, self.parser_params) - self.state = KnowledgePipelineState.RUNNING - if self.state == KnowledgePipelineState.RUNNING: - async for input in self.input.next(): - if input is None: - self.state = KnowledgePipelineState.FINISHED - self.env.journal.insert(None, None) - return - (object_id, input_journal) = input - if object_id is not None: - parser_journal = await self.parser.parse(object_id) - self.env.journal.insert(input_journal, parser_journal) - else: - return - if self.state == KnowledgePipelineState.STOPPED: - return - if self.state == KnowledgePipelineState.FINISHED: - return - - - \ No newline at end of file diff --git a/src/aios/knowledge/vector/__init__.py b/src/aios/knowledge/vector/__init__.py deleted file mode 100644 index b3f0d78..0000000 --- a/src/aios/knowledge/vector/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .vector_base import VectorBase -from .chroma_store import ChromaVectorStore diff --git a/src/aios/knowledge/vector/chroma_store.py b/src/aios/knowledge/vector/chroma_store.py deleted file mode 100644 index 8cd3084..0000000 --- a/src/aios/knowledge/vector/chroma_store.py +++ /dev/null @@ -1,51 +0,0 @@ -from .vector_base import VectorBase -from ..object import ObjectID -import chromadb -import logging -import os - - -class ChromaVectorStore(VectorBase): - def __init__(self, root_dir, model_name: str) -> None: - super().__init__(model_name) - - logging.info( - "will init chroma vector store, model={}".format(model_name) - ) - - directory = os.path.join(root_dir, "vector") - logging.info("will use vector store: {}".format(directory)) - - client = chromadb.PersistentClient( - path=directory, settings=chromadb.Settings(anonymized_telemetry=False) - ) - # client = chromadb.Client() - - collection_name = "coll_{}".format(model_name) - logging.info("will init chroma colletion: %s", collection_name) - - collection = client.get_or_create_collection(collection_name) - self.collection = collection - - async def insert(self, vector: [float], id: ObjectID): - logging.info(f"will insert vector: {len(vector)} id: {str(id)}") - logging.debug(f"vector is {vector}") - self.collection.add( - embeddings=vector, - ids=str(id), - ) - - async def query(self, vector: [float], top_k: int) -> [ObjectID]: - ret = self.collection.query( - query_embeddings=vector, - n_results=top_k, - ) - logging.info(f"query result {ret}") - if len(ret['ids']) == 0: - return [] - return list(map(ObjectID.from_base58, ret["ids"][0])) - - async def delete(self, id: ObjectID): - self.collection.delete( - ids=id, - ) diff --git a/src/aios/knowledge/vector/vector_base.py b/src/aios/knowledge/vector/vector_base.py deleted file mode 100644 index 83276ed..0000000 --- a/src/aios/knowledge/vector/vector_base.py +++ /dev/null @@ -1,16 +0,0 @@ -# import the ObjectID class -from ..object import ObjectID - -# define a vector base class -class VectorBase: - def __init__(self, model_name) -> None: - self.model_name = model_name - - async def insert(self, vector: [float], id: ObjectID): - pass - - async def query(self, vector: [float], top_k: int) -> [ObjectID]: - pass - - async def delete(self, id: ObjectID): - pass \ No newline at end of file diff --git a/src/aios/proto/agent_msg.py b/src/aios/proto/agent_msg.py index b9665c4..11c0552 100644 --- a/src/aios/proto/agent_msg.py +++ b/src/aios/proto/agent_msg.py @@ -61,7 +61,7 @@ class AgentMsg: self.mentions:[] = None #use in group chat only #self.title:str = None self.body:str = None - self.body_mime:str = None #//default is "text/plain",encode is utf8 + self.body_mime:str = "text/plain" #//default is "text/plain",encode is utf8 #type is call / action self.func_name = None diff --git a/src/component/common_environment/__init__.py b/src/component/common_environment/__init__.py index 5587238..8fb9cbb 100644 --- a/src/component/common_environment/__init__.py +++ b/src/component/common_environment/__init__.py @@ -1,3 +1,3 @@ -from .local_document import LocalKnowledgeBase, ScanLocalDocument, ParseLocalDocument +#from .local_document import LocalKnowledgeBase, ScanLocalDocument, ParseLocalDocument from .local_file_system import FilesystemEnvironment from .shell import ShellEnvironment \ No newline at end of file diff --git a/src/component/discord_tunnel.py b/src/component/discord_tunnel.py index 1332e58..5c43c8c 100644 --- a/src/component/discord_tunnel.py +++ b/src/component/discord_tunnel.py @@ -9,7 +9,7 @@ import aiofiles from urllib.parse import urlparse from typing import Optional -from aios import KnowledgeStore, ObjectType +#from aios import KnowledgeStore, ObjectType from aios.frame.tunnel import AgentTunnel from aios.proto.agent_msg import AgentMsg, AgentMsgType import discord @@ -165,26 +165,26 @@ class DiscordTunnel(AgentTunnel): if len(resp_msg.body) < 1: return - knownledge_object = KnowledgeStore().parse_object_in_message(resp_msg.body) - if knownledge_object is not None: - if knownledge_object.get_object_type() == ObjectType.Image: - image = KnowledgeStore().bytes_from_object(knownledge_object) - try: - async with aiofiles.open("image.jpg", "wb") as f: - await f.write(image) - await message.channel.send(file=discord.File("image.jpg")) - except Exception as e: - logger.error(f"save image error:{e}") - logger.exception(e) - return - else: - pos = resp_msg.body.find("audio file") - if pos != -1: - audio_file = resp_msg.body[pos+11:].strip() - if audio_file.startswith("\""): - audio_file = audio_file[1:-1] - await message.channel.send(file=discord.File(audio_file)) - return + # knownledge_object = KnowledgeStore().parse_object_in_message(resp_msg.body) + # if knownledge_object is not None: + # if knownledge_object.get_object_type() == ObjectType.Image: + # image = KnowledgeStore().bytes_from_object(knownledge_object) + # try: + # async with aiofiles.open("image.jpg", "wb") as f: + # await f.write(image) + # await message.channel.send(file=discord.File("image.jpg")) + # except Exception as e: + # logger.error(f"save image error:{e}") + # logger.exception(e) + # return + # else: + # pos = resp_msg.body.find("audio file") + # if pos != -1: + # audio_file = resp_msg.body[pos+11:].strip() + # if audio_file.startswith("\""): + # audio_file = audio_file[1:-1] + # await message.channel.send(file=discord.File(audio_file)) + # return await message.channel.send(resp_msg.body) else: if resp_msg.is_image_msg(): diff --git a/src/component/slack_tunnel.py b/src/component/slack_tunnel.py index 52a5b63..1adc899 100644 --- a/src/component/slack_tunnel.py +++ b/src/component/slack_tunnel.py @@ -12,7 +12,7 @@ import aiohttp from slack_bolt.adapter.socket_mode.websockets import AsyncSocketModeHandler from slack_bolt.app.async_app import AsyncApp -from aios import KnowledgeStore, ObjectType +#from aios import KnowledgeStore, ObjectType from aios.frame.tunnel import AgentTunnel from aios.proto.agent_msg import AgentMsg, AgentMsgType from aios.storage.storage import AIStorage @@ -189,26 +189,26 @@ class SlackTunnel(AgentTunnel): if len(resp_msg.body) < 1: return - knownledge_object = KnowledgeStore().parse_object_in_message(resp_msg.body) - if knownledge_object is not None: - if knownledge_object.get_object_type() == ObjectType.Image: - image = KnowledgeStore().bytes_from_object(knownledge_object) - try: - async with aiofiles.open("image.jpg", "wb") as f: - await f.write(image) - await app.client.files_upload_v2(channel=event["channel"], file="image.jpg") - except Exception as e: - logger.error(f"save image error:{e}") - logger.exception(e) - return - else: - pos = resp_msg.body.find("audio file") - if pos != -1: - audio_file = resp_msg.body[pos+11:].strip() - if audio_file.startswith("\""): - audio_file = audio_file[1:-1] - await app.client.files_upload_v2(channel=event["channel"], file=audio_file) - return + # knownledge_object = KnowledgeStore().parse_object_in_message(resp_msg.body) + # if knownledge_object is not None: + # if knownledge_object.get_object_type() == ObjectType.Image: + # image = KnowledgeStore().bytes_from_object(knownledge_object) + # try: + # async with aiofiles.open("image.jpg", "wb") as f: + # await f.write(image) + # await app.client.files_upload_v2(channel=event["channel"], file="image.jpg") + # except Exception as e: + # logger.error(f"save image error:{e}") + # logger.exception(e) + # return + # else: + # pos = resp_msg.body.find("audio file") + # if pos != -1: + # audio_file = resp_msg.body[pos+11:].strip() + # if audio_file.startswith("\""): + # audio_file = audio_file[1:-1] + # await app.client.files_upload_v2(channel=event["channel"], file=audio_file) + # return await app.client.chat_postMessage(channel=event["channel"], text=resp_msg.body) else: if resp_msg.is_image_msg(): diff --git a/src/component/st_node/__init__.py b/src/component/st_node/__init__.py deleted file mode 100644 index a1fd2e0..0000000 --- a/src/component/st_node/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .local_st_compute_node import * \ No newline at end of file diff --git a/src/component/st_node/local_st_compute_node.py b/src/component/st_node/local_st_compute_node.py deleted file mode 100644 index 1887ae3..0000000 --- a/src/component/st_node/local_st_compute_node.py +++ /dev/null @@ -1,212 +0,0 @@ -import logging -import requests -from typing import Optional, List -from pydantic import BaseModel -from typing import Union -from PIL import Image -import io - -from aios import ComputeTask, ComputeTaskResult, ComputeTaskState, ComputeTaskType,ComputeTaskResultCode,ComputeNode,AIStorage,UserConfig,ObjectID,Queue_ComputeNode - -logger = logging.getLogger(__name__) - -class LocalSentenceTransformer_Text_ComputeNode(Queue_ComputeNode): - # For valid pretrained models, see https://www.sbert.net/docs/pretrained_models.html - def __init__(self, model_name: str = "all-MiniLM-L6-v2"): - super().__init__() - - self.node_id = "local_sentence_transformer_text_embedding_node" - self.model_name = model_name - self.model = None - - def initial(self) -> bool: - logger.info( - f"LocalSentenceTransformer_Text_ComputeNode init, model_name: {self.model_name}" - ) - - assert self.model_name is not None - assert self.model is None - try: - from sentence_transformers import SentenceTransformer - - self.model = SentenceTransformer(self.model_name) - except Exception as err: - logger.error(f"load model {self.model} failed: {err}") - return False - self.start() - return True - - async def execute_task(self, task: ComputeTask) : - result = ComputeTaskResult() - result.result_code = ComputeTaskResultCode.ERROR - result.set_from_task(task) - result.worker_id = self.node_id - try: - # logger.debug(f"LocalSentenceTransformer_Text_ComputeNode task: {task}") - if task.task_type == ComputeTaskType.TEXT_EMBEDDING: - input = task.params["input"] - logger.debug( - f"LocalSentenceTransformer_Text_ComputeNode task input: {input}" - ) - sentence_embeddings = self.model.encode(input, show_progress_bar=False).tolist() - # logger.debug(f"LocalSentenceTransformer_Text_ComputeNode task sentence_embeddings: {sentence_embeddings}") - result.result_code = ComputeTaskResultCode.OK - result.result["content"] = sentence_embeddings - - else: - result.error_str = f"unsupport embedding task type: {task.task_type}" - except Exception as err: - import traceback - - logger.error(f"{traceback.format_exc()}, error: {err}") - result.error_str = f"{traceback.format_exc()}, error: {err}" - - return result - - - def display(self) -> str: - return f"LocalSentenceTransformer_Text_ComputeNode: {self.node_id}, {self.model_name}" - - def get_capacity(self): - pass - - def is_support(self, task: ComputeTask) -> bool: - return task.task_type == ComputeTaskType.TEXT_EMBEDDING and task.params["model_name"] == "all-MiniLM-L6-v2" - - def is_local(self) -> bool: - return True - - -class LocalSentenceTransformer_Image_ComputeNode(Queue_ComputeNode): - # For valid pretrained models, see https://www.sbert.net/docs/pretrained_models.html - def __init__( - self, - model_name: str = "clip-ViT-B-32", - multi_model_name: str = "clip-ViT-B-32-multilingual-v1", - ): - super().__init__() - - self.node_id = "local_sentence_transformer_image_embedding_node" - self.model_name = model_name - self.multi_model_name = multi_model_name - self.model = None - self.multi_model = None - - def initial(self) -> bool: - logger.info( - f"LocalSentenceTransformer_Image_ComputeNode init, model_name: {self.model_name} {self.multi_model_name}" - ) - - assert self.model_name is not None - assert self.multi_model_name is not None - assert self.model is None - assert self.multi_model is None - - try: - from sentence_transformers import SentenceTransformer - - self.model = SentenceTransformer(self.model_name) - self.multi_model = SentenceTransformer(self.multi_model_name) - except Exception as err: - logger.error(f"load model {self.model} failed: {err}") - return False - self.start() - return True - - def _load_image(self, source: Union[ObjectID, bytes]) -> Optional[Image]: - image_data = None - if isinstance(source, ObjectID): - from aios import KnowledgeStore, ImageObject - - buf = KnowledgeStore().get_object_store().get_object(source) - if buf is None: - logger.error(f"load image object but not found! {source}") - return None - - try: - image_obj = ImageObject.decode(buf) - except Exception as err: - logger.error(f"decode ImageObject from buffer failed: {source}, {err}") - return None - - file_size = image_obj.get_file_size() - # print(f"got image object: {source.to_base58()}, size: {file_size}") - - image_data = ( - KnowledgeStore() - .get_chunk_reader() - .read_chunk_list_to_single_bytes(image_obj.get_chunk_list()) - ) - - elif isinstance(source, bytes): - image_data = source - else: - logger.error(f"unsupport image source type: {type(source)}, {source}") - return None - - try: - img = Image.open(io.BytesIO(image_data)) - - return img - except Exception as err: - logger.error(f"load image from buffer failed: {source}, {err}") - return None - - async def execute_task( - self, task: ComputeTask - ) -> ComputeTaskResult: - result = ComputeTaskResult() - result.result_code = ComputeTaskResultCode.ERROR - result.set_from_task(task) - result.worker_id = self.node_id - try: - # logger.debug(f"LocalSentenceTransformer_Text_ComputeNode task: {task}") - if task.task_type == ComputeTaskType.TEXT_EMBEDDING: - input = task.params["input"] - logger.debug( - f"LocalSentenceTransformer_Text_ComputeNode task text input: {input}" - ) - sentence_embeddings = self.multi_model.encode(input, show_progress_bar=False).tolist() - # logger.debug(f"LocalSentenceTransformer_Text_ComputeNode task sentence_embeddings: {sentence_embeddings}") - result.result_code = ComputeTaskResultCode.OK - result.result["content"] = sentence_embeddings - - elif task.task_type == ComputeTaskType.IMAGE_EMBEDDING: - input = task.params["input"] - logger.debug( - f"LocalSentenceTransformer_Image_ComputeNode task image input: {input}" - ) - - img = self._load_image(input) - if img is None: - result.error_str = f"load image failed: {input}" - return result - - sentence_embeddings = self.model.encode(img, show_progress_bar=False).tolist() - result.result_code = ComputeTaskResultCode.OK - result.result["content"] = sentence_embeddings - else: - result.error_str = f"unsupport embedding task type: {task.task_type}" - except Exception as err: - import traceback - - logger.error(f"{traceback.format_exc()}, error: {err}") - result.error_str = f"{traceback.format_exc()}, error: {err}" - - - return result - - def display(self) -> str: - return f"LocalSentenceTransformer_Image_ComputeNode: {self.node_id}, {self.model_name}" - - def get_capacity(self): - pass - - def is_support(self, task: ComputeTask) -> bool: - return ( - (task.task_type == ComputeTaskType.TEXT_EMBEDDING and task.params["model_name"] == "clip-ViT-B-32") - or task.task_type == ComputeTaskType.IMAGE_EMBEDDING - ) - - def is_local(self) -> bool: - return True diff --git a/src/component/tg_tunnel.py b/src/component/tg_tunnel.py index 094a98e..088fadc 100644 --- a/src/component/tg_tunnel.py +++ b/src/component/tg_tunnel.py @@ -12,7 +12,7 @@ from telegram import Bot from telegram.ext import Updater from telegram.error import Forbidden, NetworkError -from aios import ObjectType, KnowledgeStore,AgentTunnel,AIStorage,ContactManager,Contact,AgentMsg,AgentMsgType +from aios import AgentTunnel,AIStorage,ContactManager,Contact,AgentMsg,AgentMsgType logger = logging.getLogger(__name__) @@ -160,6 +160,68 @@ class TelegramTunnel(AgentTunnel): if not os.path.exists(path): os.makedirs(path) return path + + async def conver_agent_msg_to_tg_msg(self,resp_msg:AgentMsg,update: Update): + + if resp_msg.body_mime is None: + if resp_msg.body is None: + return + + if len(resp_msg.body) < 1: + await update.message.reply_text("") + return + + # knowledge_object = KnowledgeStore().parse_object_in_message(resp_msg.body) + # knowledge_object = None + # if knowledge_object is not None: + # if knowledge_object.get_object_type() == ObjectType.Image: + # image = KnowledgeStore().bytes_from_object(knowledge_object) + # try: + # async with aiofiles.open("tg_send_temp.png", mode='wb') as local_file: + # if local_file: + # await local_file.write(image) + # await update.message.reply_photo("tg_send_temp.png") + # except Exception as e: + # logger.error(f"save image error: {e}") + # return + # else: + # pos = resp_msg.body.find("audio file") + # if pos != -1: + # audio_file = resp_msg.body[pos+11:].strip() + # if audio_file.startswith("\""): + # audio_file = audio_file[1:-1] + # await update.message.reply_voice(audio_file) + # return + await update.message.reply_text(resp_msg.body) + else: + if resp_msg.is_image_msg(): + text, images = resp_msg.get_image_body() + if text is not None: + await update.message.reply_text(text) + for image in images: + if os.path.exists(image): + await update.message.reply_photo(image) + else: + await update.message.reply_text(image) + elif resp_msg.is_video_msg(): + text, video_file = resp_msg.get_video_body() + if text is not None: + await update.message.reply_text(text) + if os.path.exists(video_file): + await update.message.reply_video(video_file) + else: + await update.message.reply_text(video_file) + elif resp_msg.is_audio_msg(): + text, audio_file = resp_msg.get_audio_body() + if text is not None: + await update.message.reply_text(text) + + if os.path.exists(audio_file): + await update.message.reply_voice(audio_file) + else: + await update.message.reply_text(audio_file) + else: + await update.message.reply_text(resp_msg.body) async def conver_tg_msg_to_agent_msg(self,message:Message) -> AgentMsg: agent_msg = AgentMsg() @@ -246,6 +308,7 @@ class TelegramTunnel(AgentTunnel): return False + # main entry for telegram message async def on_message(self, bot:Bot, update: Update) -> None: message = update.message logger.info(f"on_message: {message.message_id} from {message.from_user.username} ({update.effective_user.username}) to {message.chat.title}({message.chat.id})") @@ -312,63 +375,8 @@ class TelegramTunnel(AgentTunnel): if resp_msg is None: await update.message.reply_text(f"System Error: Timeout,{self.target_id} no resopnse! Please check logs/aios.log for more details!") else: - if resp_msg.body_mime is None: - if resp_msg.body is None: - return + await self.conver_agent_msg_to_tg_msg(resp_msg,update) - if len(resp_msg.body) < 1: - await update.message.reply_text("") - return - - knowledge_object = KnowledgeStore().parse_object_in_message(resp_msg.body) - if knowledge_object is not None: - if knowledge_object.get_object_type() == ObjectType.Image: - image = KnowledgeStore().bytes_from_object(knowledge_object) - try: - async with aiofiles.open("tg_send_temp.png", mode='wb') as local_file: - if local_file: - await local_file.write(image) - await update.message.reply_photo("tg_send_temp.png") - except Exception as e: - logger.error(f"save image error: {e}") - return - else: - pos = resp_msg.body.find("audio file") - if pos != -1: - audio_file = resp_msg.body[pos+11:].strip() - if audio_file.startswith("\""): - audio_file = audio_file[1:-1] - await update.message.reply_voice(audio_file) - return - await update.message.reply_text(resp_msg.body) - else: - if resp_msg.is_image_msg(): - text, images = resp_msg.get_image_body() - if text is not None: - await update.message.reply_text(text) - for image in images: - if os.path.exists(image): - await update.message.reply_photo(image) - else: - await update.message.reply_text(image) - elif resp_msg.is_video_msg(): - text, video_file = resp_msg.get_video_body() - if text is not None: - await update.message.reply_text(text) - if os.path.exists(video_file): - await update.message.reply_video(video_file) - else: - await update.message.reply_text(video_file) - elif resp_msg.is_audio_msg(): - text, audio_file = resp_msg.get_audio_body() - if text is not None: - await update.message.reply_text(text) - - if os.path.exists(audio_file): - await update.message.reply_voice(audio_file) - else: - await update.message.reply_text(audio_file) - else: - await update.message.reply_text(resp_msg.body) + diff --git a/src/requirements.txt b/src/requirements.txt index 656343eea3b71d89948150211c7951578b6c45e1..73f511aa9bd6683cd37b721cc38e1272c7c1cf6f 100644 GIT binary patch literal 7112 zcmb`MNsm)k5QX15QhthC*lwDz$btxE5hW54tE^CaXxnsqzypo@$0s>oomZ}BH$g&* zEE{-Lw}w;Wef;zHZL?_B&7FQ9nrX9Ze%5Eu44W(cUN!yZ$L5E|Jo9F+&n7sa|D^9t zhrVc*;@n8iq8SBOUmPzkr0hhUH7jvGoOqDd)wGi&G?gSco=f@^9DCV~)uz7=PFuZd zj^Q2D>(GG|({|y_y1Ccs)yeOxmM?jZqK%uLc=2v3qz?Lf%0uI@l?OK4lN@`L+D7tr zEKT5i7pK9t@P)pU^r7%9PZE=)z7*n6eXgiu={*XqbGwY^NK$XZ=7lsLgx4;7*vQZH zq8}0m;m0qMwU#81hfV|?MGzT>A9Kmvi)$Uy;CdN2;E}nek^*96l`VcD8koEazmpF% zx_WLzb$rInT)MW+By^-jqPdeT7LAH!GY zEAhla$8;PN;)Z^*9VA}0KH@z(Z$la!oF>a$6XU=Kj;XwWch+V{Kk#z48ZRtbh8CVW zH%!UMavEJ5S$z!7>ozV|k_InGjTGUTc+7#dSZ&{-V-r${p7Ft6zUFx=o=sRxq^YY( z=t7dS?z+{rkS~@t5iK>sQ)o{V391al!6)r;1Sfh5wizj64~MZgbuG!xYH)!gec|jE z{cEq^;`phx?hukELIVvmY3j)mDq*W9^RNaxLC5G#B?o+{?uGJV7Cg?kth76w#~-5L ztaS`ugwOUW`zZ0w%t2E>cwCcr$_CIST2ufCbo(yVG-$KvF3=;NSCNhAPb`UBYTZe~ zCOm*T&~w*YKeEzXvyg(eg>38TU&Ih^+&@9TA^>V+9a90Bc!9qutEjyek)EVE-)%#B z{q++d?MOJ^k?-v4YDOkl9@AUTF!H?l{qtx|licRPaVpnYSt7Wy8F3R(I*-E19;J=cVtzxzGDC zQS9>0WoJKhI%6wy=F*KVJmHjmnafU3tfW=s*#ee|3AJa0;c>=UI+EN2b7Sb{g?XyW zCeOJCaMd(Oo-ZdK2Cc<(zPT`(>EpVkO2@K{{(flp5bH#qGYks5%1YHoH4rs=boNd7 z6A30OH3KKT-qm2+@DvLcI?YA{5|zi^hp4)Yxcg((*f~cpy1_E^x&L4Xk#S#xd!yW8 zwt6$_g+1iSL|^{I0&JtR*@Lg0B2&Ew&<}I#yYz{!?*6MWB0n&Uvm8@J!|d?k+CHVfS;vmQQ5=BV|Q)5fi+hgLVy`5-^Qgh;yf zu*25cic9Y5RP`|6)6-Rdq@HAQ=4kCNe@D+++2xm;NHBJZci*% zUY@t5oCn2~bZ*5*4pMi<9UC2W$CRyxJS9sKbK7D$rjCU^8FXG&`q>6|B$oQhlCvlQ;GK0LXcjP9fEkMulpG%E>>87Cr^@r0|p z;kjR1#yiQF$rjgMH`~eSMD-Z9j${uy>=7Ek1qtt(x6QB82Q{;)Y;=w;>`>suA{0lPRGEW{VgZ;@T3hnWy^cn%8jyejOs6IatG@LcFY3bWt~spi|2)1 zm!IwqQlCRNSYv6SRJ?RJHfXbll^Sz-EM|<-<{p3;Q1s( zy@^yl4KL($d6=5dw+8fDx^+{fAVRdA6+B_jb4=*R+3^x@M~2%wxkZ2KMZHn)-Xc?r z<84lGMiVr8byhc%i(2E?Hz&3S>Tf4I+;{2U<+ZC7`!CP3ZE(*%-#A~NJo9aY-tbXp znTT0i{n(%#ubM5kwKlw+{PfL~9>_a5=aczrn=8H#(v$NX+0}rErqGfeGW^ZRTtl@gG^)t6rSD$J-wjfK+KV`5B3sXNAFv8 zX)ez&a*Ay6n~TJZ{UCjU#s)EqxzBqtsz1H6b|luc-0Q7+_bo%cW3+8-5o~h&-_J07 z1VdL_UvjYNpc-LzfVL|i4E-*Lzn%B{MjEMiI#2A|Av!o#}YX!}Yt>rLRsUK22jl+)36*wf6R40ng85rHCs1$kVX`I_2v!}d-(n+!@Ty>RkW5YRYIm1OU3A0G=F*-OTm5vpt^`Zd}0WSL9 z*=+C*S{&hlV>lH>>KvQnL7D?WE1!@i3zjZz^24I>&iW-AKaE|dZP!Xg*=bC5FAn(* z@?P53YqE`*;EDa{#-jFH>|#DU;(0?MF_G&!YukRHZtl-Bq=WMj-OW5W z?|!sQ0H1*QLI`&u?{&jLJf+qSU_f&A$b+dGq-`X2dF(ghMH}PZa2%;)Ik*r{4yxSu zQlJGL1~1h;FR+bi9%9CoK5In9TH7BijxHO_n7UQ++zkLp`V5mzHYsvG!z6=EUF1s4 ztE_j;nC=OfO43FmB_&>_7k!>7c+TNt@+2FhEp1bGIFkvn+N_b0fG)0$5;qHiUc@ev zy{v^@gpmw&pew5#qAEAIh_Lg)KT(`*&iNN`{qN7eeyv^f^n0WbQ^_y~8{q@O+99TZ z-W&QaILjdwKM+&Um^$==XE0dZwmd&`H~C@-nl1^Sy#TP@P>fij)K9EGE!{0j5#5;!8P;wyAjOxsPoy!7AuTJ8hzkmHS_(bU^(N73>zHr45Cgp~% z%xZ0=pAfBUY9;W|pbRxN-pk&U{0_6D_cG!KJ60Ejz?Z6Mbif1K=23.2.1 +aiohttp>=3.8.5 +aioimaplib>=1.0.1 +aiosignal>=1.3.1 +aiosmtplib>=2.0.2 +anyio>=4.0.0 +async-timeout>=4.0.3 +attrs>=23.1.0 +backoff>=2.2.1 +base36>=0.1.1 +base58>=2.1.1 +beautifulsoup4>=4.12.2 +cachetools>=5.3.1 +certifi>=2023.7.22 +charset-normalizer>=3.2.0 +chroma-hnswlib>=0.7.1 +chromadb>=0.4.0 +click>=8.1.7 +colorama>=0.4.6 +coloredlogs>=15.0.1 +decorator>=4.4.2 +fastapi +filelock>=3.12.3 +flatbuffers>=23.5.26 +frozenlist>=1.4.0 +fsspec>=2023.9.0 +google>=3.0.0 +google-api-core>=2.11.1 +google-auth>=2.23.0 +google-cloud>=0.34.0 +google-cloud-texttospeech>=2.14.1 +googleapis-common-protos>=1.60.0 +h11>=0.14.0 +httpcore>=0.17.3 +httptools>=0.6.0 +httpx>=0.24.1 +huggingface-hub>=0.16.4 +humanfriendly>=10.0 +idna>=3.4 +imageio>=2.31.3 +imageio-ffmpeg>=0.4.8 +importlib-resources>=6.0.1 +mail-parser>=3.15.0 +monotonic>=1.6 +moviepy>=1.0.0 +mpmath>=1.3.0 +multidict>=6.0.4 +numpy>=1.25.2 +onnxruntime>=1.15.1 + +overrides>=7.4.0 +packaging>=23.1 +pandas>=2.1.0 +Pillow>=10.0.0 +posthog>=3.0.2 +proglog>=0.1.10 +prompt-toolkit>=3.0.39 +proto-plus>=1.22.3 +pulsar-client>=3.3.0 +pyasn1>=0.5.0 +pyasn1-modules>=0.3.0 +pydantic +PyPika>=0.48.9 +pyreadline3>=3.4.1 +python-dateutil>=2.8.2 +python-dotenv>=1.0.0 +python-telegram-bot>=20.5 +pytz>=2023.3.post1 +PyYAML>=6.0.1 +requests>=2.31.0 +rsa>=4.9 +simplejson>=3.19.1 +six>=1.16.0 +sniffio>=1.3.0 +soupsieve>=2.5 +starlette>=0.27.0 +sympy>=1.12 +tokenizers>=0.14.0 +toml>=0.10.0 +protobuf +grpcio +grpcio-status +h11==0.14.0 +httpcore==0.17.3 +httptools==0.6.0 +httpx==0.24.1 +huggingface-hub==0.16.4 +humanfriendly==10.0 +idna==3.4 +imageio==2.31.3 +imageio-ffmpeg==0.4.8 +importlib-resources==6.0.1 +mail-parser==3.15.0 +monotonic==1.6 +moviepy==1.0.0 +mpmath==1.3.0 +multidict==6.0.4 +numpy==1.25.2 +onnxruntime==1.15.1 +overrides==7.4.0 +packaging==23.1 +pandas==2.1.0 +Pillow==10.0.0 +posthog==3.0.2 +proglog==0.1.10 +prompt-toolkit==3.0.39 +proto-plus==1.22.3 +protobuf +pulsar-client==3.3.0 +pyasn1==0.5.0 +pyasn1-modules==0.3.0 +pydantic==1.10.12 +PyPika==0.48.9 +pyreadline3==3.4.1 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-telegram-bot==20.5 +pytz==2023.3.post1 +PyYAML==6.0.1 +requests==2.31.0 +rsa==4.9 +simplejson==3.19.1 +six==1.16.0 +sniffio==1.3.0 +soupsieve==2.5 +starlette==0.27.0 +sympy==1.12 +telegram==0.0.1 +tokenizers==0.14.0 +toml==0.10.0 +pysocks +chardet +pydub +aiosqlite +python-telegram-bot +pydub +stability_sdk +sentence-transformers==2.2.2 +tiktoken +markdown +PyPDF2 +srt +webvtt-py +openai +docker +generic_escape +duckduckgo-search +SQLAlchemy +mysqlclient +psycopg2-binary +pyodbc +oracledb +html2text +docx2txt +opencv-python +discord.py +slack_bolt +wget +moviepy diff --git a/src/service/aios_shell/aios_shell.py b/src/service/aios_shell/aios_shell.py index 79a77cc..92bf9ba 100644 --- a/src/service/aios_shell/aios_shell.py +++ b/src/service/aios_shell/aios_shell.py @@ -38,16 +38,16 @@ from google_node import * from llama_node import * from openai_node import * from sd_node import * -from st_node import * from agent_manager import AgentManager from workflow_manager import WorkflowManager -from knowledge_manager import KnowledgePipelineManager +#from knowledge_manager import KnowledgePipelineManager from tg_tunnel import TelegramTunnel from email_tunnel import EmailTunnel from discord_tunnel import DiscordTunnel from slack_tunnel import SlackTunnel -from common_environment import LocalKnowledgeBase, FilesystemEnvironment, ShellEnvironment, ScanLocalDocument, ParseLocalDocument +from common_environment import FilesystemEnvironment, ShellEnvironment +#from common_environment import ScanLocalDocument, ParseLocalDocument from compute_node_config import * @@ -214,17 +214,17 @@ class AIOS_Shell: - local_st_text_compute_node = LocalSentenceTransformer_Text_ComputeNode() - if local_st_text_compute_node.initial() is not True: - logger.error("local sentence transformer text embedding node initial failed!") - else: - ComputeKernel.get_instance().add_compute_node(local_st_text_compute_node) + # local_st_text_compute_node = LocalSentenceTransformer_Text_ComputeNode() + # if local_st_text_compute_node.initial() is not True: + # logger.error("local sentence transformer text embedding node initial failed!") + # else: + # ComputeKernel.get_instance().add_compute_node(local_st_text_compute_node) - local_st_image_compute_node = LocalSentenceTransformer_Image_ComputeNode() - if local_st_image_compute_node.initial() is not True: - logger.error("local sentence transformer image embedding node initial failed!") - else: - ComputeKernel.get_instance().add_compute_node(local_st_image_compute_node) + # local_st_image_compute_node = LocalSentenceTransformer_Image_ComputeNode() + # if local_st_image_compute_node.initial() is not True: + # logger.error("local sentence transformer image embedding node initial failed!") + # else: + # ComputeKernel.get_instance().add_compute_node(local_st_image_compute_node) await ComputeKernel.get_instance().start() @@ -233,12 +233,12 @@ class AIOS_Shell: #AIBus().get_default_bus().register_message_handler(self.username,self._user_process_msg) - pipelines = KnowledgePipelineManager.initial(os.path.join(AIStorage().get_instance().get_myai_dir(), "knowledge/pipelines")) - pipelines.register_input("scan_local", ScanLocalDocument) - pipelines.register_parser("parse_local", ParseLocalDocument) - pipelines.load_dir(os.path.join(AIStorage().get_instance().get_system_app_dir(), "knowledge_pipelines")) - pipelines.load_dir(os.path.join(AIStorage().get_instance().get_myai_dir(), "knowledge_pipelines")) - asyncio.create_task(pipelines.run()) + #pipelines = KnowledgePipelineManager.initial(os.path.join(AIStorage().get_instance().get_myai_dir(), "knowledge/pipelines")) + #pipelines.register_input("scan_local", ScanLocalDocument) + #pipelines.register_parser("parse_local", ParseLocalDocument) + #pipelines.load_dir(os.path.join(AIStorage().get_instance().get_system_app_dir(), "knowledge_pipelines")) + #pipelines.load_dir(os.path.join(AIStorage().get_instance().get_myai_dir(), "knowledge_pipelines")) + #asyncio.create_task(pipelines.run()) TelegramTunnel.register_to_loader() EmailTunnel.register_to_loader()