From f80ec04272248ca25f04d8951bac76d2ca349897 Mon Sep 17 00:00:00 2001 From: tsukasa Date: Thu, 19 Oct 2023 10:47:45 +0800 Subject: [PATCH] add knowledge pipeline manager --- src/aios_kernel/__init__.py | 2 -- src/aios_kernel/agent.py | 4 +--- src/aios_kernel/tg_tunnel.py | 8 +++----- src/knowledge/core_object/document_object.py | 7 +++---- src/knowledge/core_object/email_object.py | 7 +++---- src/knowledge/core_object/image_object.py | 5 ++--- src/knowledge/core_object/video_object.py | 5 ++--- src/knowledge/pipeline.py | 1 + src/knowledge/store.py | 4 ++-- 9 files changed, 17 insertions(+), 26 deletions(-) diff --git a/src/aios_kernel/__init__.py b/src/aios_kernel/__init__.py index e19ba35..fc11ad0 100644 --- a/src/aios_kernel/__init__.py +++ b/src/aios_kernel/__init__.py @@ -5,8 +5,6 @@ from .agent import AIAgent,AIAgentTemplete from .compute_kernel import ComputeKernel,ComputeTask,ComputeTaskResult,ComputeTaskState,ComputeTaskType from .compute_node import ComputeNode,LocalComputeNode from .open_ai_node import OpenAI_ComputeNode -from .knowledge_base import KnowledgeBase, KnowledgeEnvironment -from .knowledge_pipeline import KnowledgeEmailSource, KnowledgeDirSource, KnowledgePipline from .role import AIRole,AIRoleGroup from .workflow import Workflow from .bus import AIBus diff --git a/src/aios_kernel/agent.py b/src/aios_kernel/agent.py index 5d9c756..bacafe5 100644 --- a/src/aios_kernel/agent.py +++ b/src/aios_kernel/agent.py @@ -16,7 +16,6 @@ from .compute_task import ComputeTaskResult,ComputeTaskResultCode from .ai_function import AIFunction from .environment import Environment from .contact_manager import ContactManager,Contact,FamilyMember -from .knowledge_base import KnowledgeBase from .compute_kernel import ComputeKernel from .bus import AIBus @@ -550,8 +549,7 @@ class AIAgent: def parser_learn_llm_result(self,llm_result:str): pass - async def _llm_read_article(self,kb:KnowledgeBase,item:KnowledgeObject) -> ComputeTaskResult: - #kb_env = KnowledgeBaseFileSystemEnvironment() + async def _llm_read_article(self,item:KnowledgeObject) -> ComputeTaskResult: full_content = item.get_article_full_content() full_content_len = ComputeKernel.llm_num_tokens_from_text(full_content,self.get_llm_model_name()) if full_content_len < self.get_llm_learn_token_limit(): diff --git a/src/aios_kernel/tg_tunnel.py b/src/aios_kernel/tg_tunnel.py index dcc7abd..923fb14 100644 --- a/src/aios_kernel/tg_tunnel.py +++ b/src/aios_kernel/tg_tunnel.py @@ -10,9 +10,7 @@ from telegram import Bot from telegram.ext import Updater from telegram.error import Forbidden, NetworkError -from knowledge.object.object_id import ObjectType - -from .knowledge_base import KnowledgeBase +from knowledge import ObjectType, KnowledgeStore from .tunnel import AgentTunnel from .storage import AIStorage @@ -236,10 +234,10 @@ class TelegramTunnel(AgentTunnel): await update.message.reply_text("") return - knowledge_object = KnowledgeBase().parse_object_in_message(resp_msg.body) + knowledge_object = KnowledgeStore().parse_object_in_message(resp_msg.body) if knowledge_object is not None: if knowledge_object.get_object_type() == ObjectType.Image: - image = KnowledgeBase().bytes_from_object(knowledge_object) + image = KnowledgeStore().bytes_from_object(knowledge_object) try: async with aiofiles.open("tg_send_temp.png", mode='wb') as local_file: if local_file: diff --git a/src/knowledge/core_object/document_object.py b/src/knowledge/core_object/document_object.py index b69f59f..43e8b9e 100644 --- a/src/knowledge/core_object/document_object.py +++ b/src/knowledge/core_object/document_object.py @@ -1,7 +1,6 @@ from ..object import KnowledgeObject, ObjectRelationStore from ..data import ChunkList, ChunkListWriter from ..object import ObjectType -from .. import KnowledgeStore # desc # meta @@ -49,13 +48,13 @@ class DocumentObjectBuilder: self.text = text return self - def build(self) -> DocumentObject: - chunk_list = KnowledgeStore().get_chunk_list_writer().create_chunk_list_from_text(self.text) + def build(self, store) -> DocumentObject: + chunk_list = store.get_chunk_list_writer().create_chunk_list_from_text(self.text) doc = DocumentObject(self.meta, self.tags, chunk_list) doc_id = doc.calculate_id() # Add relation to store for chunk_id in chunk_list.chunk_list: - KnowledgeStore().get_relation_store().add_relation(chunk_id, doc_id) + store.get_relation_store().add_relation(chunk_id, doc_id) return doc diff --git a/src/knowledge/core_object/email_object.py b/src/knowledge/core_object/email_object.py index 108c9cc..64d78b3 100644 --- a/src/knowledge/core_object/email_object.py +++ b/src/knowledge/core_object/email_object.py @@ -1,4 +1,3 @@ -from .. import KnowledgeStore from .rich_text_object import RichTextObject, RichTextObjectBuilder from ..object import ObjectID, ObjectType, KnowledgeObject from .document_object import DocumentObjectBuilder @@ -68,11 +67,11 @@ class EmailObjectBuilder: self.folder = folder return self - def build(self) -> EmailObject: + def build(self, store) -> EmailObject: # Just get the object store and relation store from global KnowledgeStore - store = KnowledgeStore().get_object_store() - relation = KnowledgeStore().get_relation_store() + store = store.get_object_store() + relation = store.get_relation_store() # Read meta.json meta = {} diff --git a/src/knowledge/core_object/image_object.py b/src/knowledge/core_object/image_object.py index 340bcc2..1033bf7 100644 --- a/src/knowledge/core_object/image_object.py +++ b/src/knowledge/core_object/image_object.py @@ -1,7 +1,6 @@ from ..object import KnowledgeObject from ..data import ChunkList, ChunkListWriter from ..object import ObjectType -from .. import KnowledgeStore import os # desc @@ -86,10 +85,10 @@ class ImageObjectBuilder: self.restore_file = restore_file return self - def build(self) -> ImageObject: + def build(self, store) -> ImageObject: file_size = os.path.getsize(self.image_file) - chunk_list = KnowledgeStore().get_chunk_list_writer().create_chunk_list_from_file( + chunk_list = store.get_chunk_list_writer().create_chunk_list_from_file( self.image_file, 1024 * 1024 * 4, self.restore_file ) exif = get_exif_data(self.image_file) diff --git a/src/knowledge/core_object/video_object.py b/src/knowledge/core_object/video_object.py index 56e9829..1ac9338 100644 --- a/src/knowledge/core_object/video_object.py +++ b/src/knowledge/core_object/video_object.py @@ -1,7 +1,6 @@ from ..object import KnowledgeObject from ..data import ChunkList, ChunkListWriter from ..object import ObjectType -from .. import KnowledgeStore # desc # meta @@ -76,8 +75,8 @@ class VideoObjectBuilder: self.restore_file = restore_file return self - def build(self) -> VideoObject: - chunk_list = KnowledgeStore().get_chunk_list_writer().create_chunk_list_from_file( + def build(self, store) -> VideoObject: + chunk_list = store.get_chunk_list_writer().create_chunk_list_from_file( self.video_file, 1024 * 1024 * 4, self.restore_file ) info = get_video_info(self.video_file) diff --git a/src/knowledge/pipeline.py b/src/knowledge/pipeline.py index 9c7e349..e051925 100644 --- a/src/knowledge/pipeline.py +++ b/src/knowledge/pipeline.py @@ -7,6 +7,7 @@ import sqlite3 import os from . import ObjectID, KnowledgeStore import asyncio +from enum import Enum class KnowledgePipelineJournal: def __init__(self, time: datetime.datetime, object_id: str, input: str, parser: str): diff --git a/src/knowledge/store.py b/src/knowledge/store.py index ebfda27..803c3b6 100644 --- a/src/knowledge/store.py +++ b/src/knowledge/store.py @@ -1,7 +1,7 @@ import os -from .object import ObjectStore, ObjectRelationStore, ObjectID, ObjectType, KnowledgeObject, DocumentObject, ImageObject, VideoObject, RichTextObject, EmailObject -from core_object import DocumentObject, ImageObject, VideoObject, RichTextObject, EmailObject +from .object import ObjectStore, ObjectRelationStore, ObjectID, ObjectType, KnowledgeObject +from .core_object import DocumentObject, ImageObject, VideoObject, RichTextObject, EmailObject from .data import ChunkStore, ChunkTracker, ChunkListWriter, ChunkReader import json import logging