add knowledge pipeline manager

src/aios_kernel/__init__.py

@@ -5,8 +5,6 @@ from .agent import AIAgent,AIAgentTemplete
 from .compute_kernel import ComputeKernel,ComputeTask,ComputeTaskResult,ComputeTaskState,ComputeTaskType
 from .compute_node import ComputeNode,LocalComputeNode
 from .open_ai_node import OpenAI_ComputeNode
-from .knowledge_base import KnowledgeBase, KnowledgeEnvironment
-from .knowledge_pipeline import KnowledgeEmailSource, KnowledgeDirSource, KnowledgePipline
 from .role import AIRole,AIRoleGroup
 from .workflow import Workflow
 from .bus import AIBus

src/aios_kernel/agent.py

@@ -16,7 +16,6 @@ from .compute_task import ComputeTaskResult,ComputeTaskResultCode
 from .ai_function import AIFunction
 from .environment import Environment
 from .contact_manager import ContactManager,Contact,FamilyMember
-from .knowledge_base import KnowledgeBase
 from .compute_kernel import ComputeKernel
 from .bus import AIBus
 
@@ -550,8 +549,7 @@ class AIAgent:
     def parser_learn_llm_result(self,llm_result:str):
         pass
 
-    async def _llm_read_article(self,kb:KnowledgeBase,item:KnowledgeObject) -> ComputeTaskResult:
+    async def _llm_read_article(self,item:KnowledgeObject) -> ComputeTaskResult:
-        #kb_env = KnowledgeBaseFileSystemEnvironment()
         full_content = item.get_article_full_content()
         full_content_len = ComputeKernel.llm_num_tokens_from_text(full_content,self.get_llm_model_name())
         if full_content_len < self.get_llm_learn_token_limit():

src/aios_kernel/tg_tunnel.py

@@ -10,9 +10,7 @@ from telegram import Bot
 from telegram.ext import Updater
 from telegram.error import Forbidden, NetworkError
 
-from knowledge.object.object_id import ObjectType
+from knowledge import ObjectType, KnowledgeStore
-
-from .knowledge_base import KnowledgeBase
 
 from .tunnel import AgentTunnel
 from .storage import AIStorage
@@ -236,10 +234,10 @@ class TelegramTunnel(AgentTunnel):
                     await update.message.reply_text("")
                     return
                 
-                knowledge_object = KnowledgeBase().parse_object_in_message(resp_msg.body)
+                knowledge_object = KnowledgeStore().parse_object_in_message(resp_msg.body)
                 if knowledge_object is not None:
                     if knowledge_object.get_object_type() == ObjectType.Image:
-                        image = KnowledgeBase().bytes_from_object(knowledge_object)
+                        image = KnowledgeStore().bytes_from_object(knowledge_object)
                         try:
                             async with aiofiles.open("tg_send_temp.png", mode='wb') as local_file:
                                 if local_file:

src/knowledge/core_object/document_object.py

@@ -1,7 +1,6 @@
 from ..object import KnowledgeObject, ObjectRelationStore
 from ..data import ChunkList, ChunkListWriter
 from ..object import ObjectType
-from .. import KnowledgeStore
 
 # desc
 #   meta
@@ -49,13 +48,13 @@ class DocumentObjectBuilder:
         self.text = text
         return self
 
-    def build(self) -> DocumentObject:
+    def build(self, store) -> DocumentObject:
-        chunk_list = KnowledgeStore().get_chunk_list_writer().create_chunk_list_from_text(self.text)
+        chunk_list = store.get_chunk_list_writer().create_chunk_list_from_text(self.text)
         doc = DocumentObject(self.meta, self.tags, chunk_list)
         doc_id = doc.calculate_id()
         
         # Add relation to store
         for chunk_id in chunk_list.chunk_list:
-            KnowledgeStore().get_relation_store().add_relation(chunk_id, doc_id)
+            store.get_relation_store().add_relation(chunk_id, doc_id)
             
         return doc

src/knowledge/core_object/email_object.py

@@ -1,4 +1,3 @@
-from .. import KnowledgeStore
 from .rich_text_object import RichTextObject, RichTextObjectBuilder
 from ..object import ObjectID, ObjectType, KnowledgeObject
 from .document_object import DocumentObjectBuilder
@@ -68,11 +67,11 @@ class EmailObjectBuilder:
         self.folder = folder
         return self
 
-    def build(self) -> EmailObject:
+    def build(self, store) -> EmailObject:
         
         # Just get the object store and relation store from global KnowledgeStore
-        store = KnowledgeStore().get_object_store()
+        store = store.get_object_store()
-        relation = KnowledgeStore().get_relation_store()
+        relation = store.get_relation_store()
         
         # Read meta.json
         meta = {}

src/knowledge/core_object/image_object.py

@@ -1,7 +1,6 @@
 from ..object import KnowledgeObject
 from ..data import ChunkList, ChunkListWriter
 from ..object import ObjectType
-from .. import KnowledgeStore
 import os
 
 # desc
@@ -86,10 +85,10 @@ class ImageObjectBuilder:
         self.restore_file = restore_file
         return self
 
-    def build(self) -> ImageObject:
+    def build(self, store) -> ImageObject:
         
         file_size = os.path.getsize(self.image_file)
-        chunk_list = KnowledgeStore().get_chunk_list_writer().create_chunk_list_from_file(
+        chunk_list = store.get_chunk_list_writer().create_chunk_list_from_file(
             self.image_file, 1024 * 1024 * 4, self.restore_file
         )
         exif = get_exif_data(self.image_file)

src/knowledge/core_object/video_object.py

@@ -1,7 +1,6 @@
 from ..object import KnowledgeObject
 from ..data import ChunkList, ChunkListWriter
 from ..object import ObjectType
-from .. import KnowledgeStore
 
 # desc
 #   meta
@@ -76,8 +75,8 @@ class VideoObjectBuilder:
         self.restore_file = restore_file
         return self
 
-    def build(self) -> VideoObject:
+    def build(self, store) -> VideoObject:
-        chunk_list = KnowledgeStore().get_chunk_list_writer().create_chunk_list_from_file(
+        chunk_list = store.get_chunk_list_writer().create_chunk_list_from_file(
             self.video_file, 1024 * 1024 * 4, self.restore_file
         )
         info = get_video_info(self.video_file)

src/knowledge/pipeline.py

@@ -7,6 +7,7 @@ import sqlite3
 import os
 from . import ObjectID, KnowledgeStore
 import asyncio
+from enum import Enum
 
 class KnowledgePipelineJournal:
     def __init__(self, time: datetime.datetime, object_id: str, input: str, parser: str):

src/knowledge/store.py

@@ -1,7 +1,7 @@
 import os
 
-from .object import ObjectStore, ObjectRelationStore, ObjectID, ObjectType, KnowledgeObject, DocumentObject, ImageObject, VideoObject, RichTextObject, EmailObject
+from .object import ObjectStore, ObjectRelationStore, ObjectID, ObjectType, KnowledgeObject
-from core_object import DocumentObject, ImageObject, VideoObject, RichTextObject, EmailObject
+from .core_object import DocumentObject, ImageObject, VideoObject, RichTextObject, EmailObject
 from .data import ChunkStore, ChunkTracker, ChunkListWriter, ChunkReader
 import json
 import logging