add knowledge pipeline manager

src/knowledge/core_object/document_object.py

@@ -1,7 +1,6 @@
 from ..object import KnowledgeObject, ObjectRelationStore
 from ..data import ChunkList, ChunkListWriter
 from ..object import ObjectType
-from .. import KnowledgeStore
 
 # desc
 #   meta
@@ -49,13 +48,13 @@ class DocumentObjectBuilder:
         self.text = text
         return self
 
-    def build(self) -> DocumentObject:
-        chunk_list = KnowledgeStore().get_chunk_list_writer().create_chunk_list_from_text(self.text)
+    def build(self, store) -> DocumentObject:
+        chunk_list = store.get_chunk_list_writer().create_chunk_list_from_text(self.text)
         doc = DocumentObject(self.meta, self.tags, chunk_list)
         doc_id = doc.calculate_id()
         
         # Add relation to store
         for chunk_id in chunk_list.chunk_list:
-            KnowledgeStore().get_relation_store().add_relation(chunk_id, doc_id)
+            store.get_relation_store().add_relation(chunk_id, doc_id)
             
         return doc

src/knowledge/core_object/email_object.py

@@ -1,4 +1,3 @@
-from .. import KnowledgeStore
 from .rich_text_object import RichTextObject, RichTextObjectBuilder
 from ..object import ObjectID, ObjectType, KnowledgeObject
 from .document_object import DocumentObjectBuilder
@@ -68,11 +67,11 @@ class EmailObjectBuilder:
         self.folder = folder
         return self
 
-    def build(self) -> EmailObject:
+    def build(self, store) -> EmailObject:
         
         # Just get the object store and relation store from global KnowledgeStore
-        store = KnowledgeStore().get_object_store()
-        relation = KnowledgeStore().get_relation_store()
+        store = store.get_object_store()
+        relation = store.get_relation_store()
         
         # Read meta.json
         meta = {}

src/knowledge/core_object/image_object.py

@@ -1,7 +1,6 @@
 from ..object import KnowledgeObject
 from ..data import ChunkList, ChunkListWriter
 from ..object import ObjectType
-from .. import KnowledgeStore
 import os
 
 # desc
@@ -86,10 +85,10 @@ class ImageObjectBuilder:
         self.restore_file = restore_file
         return self
 
-    def build(self) -> ImageObject:
+    def build(self, store) -> ImageObject:
         
         file_size = os.path.getsize(self.image_file)
-        chunk_list = KnowledgeStore().get_chunk_list_writer().create_chunk_list_from_file(
+        chunk_list = store.get_chunk_list_writer().create_chunk_list_from_file(
             self.image_file, 1024 * 1024 * 4, self.restore_file
         )
         exif = get_exif_data(self.image_file)

src/knowledge/core_object/video_object.py

@@ -1,7 +1,6 @@
 from ..object import KnowledgeObject
 from ..data import ChunkList, ChunkListWriter
 from ..object import ObjectType
-from .. import KnowledgeStore
 
 # desc
 #   meta
@@ -76,8 +75,8 @@ class VideoObjectBuilder:
         self.restore_file = restore_file
         return self
 
-    def build(self) -> VideoObject:
-        chunk_list = KnowledgeStore().get_chunk_list_writer().create_chunk_list_from_file(
+    def build(self, store) -> VideoObject:
+        chunk_list = store.get_chunk_list_writer().create_chunk_list_from_file(
             self.video_file, 1024 * 1024 * 4, self.restore_file
         )
         info = get_video_info(self.video_file)

src/knowledge/pipeline.py

@@ -7,6 +7,7 @@ import sqlite3
 import os
 from . import ObjectID, KnowledgeStore
 import asyncio
+from enum import Enum
 
 class KnowledgePipelineJournal:
     def __init__(self, time: datetime.datetime, object_id: str, input: str, parser: str):

src/knowledge/store.py

@@ -1,7 +1,7 @@
 import os
 
-from .object import ObjectStore, ObjectRelationStore, ObjectID, ObjectType, KnowledgeObject, DocumentObject, ImageObject, VideoObject, RichTextObject, EmailObject
-from core_object import DocumentObject, ImageObject, VideoObject, RichTextObject, EmailObject
+from .object import ObjectStore, ObjectRelationStore, ObjectID, ObjectType, KnowledgeObject
+from .core_object import DocumentObject, ImageObject, VideoObject, RichTextObject, EmailObject
 from .data import ChunkStore, ChunkTracker, ChunkListWriter, ChunkReader
 import json
 import logging