objected knowledge base implement for email code structure

doc/mvp/workflow.drawio

@@ -1,6 +1,6 @@
 <mxfile host="65bd71144e" pages="3">
     <diagram id="C5RBs43oDa-KdzZeNtuy" name="Page-1">
-        <mxGraphModel dx="1800" dy="877" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
+        <mxGraphModel dx="500" dy="864" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
             <root>
                 <mxCell id="WIyWlLk6GJQsqaUBKTNV-0"/>
                 <mxCell id="WIyWlLk6GJQsqaUBKTNV-1" parent="WIyWlLk6GJQsqaUBKTNV-0"/>
@@ -123,7 +123,7 @@
         </mxGraphModel>
     </diagram>
     <diagram id="kWxfmPxtNxOAf0a73TCG" name="Page-2">
-        <mxGraphModel dx="951" dy="944" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
+        <mxGraphModel dx="500" dy="864" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
             <root>
                 <mxCell id="0"/>
                 <mxCell id="1" parent="0"/>
@@ -220,7 +220,7 @@
         </mxGraphModel>
     </diagram>
     <diagram id="7NYJTgo0U9cdVshLy85U" name="Page-3">
-        <mxGraphModel dx="1800" dy="877" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
+        <mxGraphModel dx="500" dy="864" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
             <root>
                 <mxCell id="0"/>
                 <mxCell id="1" parent="0"/>

doc/未命名绘图.drawio

@@ -1,10 +0,0 @@
-<mxfile host="app.diagrams.net" modified="2023-08-24T07:54:22.026Z" agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0" version="21.6.9" etag="NslzN5rQopvVPPp-3Tpy" type="github">
-  <diagram name="第 1 页" id="QQhyyV3wXBRqc4yhfBeY">
-    <mxGraphModel>
-      <root>
-        <mxCell id="0" />
-        <mxCell id="1" parent="0" />
-      </root>
-    </mxGraphModel>
-  </diagram>
-</mxfile>

src/aios_kernel/knowledge/__init__.py

@@ -0,0 +1,2 @@
+from .object import EmailObject, ImageObject, TextChunkObject, ObjectID
+from .knowledge_base import KnowledgeBase

src/aios_kernel/knowledge/knowledge_base.py

@@ -0,0 +1,10 @@
+# define a knowledge base class
+from src.aios_kernel.agent import AgentPrompt
+from .object import KnowledgeObject
+
+class KnowledgeBase:
+    async def insert(self, object: KnowledgeObject):
+        pass
+
+    async def query(self, prompt: AgentPrompt) -> AgentPrompt:
+        pass

src/aios_kernel/knowledge/object.py

@@ -0,0 +1,65 @@
+
+# define a object type enum
+from abc import ABC
+
+
+class ObjectType(Enum):
+    TextChunk = 1
+    Image = 2
+    Email = 101
+
+
+# define a object ID class to identify a object
+class ObjectID: # pylint: disable=too-few-public-methods
+    def __init__(self, object_type, digist):
+        self.object_type = object_type
+        self.digist = digist
+
+    def __str__(self):
+        return f"{self.object_type.name}:{self.digist}"
+    
+
+# define a object class
+class KnowledgeObject(ABC): # pylint: disable=too-few-public-methods
+    def __init__(self, object_type):
+        self.object_type = object_type
+
+    @abstractmethod
+    def get_id(self) -> ObjectID:
+        pass
+
+    # define a to binary method to convert object to binary
+    @abstractmethod
+    def to_binary(self) -> bytes:
+        pass
+
+    # define a from binary method to convert binary to object
+    @abstractmethod
+    def from_binary(self, binary: bytes):
+        pass
+
+
+# define a text chunk class
+class TextChunkObject(KnowledgeObject): # pylint: disable=too-few-public-methods
+    def __init__(self, text):
+        super().__init__(ObjectType.TextChunk)
+        self.text = text
+
+
+# define a image class
+class ImageObject(KnowledgeObject): # pylint: disable=too-few-public-methods
+    def __init__(self, meta, path):
+        super().__init__(ObjectType.Image)
+        self.meta = meta
+        self.path = path
+
+
+# define a email class
+class EmailObject(KnowledgeObject): # pylint: disable=too-few-public-methods
+    def __init__(self, meta):
+        super().__init__(ObjectType.Email)
+        self.meta = meta
+        self.text = []
+        self.images = []
+
+    

src/aios_kernel/knowledge/object_storage.py

@@ -0,0 +1,33 @@
+# import RDB LargeBinary
+from sqlalchemy import Column, String, LargeBinary, create_engine, sessionmaker, pickle
+from .object import KnowledgeObject
+
+# implement object storage with RDB
+# define object storage table
+class ObjectStorageTable(Base):
+    __tablename__ = 'object_storage'
+    id = Column(String, primary_key=True)
+    parent = Column(String, nullable=True)
+    object = Column(LargeBinary, nullable=False)
+
+    def __init__(self, id, parent, object): # pylint: disable=redefined-builtin
+        self.id = id
+        self.parent = parent
+        self.object = object
+
+# define object storage class
+class ObjectStorage:
+    async def __init__(self, db_url):
+        self.engine = create_engine(db_url)
+        self.session = sessionmaker(bind=self.engine)() # pylint: disable=not-callable
+
+    async def get(self, id) -> [KnowledgeObject, KnowledgeObject]:
+        obj = self.session.query(ObjectStorageTable).filter(ObjectStorageTable.id == id).first()
+        if obj is None:
+            return None
+        return pickle.loads(obj.object)
+    
+    # define insert method
+    async def insert(self, object, parent): # pylint: disable=redefined-builtin
+        obj = ObjectStorageTable(id, parent, pickle.dumps(object))
+        

src/aios_kernel/knowledge/vector_base.py

@@ -0,0 +1,14 @@
+# import the ObjectID class
+from .object import ObjectID
+
+# define a vector base class
+class VectorBase:
+    def __init__(self, db_url, model_name) -> None:
+        self.db_url = db_url
+        self.model_name = model_name
+
+    async def insert(self, vector: [float], id: ObjectID):
+        pass
+
+    async def query(self, vector: [float], top_k: int) -> [ObjectID]:
+        pass

src/service/email_spider/converter.py

@@ -0,0 +1,24 @@
+from aios_kernel.knowledge import KnowledgeBase, EmailObject
+
+# define a email converter class
+
+class EmailConverter:
+    # define init method
+    def __init__(self, local_dir, knowledge_base: KnowledgeBase) -> None:
+        pass
+
+    async def run(self):
+        # convert the email to knowledge object
+        for email_dir in self._next():
+            # convert the email to knowledge object
+            knowledge_object = self._convert(email_dir)
+            # insert the knowledge object to knowledge base
+            await self.knowledge_base.insert(knowledge_object)
+
+    def _next(self) -> str:
+        pass
+
+    def _convert(self, email_dir) -> EmailObject:
+        pass
+
+    

src/service/email_spider/main.py

@@ -0,0 +1,12 @@
+import asyncio
+from .spider import EmailSpider, EmailConverter
+
+
+if __name__ == "__main__":
+    spider = EmailSpider("smtp.163.com","user","pwd","./email")
+    asyncio.run(spider.run())
+
+    converter = EmailConverter("./email",KnowledgeBase())
+    asyncio.run(converter.run())
+
+    

src/service/email_spider/spider.py

@@ -0,0 +1,17 @@
+# define a email spider class
+
+class EmailSpider:
+    def __init__(self, address, account, pwd, local_dir) -> None:
+        pass
+
+    async def run(self): 
+        # spide the email from the email server
+        for email_link in self._next():
+            # save the email to local directory
+            self._save(email_link)
+
+    def _next(self):
+        pass
+
+    def _save(self, email_link) -> str:
+        pass