diff --git a/doc/mvp/workflow.drawio b/doc/mvp/workflow.drawio index 2b3e239..c6659c1 100644 --- a/doc/mvp/workflow.drawio +++ b/doc/mvp/workflow.drawio @@ -1,6 +1,6 @@ - + @@ -123,7 +123,7 @@ - + @@ -220,7 +220,7 @@ - + diff --git a/doc/未命名绘图.drawio b/doc/未命名绘图.drawio deleted file mode 100644 index f8db6cf..0000000 --- a/doc/未命名绘图.drawio +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - diff --git a/src/aios_kernel/knowledge/__init__.py b/src/aios_kernel/knowledge/__init__.py new file mode 100644 index 0000000..13bb147 --- /dev/null +++ b/src/aios_kernel/knowledge/__init__.py @@ -0,0 +1,2 @@ +from .object import EmailObject, ImageObject, TextChunkObject, ObjectID +from .knowledge_base import KnowledgeBase diff --git a/src/aios_kernel/knowledge/knowledge_base.py b/src/aios_kernel/knowledge/knowledge_base.py new file mode 100644 index 0000000..59e4f25 --- /dev/null +++ b/src/aios_kernel/knowledge/knowledge_base.py @@ -0,0 +1,10 @@ +# define a knowledge base class +from src.aios_kernel.agent import AgentPrompt +from .object import KnowledgeObject + +class KnowledgeBase: + async def insert(self, object: KnowledgeObject): + pass + + async def query(self, prompt: AgentPrompt) -> AgentPrompt: + pass \ No newline at end of file diff --git a/src/aios_kernel/knowledge/object.py b/src/aios_kernel/knowledge/object.py new file mode 100644 index 0000000..320cd8f --- /dev/null +++ b/src/aios_kernel/knowledge/object.py @@ -0,0 +1,65 @@ + +# define a object type enum +from abc import ABC + + +class ObjectType(Enum): + TextChunk = 1 + Image = 2 + Email = 101 + + +# define a object ID class to identify a object +class ObjectID: # pylint: disable=too-few-public-methods + def __init__(self, object_type, digist): + self.object_type = object_type + self.digist = digist + + def __str__(self): + return f"{self.object_type.name}:{self.digist}" + + +# define a object class +class KnowledgeObject(ABC): # pylint: disable=too-few-public-methods + def __init__(self, object_type): + self.object_type = object_type + + @abstractmethod + def get_id(self) -> ObjectID: + pass + + # define a to binary method to convert object to binary + @abstractmethod + def to_binary(self) -> bytes: + pass + + # define a from binary method to convert binary to object + @abstractmethod + def from_binary(self, binary: bytes): + pass + + +# define a text chunk class +class TextChunkObject(KnowledgeObject): # pylint: disable=too-few-public-methods + def __init__(self, text): + super().__init__(ObjectType.TextChunk) + self.text = text + + +# define a image class +class ImageObject(KnowledgeObject): # pylint: disable=too-few-public-methods + def __init__(self, meta, path): + super().__init__(ObjectType.Image) + self.meta = meta + self.path = path + + +# define a email class +class EmailObject(KnowledgeObject): # pylint: disable=too-few-public-methods + def __init__(self, meta): + super().__init__(ObjectType.Email) + self.meta = meta + self.text = [] + self.images = [] + + diff --git a/src/aios_kernel/knowledge/object_storage.py b/src/aios_kernel/knowledge/object_storage.py new file mode 100644 index 0000000..1a6f85e --- /dev/null +++ b/src/aios_kernel/knowledge/object_storage.py @@ -0,0 +1,33 @@ +# import RDB LargeBinary +from sqlalchemy import Column, String, LargeBinary, create_engine, sessionmaker, pickle +from .object import KnowledgeObject + +# implement object storage with RDB +# define object storage table +class ObjectStorageTable(Base): + __tablename__ = 'object_storage' + id = Column(String, primary_key=True) + parent = Column(String, nullable=True) + object = Column(LargeBinary, nullable=False) + + def __init__(self, id, parent, object): # pylint: disable=redefined-builtin + self.id = id + self.parent = parent + self.object = object + +# define object storage class +class ObjectStorage: + async def __init__(self, db_url): + self.engine = create_engine(db_url) + self.session = sessionmaker(bind=self.engine)() # pylint: disable=not-callable + + async def get(self, id) -> [KnowledgeObject, KnowledgeObject]: + obj = self.session.query(ObjectStorageTable).filter(ObjectStorageTable.id == id).first() + if obj is None: + return None + return pickle.loads(obj.object) + + # define insert method + async def insert(self, object, parent): # pylint: disable=redefined-builtin + obj = ObjectStorageTable(id, parent, pickle.dumps(object)) + \ No newline at end of file diff --git a/src/aios_kernel/knowledge/vector_base.py b/src/aios_kernel/knowledge/vector_base.py new file mode 100644 index 0000000..b2cf442 --- /dev/null +++ b/src/aios_kernel/knowledge/vector_base.py @@ -0,0 +1,14 @@ +# import the ObjectID class +from .object import ObjectID + +# define a vector base class +class VectorBase: + def __init__(self, db_url, model_name) -> None: + self.db_url = db_url + self.model_name = model_name + + async def insert(self, vector: [float], id: ObjectID): + pass + + async def query(self, vector: [float], top_k: int) -> [ObjectID]: + pass \ No newline at end of file diff --git a/src/service/email_spider/converter.py b/src/service/email_spider/converter.py new file mode 100644 index 0000000..193b220 --- /dev/null +++ b/src/service/email_spider/converter.py @@ -0,0 +1,24 @@ +from aios_kernel.knowledge import KnowledgeBase, EmailObject + +# define a email converter class + +class EmailConverter: + # define init method + def __init__(self, local_dir, knowledge_base: KnowledgeBase) -> None: + pass + + async def run(self): + # convert the email to knowledge object + for email_dir in self._next(): + # convert the email to knowledge object + knowledge_object = self._convert(email_dir) + # insert the knowledge object to knowledge base + await self.knowledge_base.insert(knowledge_object) + + def _next(self) -> str: + pass + + def _convert(self, email_dir) -> EmailObject: + pass + + \ No newline at end of file diff --git a/src/service/email_spider/main.py b/src/service/email_spider/main.py new file mode 100644 index 0000000..cb75abd --- /dev/null +++ b/src/service/email_spider/main.py @@ -0,0 +1,12 @@ +import asyncio +from .spider import EmailSpider, EmailConverter + + +if __name__ == "__main__": + spider = EmailSpider("smtp.163.com","user","pwd","./email") + asyncio.run(spider.run()) + + converter = EmailConverter("./email",KnowledgeBase()) + asyncio.run(converter.run()) + + diff --git a/src/service/email_spider/spider.py b/src/service/email_spider/spider.py new file mode 100644 index 0000000..0b5a212 --- /dev/null +++ b/src/service/email_spider/spider.py @@ -0,0 +1,17 @@ +# define a email spider class + +class EmailSpider: + def __init__(self, address, account, pwd, local_dir) -> None: + pass + + async def run(self): + # spide the email from the email server + for email_link in self._next(): + # save the email to local directory + self._save(email_link) + + def _next(self): + pass + + def _save(self, email_link) -> str: + pass \ No newline at end of file