From 076eeaf977d015ef57c095d13cef374f2ecbe149 Mon Sep 17 00:00:00 2001 From: Si Changjun Date: Thu, 24 Aug 2023 15:54:52 +0800 Subject: [PATCH 1/3] knowleadge.drawio --- doc/未命名绘图.drawio | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/未命名绘图.drawio diff --git a/doc/未命名绘图.drawio b/doc/未命名绘图.drawio new file mode 100644 index 0000000..f8db6cf --- /dev/null +++ b/doc/未命名绘图.drawio @@ -0,0 +1,10 @@ + + + + + + + + + + From 67677ba40fcd11300c3a86cdb47e1682e8f35d30 Mon Sep 17 00:00:00 2001 From: tsukasa Date: Tue, 29 Aug 2023 16:45:05 +0800 Subject: [PATCH 2/3] objected knowledge base design --- doc/mvp/knowleadge.drawio | 1478 +++++++++++++++++++++++++++++++++++++ doc/mvp/knowleage.md | 20 + doc/未命名绘图.drawio | 10 - 3 files changed, 1498 insertions(+), 10 deletions(-) create mode 100644 doc/mvp/knowleadge.drawio create mode 100644 doc/mvp/knowleage.md delete mode 100644 doc/未命名绘图.drawio diff --git a/doc/mvp/knowleadge.drawio b/doc/mvp/knowleadge.drawio new file mode 100644 index 0000000..bdb5534 --- /dev/null +++ b/doc/mvp/knowleadge.drawio @@ -0,0 +1,1478 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/mvp/knowleage.md b/doc/mvp/knowleage.md new file mode 100644 index 0000000..7b48cac --- /dev/null +++ b/doc/mvp/knowleage.md @@ -0,0 +1,20 @@ +# Objected knowleadge base, a specialized implemention for emails +## Vectorized Knowledge +Large language models are trained on general corpora and without fine-tuning on user-specific data, they struggle to utilize user-related context effectively. + +Users accumulate a vast amount of content that reflects their personality during their regular internet usage. This includes personal photos, tweets, Facebook posts, emails, etc. While it's possible to include all this content in the prompt during each interaction with the large language model, this approach is costly and can easily reach the token limit. + +A common solution is to generate feature vectors from this content using word embedding techniques and store them in a vector database. During an interaction, the vector that is most relevant to the prompt is retrieved from the database, merged with the prompt, and then passed to the large language model. + +We refer to this vectorized content as "knowledge". + +## Objected knownleadge base +In a personal AI system, to build a user's own knowledge base, we first need to implement various spider programs to crawl and retrieve all user-related data. Modern web content is typically rich text, including text, images, videos, hyperlinks, etc. Organizing this rich text in a tree-like structure similar to HTML is necessary, hence the need to introduce an object structure to represent this content. + +Different parts of this content cannot be vectorized using the same embedding model. For instance, text and images, as well as the content of an image and its EXIF information, need separate embeddings. This means that in the vector database, the same content may have multiple vector values, and a row can represent a whole content item or just a part of it. + +We need a comprehensive object structure to represent the hierarchy and relationships of content, as well as to implement the indexing and storage of objects. In the Minimum Viable Product (MVP) version, we'll implement a specialized solution for email content. In future versions, we can generalize this to handle other types of content, such as Facebook posts, tweets, etc. + +## Agent with knowleadge base +At the same time, we also need to explore the paradigm of using the knowledge base in Agents and workflows, so that the agent can better complete tasks in interaction with users through the context provided by the knowledge base. + diff --git a/doc/未命名绘图.drawio b/doc/未命名绘图.drawio deleted file mode 100644 index f8db6cf..0000000 --- a/doc/未命名绘图.drawio +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - From a7960861d12fd34411ce18af1b58d77e7630d826 Mon Sep 17 00:00:00 2001 From: tsukasa Date: Wed, 30 Aug 2023 17:16:37 +0800 Subject: [PATCH 3/3] objected knowledge base implement for email code structure --- doc/mvp/workflow.drawio | 6 +- doc/未命名绘图.drawio | 10 ---- src/aios_kernel/knowledge/__init__.py | 2 + src/aios_kernel/knowledge/knowledge_base.py | 10 ++++ src/aios_kernel/knowledge/object.py | 65 +++++++++++++++++++++ src/aios_kernel/knowledge/object_storage.py | 33 +++++++++++ src/aios_kernel/knowledge/vector_base.py | 14 +++++ src/service/email_spider/converter.py | 24 ++++++++ src/service/email_spider/main.py | 12 ++++ src/service/email_spider/spider.py | 17 ++++++ 10 files changed, 180 insertions(+), 13 deletions(-) delete mode 100644 doc/未命名绘图.drawio create mode 100644 src/aios_kernel/knowledge/__init__.py create mode 100644 src/aios_kernel/knowledge/knowledge_base.py create mode 100644 src/aios_kernel/knowledge/object.py create mode 100644 src/aios_kernel/knowledge/object_storage.py create mode 100644 src/aios_kernel/knowledge/vector_base.py create mode 100644 src/service/email_spider/converter.py create mode 100644 src/service/email_spider/main.py create mode 100644 src/service/email_spider/spider.py diff --git a/doc/mvp/workflow.drawio b/doc/mvp/workflow.drawio index e49374a..c6659c1 100644 --- a/doc/mvp/workflow.drawio +++ b/doc/mvp/workflow.drawio @@ -1,6 +1,6 @@ - + @@ -123,7 +123,7 @@ - + @@ -220,7 +220,7 @@ - + diff --git a/doc/未命名绘图.drawio b/doc/未命名绘图.drawio deleted file mode 100644 index f8db6cf..0000000 --- a/doc/未命名绘图.drawio +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - diff --git a/src/aios_kernel/knowledge/__init__.py b/src/aios_kernel/knowledge/__init__.py new file mode 100644 index 0000000..13bb147 --- /dev/null +++ b/src/aios_kernel/knowledge/__init__.py @@ -0,0 +1,2 @@ +from .object import EmailObject, ImageObject, TextChunkObject, ObjectID +from .knowledge_base import KnowledgeBase diff --git a/src/aios_kernel/knowledge/knowledge_base.py b/src/aios_kernel/knowledge/knowledge_base.py new file mode 100644 index 0000000..59e4f25 --- /dev/null +++ b/src/aios_kernel/knowledge/knowledge_base.py @@ -0,0 +1,10 @@ +# define a knowledge base class +from src.aios_kernel.agent import AgentPrompt +from .object import KnowledgeObject + +class KnowledgeBase: + async def insert(self, object: KnowledgeObject): + pass + + async def query(self, prompt: AgentPrompt) -> AgentPrompt: + pass \ No newline at end of file diff --git a/src/aios_kernel/knowledge/object.py b/src/aios_kernel/knowledge/object.py new file mode 100644 index 0000000..320cd8f --- /dev/null +++ b/src/aios_kernel/knowledge/object.py @@ -0,0 +1,65 @@ + +# define a object type enum +from abc import ABC + + +class ObjectType(Enum): + TextChunk = 1 + Image = 2 + Email = 101 + + +# define a object ID class to identify a object +class ObjectID: # pylint: disable=too-few-public-methods + def __init__(self, object_type, digist): + self.object_type = object_type + self.digist = digist + + def __str__(self): + return f"{self.object_type.name}:{self.digist}" + + +# define a object class +class KnowledgeObject(ABC): # pylint: disable=too-few-public-methods + def __init__(self, object_type): + self.object_type = object_type + + @abstractmethod + def get_id(self) -> ObjectID: + pass + + # define a to binary method to convert object to binary + @abstractmethod + def to_binary(self) -> bytes: + pass + + # define a from binary method to convert binary to object + @abstractmethod + def from_binary(self, binary: bytes): + pass + + +# define a text chunk class +class TextChunkObject(KnowledgeObject): # pylint: disable=too-few-public-methods + def __init__(self, text): + super().__init__(ObjectType.TextChunk) + self.text = text + + +# define a image class +class ImageObject(KnowledgeObject): # pylint: disable=too-few-public-methods + def __init__(self, meta, path): + super().__init__(ObjectType.Image) + self.meta = meta + self.path = path + + +# define a email class +class EmailObject(KnowledgeObject): # pylint: disable=too-few-public-methods + def __init__(self, meta): + super().__init__(ObjectType.Email) + self.meta = meta + self.text = [] + self.images = [] + + diff --git a/src/aios_kernel/knowledge/object_storage.py b/src/aios_kernel/knowledge/object_storage.py new file mode 100644 index 0000000..1a6f85e --- /dev/null +++ b/src/aios_kernel/knowledge/object_storage.py @@ -0,0 +1,33 @@ +# import RDB LargeBinary +from sqlalchemy import Column, String, LargeBinary, create_engine, sessionmaker, pickle +from .object import KnowledgeObject + +# implement object storage with RDB +# define object storage table +class ObjectStorageTable(Base): + __tablename__ = 'object_storage' + id = Column(String, primary_key=True) + parent = Column(String, nullable=True) + object = Column(LargeBinary, nullable=False) + + def __init__(self, id, parent, object): # pylint: disable=redefined-builtin + self.id = id + self.parent = parent + self.object = object + +# define object storage class +class ObjectStorage: + async def __init__(self, db_url): + self.engine = create_engine(db_url) + self.session = sessionmaker(bind=self.engine)() # pylint: disable=not-callable + + async def get(self, id) -> [KnowledgeObject, KnowledgeObject]: + obj = self.session.query(ObjectStorageTable).filter(ObjectStorageTable.id == id).first() + if obj is None: + return None + return pickle.loads(obj.object) + + # define insert method + async def insert(self, object, parent): # pylint: disable=redefined-builtin + obj = ObjectStorageTable(id, parent, pickle.dumps(object)) + \ No newline at end of file diff --git a/src/aios_kernel/knowledge/vector_base.py b/src/aios_kernel/knowledge/vector_base.py new file mode 100644 index 0000000..b2cf442 --- /dev/null +++ b/src/aios_kernel/knowledge/vector_base.py @@ -0,0 +1,14 @@ +# import the ObjectID class +from .object import ObjectID + +# define a vector base class +class VectorBase: + def __init__(self, db_url, model_name) -> None: + self.db_url = db_url + self.model_name = model_name + + async def insert(self, vector: [float], id: ObjectID): + pass + + async def query(self, vector: [float], top_k: int) -> [ObjectID]: + pass \ No newline at end of file diff --git a/src/service/email_spider/converter.py b/src/service/email_spider/converter.py new file mode 100644 index 0000000..193b220 --- /dev/null +++ b/src/service/email_spider/converter.py @@ -0,0 +1,24 @@ +from aios_kernel.knowledge import KnowledgeBase, EmailObject + +# define a email converter class + +class EmailConverter: + # define init method + def __init__(self, local_dir, knowledge_base: KnowledgeBase) -> None: + pass + + async def run(self): + # convert the email to knowledge object + for email_dir in self._next(): + # convert the email to knowledge object + knowledge_object = self._convert(email_dir) + # insert the knowledge object to knowledge base + await self.knowledge_base.insert(knowledge_object) + + def _next(self) -> str: + pass + + def _convert(self, email_dir) -> EmailObject: + pass + + \ No newline at end of file diff --git a/src/service/email_spider/main.py b/src/service/email_spider/main.py new file mode 100644 index 0000000..cb75abd --- /dev/null +++ b/src/service/email_spider/main.py @@ -0,0 +1,12 @@ +import asyncio +from .spider import EmailSpider, EmailConverter + + +if __name__ == "__main__": + spider = EmailSpider("smtp.163.com","user","pwd","./email") + asyncio.run(spider.run()) + + converter = EmailConverter("./email",KnowledgeBase()) + asyncio.run(converter.run()) + + diff --git a/src/service/email_spider/spider.py b/src/service/email_spider/spider.py new file mode 100644 index 0000000..0b5a212 --- /dev/null +++ b/src/service/email_spider/spider.py @@ -0,0 +1,17 @@ +# define a email spider class + +class EmailSpider: + def __init__(self, address, account, pwd, local_dir) -> None: + pass + + async def run(self): + # spide the email from the email server + for email_link in self._next(): + # save the email to local directory + self._save(email_link) + + def _next(self): + pass + + def _save(self, email_link) -> str: + pass \ No newline at end of file