embedding email object

This commit is contained in:
tsukasa
2023-09-12 15:28:59 +08:00
parent ddc103af38
commit 8ac6065332
10 changed files with 155 additions and 42 deletions
+1
View File
@@ -1,4 +1,5 @@
from .document_object import DocumentObject, DocumentObjectBuilder
from .image_object import ImageObject, ImageObjectBuilder
from .video_object import VideoObject, VideoObjectBuilder
from .rich_text_object import RichTextObject, RichTextObjectBuilder
from .email_object import EmailObject, EmailObjectBuilder
+1 -1
View File
@@ -18,7 +18,7 @@ class ImageObject(KnowledgeObject):
body = dict()
desc["meta"] = meta
desc["exif"] = exif
desc
desc["tags"] = tags
desc["hash"] = chunk_list.hash.to_base58()
body["chunk_list"] = chunk_list.chunk_list
-1
View File
@@ -8,7 +8,6 @@ import base36
class ObjectType(Enum):
Chunk = 7
TextChunk = 100
Image = 101
Video = 102
Document = 103
+7
View File
@@ -1,6 +1,7 @@
import os
from .object import ObjectStore
from .data import ChunkStore, ChunkTracker, ChunkListWriter, ChunkReader
from .vector import ChromaVectorStore, VectorBase
import logging
@@ -37,6 +38,7 @@ class KnowledgeStore:
self.chunk_tracker = ChunkTracker(chunk_store_dir)
self.chunk_list_writer = ChunkListWriter(self.chunk_store, self.chunk_tracker)
self.chunk_reader = ChunkReader(self.chunk_store, self.chunk_tracker)
self.vector_store = {}
def get_object_store(self) -> ObjectStore:
@@ -53,3 +55,8 @@ class KnowledgeStore:
def get_chunk_reader(self) -> ChunkReader:
return self.chunk_reader
def get_vector_store(self, model_name: str) -> VectorBase:
if model_name not in self.vector_store:
self.vector_store[model_name] = ChromaVectorStore(model_name)
return self.vector_store[model_name]
+3 -3
View File
@@ -6,11 +6,11 @@ import os
class ChromaVectorStore(VectorBase):
def __init__(self, db_url, model_name: str) -> None:
super().__init__(db_url, model_name)
def __init__(self, model_name: str) -> None:
super().__init__(model_name)
logging.info(
"will init chroma vector store, db={}, model={}".format(db_url, model_name)
"will init chroma vector store, model={}".format(model_name)
)
directory = os.path.join(
+1 -2
View File
@@ -3,8 +3,7 @@ from ..object import ObjectID
# define a vector base class
class VectorBase:
def __init__(self, db_url, model_name) -> None:
self.db_url = db_url
def __init__(self, model_name) -> None:
self.model_name = model_name
async def insert(self, vector: [float], id: ObjectID):