shell knowledge commands

This commit is contained in:
tsukasa
2023-09-21 18:32:17 +08:00
parent edbd1c41da
commit a0a45b8998
11 changed files with 409 additions and 74 deletions
+2 -2
View File
@@ -49,7 +49,7 @@ class DocumentObjectBuilder:
self.text = text
return self
def build(self, relation_store: ObjectRelationStore) -> DocumentObject:
def build(self) -> DocumentObject:
chunk_list = KnowledgeStore().get_chunk_list_writer().create_chunk_list_from_text(
self.text,
1024 * 4,
@@ -60,6 +60,6 @@ class DocumentObjectBuilder:
# Add relation to store
for chunk_id in chunk_list.chunk_list:
relation_store.add_relation(chunk_id, doc_id)
KnowledgeStore().get_relation_store().add_relation(chunk_id, doc_id)
return doc
+1 -1
View File
@@ -94,7 +94,7 @@ class EmailObjectBuilder:
with open(content_file, "r", encoding="utf-8") as f:
text = f.read()
document = DocumentObjectBuilder({}, {}, text).build(relation_store=relation)
document = DocumentObjectBuilder({}, {}, text).build()
document_id = document.calculate_id()
store.put_object(document_id, document.encode())
documents = {"email.txt": document_id}
+1 -1
View File
@@ -52,7 +52,7 @@ def get_exif_data(image_path: str):
return {
TAGS.get(key): exif_data[key]
for key in exif_data.keys()
if key in TAGS and isinstance(exif_data[key], (bytes, str))
if key in TAGS and isinstance(exif_data[key], str)
}
else:
return {}
+1 -1
View File
@@ -46,7 +46,7 @@ class ChunkListWriter:
)
file_hash = HashValue(hash_obj.digest())
print(f"calc file hash: {file_path}, {file_hash}")
# print(f"calc file hash: {file_path}, {file_hash}")
return ChunkList(chunk_list, file_hash)
+6 -10
View File
@@ -4,7 +4,7 @@ from .object import ObjectStore, ObjectRelationStore
from .data import ChunkStore, ChunkTracker, ChunkListWriter, ChunkReader
from .vector import ChromaVectorStore, VectorBase
import logging
import aios_kernel
# KnowledgeStore class, which aggregates ChunkStore, ChunkTracker, and ObjectStore, and is a global singleton that makes it easy to use these three built-in store examples
class KnowledgeStore:
@@ -13,16 +13,12 @@ class KnowledgeStore:
def __new__(cls):
if cls._instance is None:
cls._instance = super().__new__(cls)
directory = os.path.join(
os.path.dirname(__file__), "../../rootfs/data/"
)
directory = os.path.normpath(directory)
print(directory)
knowledge_dir = aios_kernel.storage.AIStorage().get_myai_dir() / "knowledge"
if not os.path.exists(directory):
os.makedirs(directory)
if not os.path.exists(knowledge_dir):
os.makedirs(knowledge_dir)
cls._instance.__singleton_init__(directory)
cls._instance.__singleton_init__(knowledge_dir)
return cls._instance
@@ -64,5 +60,5 @@ class KnowledgeStore:
def get_vector_store(self, model_name: str) -> VectorBase:
if model_name not in self.vector_store:
self.vector_store[model_name] = ChromaVectorStore(model_name)
self.vector_store[model_name] = ChromaVectorStore(self.root, model_name)
return self.vector_store[model_name]
+2 -4
View File
@@ -6,16 +6,14 @@ import os
class ChromaVectorStore(VectorBase):
def __init__(self, model_name: str) -> None:
def __init__(self, root_dir, model_name: str) -> None:
super().__init__(model_name)
logging.info(
"will init chroma vector store, model={}".format(model_name)
)
directory = os.path.join(
os.path.dirname(__file__), "../../../rootfs/data/vector"
)
directory = os.path.join(root_dir, "vector")
logging.info("will use vector store: {}".format(directory))
client = chromadb.PersistentClient(