test prompt from knowledge object

src/knowledge/core_object/document_object.py

@@ -53,7 +53,7 @@ class DocumentObjectBuilder:
         chunk_list = KnowledgeStore().get_chunk_list_writer().create_chunk_list_from_text(
             self.text,
             1024 * 4,
-            "."
+            ".?!\n"
         )
         doc = DocumentObject(self.meta, self.tags, chunk_list)
         doc_id = doc.calculate_id()

src/knowledge/data/writer.py

@@ -74,19 +74,20 @@ class ChunkListWriter:
         text: str, chunk_max_words: int, separator_chars: str = ".,"
     ) -> List[str]:
         sentences = re.split(f"[{separator_chars}]", text)
-        chunk_list = []
-        chunk = []
-        word_count = 0
-        for sentence in sentences:
-            words = sentence.split()
-            for word in words:
-                if word_count < chunk_max_words:
-                    chunk.append(word)
-                    word_count += 1
-                else:
-                    chunk_list.append(" ".join(chunk))
-                    chunk = [word]
-                    word_count = 1
-        if chunk:
-            chunk_list.append(" ".join(chunk))
-        return chunk_list
+        # chunk_list = []
+        # chunk = []
+        # word_count = 0
+        # for sentence in sentences:
+        #     words = sentence.split()
+        #     for word in words:
+        #         if word_count < chunk_max_words:
+        #             chunk.append(word)
+        #             word_count += 1
+        #         else:
+        #             chunk_list.append(" ".join(chunk))
+        #             chunk = [word]
+        #             word_count = 1
+        # if chunk:
+        #     chunk_list.append(" ".join(chunk))
+        # return chunk_list
+        return sentences

src/knowledge/object/object.py

@@ -54,7 +54,8 @@ class KnowledgeObject(ABC):
         )
         sha256 = hashlib.sha256()
         sha256.update(data.encode())
-        return ObjectID(sha256.digest())
+        hash_bytes = sha256.digest()
+        return ObjectID(bytes([self.object_type]) + hash_bytes[1:])
 
     def encode(self) -> bytes:
         return pickle.dumps(self)

src/knowledge/object/object_id.py

@@ -44,7 +44,11 @@ class ObjectID:  # pylint: disable=too-few-public-methods
 
     @staticmethod
     def new_chunk_id(chunk_hash: HashValue):
-        return ObjectID(chunk_hash.value)
+        assert len(chunk_hash.value) == 32, "ObjectID must be 32 bytes long"
+        return ObjectID(bytes([ObjectType.Chunk]) + chunk_hash.value[1:])
+    
+    def get_object_type(self) -> ObjectType:
+        return ObjectType(self.value[0])
     
     @staticmethod
     def hash_data(data: bytes):

src/knowledge/vector/chroma_store.py

@@ -30,9 +30,10 @@ class ChromaVectorStore(VectorBase):
         self.collection = collection
 
     async def insert(self, vector: [float], id: ObjectID):
+        logging.info(f"will insert vector: {vector} id: {str(id)}")
         self.collection.add(
             embeddings=vector,
-            ids=id,
+            ids=str(id),
         )
 
     async def query(self, vector: [float], top_k: int) -> [ObjectID]:
@@ -40,8 +41,10 @@ class ChromaVectorStore(VectorBase):
             query_embeddings=vector,
             n_results=top_k,
         )
-
-        return ret["ids"]
+        logging.info(f"query result {ret}")
+        if len(ret['ids']) == 0:
+            return []
+        return list(map(ObjectID.from_base58, ret["ids"][0]))
 
     async def delete(self, id: ObjectID):
         self.collection.delete(