Add object and chunk base impl and test cases

src/knowledge/data/__init__.py

@@ -1,4 +1,5 @@
 from .chunk import ChunkID, PositionType, PositionFileRange
 from .tracker import ChunkTracker
 from .chunk_store import ChunkStore
-from .gen import ChunkListGenerator
+from .writer import ChunkListWriter
+from .chunk_list import ChunkList

src/knowledge/data/chunk_list.py

@@ -0,0 +1,14 @@
+from ..object import HashValue
+from .chunk import ChunkID
+from typing import List
+
+class ChunkList:
+    def __init__(self, chunk_list: List[ChunkID], hash: HashValue):
+        self.chunk_list = chunk_list
+        self.hash = hash
+
+    def __str__(self):
+        return self.hash.to_base58()
+
+    def __repr__(self):
+        return f"chunk_list: {self.chunk_list}, hash: {self.hash}"

src/knowledge/data/writer.py

@@ -6,16 +6,16 @@ from .chunk_store import ChunkStore
 from .chunk import ChunkID, PositionFileRange, PositionType
 from ..object import HashValue
 from .tracker import ChunkTracker
+from .chunk_list import ChunkList
 
-
-class ChunkListGenerator:
+class ChunkListWriter:
     def __init__(self, chunk_store: ChunkStore, chunk_tracker: ChunkTracker):
         self.chunk_store = chunk_store
         self.chunk_tracker = chunk_tracker
 
     def create_chunk_list_from_file(
         self, file_path: str, chunk_size: int, restore: bool
-    ) -> Tuple[List[ChunkID], HashValue]:
+    ) -> ChunkList:
         assert (
             chunk_size % (1024 * 1024) == 0
         ), "chunk size should be an integral multiple of 1MB"
@@ -47,7 +47,8 @@ class ChunkListGenerator:
 
         file_hash = HashValue(hash_obj.digest())
         print(f"calc file hash: {file_path}, {file_hash}")
-        return chunk_list, file_hash
+        
+        return ChunkList(chunk_list, file_hash)
 
     def create_chunk_list_from_text(
         self, text: str, chunk_max_words: int, separator_chars: str = ".,"
@@ -55,20 +56,18 @@ class ChunkListGenerator:
         text_list = self._split_text_list(text, chunk_max_words, separator_chars)
         chunk_list = []
         hash_obj = hashlib.sha256()
-        
+
         for text in text_list:
-            chunk_bytes  = text.encode("utf-8")
+            chunk_bytes = text.encode("utf-8")
             hash_obj.update(chunk_bytes)
-            
+
             chunk_id = ChunkID.hash_data(chunk_bytes)
             chunk_list.append(chunk_id)
-            self.chunk_tracker.add_position(
-                chunk_id, "", PositionType.ChunkStore
-            )
+            self.chunk_tracker.add_position(chunk_id, "", PositionType.ChunkStore)
             self.chunk_store.put_chunk(chunk_id, chunk_bytes)
-        
+
         hash = HashValue(hash_obj.digest())
-        return chunk_list, hash
+        return ChunkList(chunk_list, hash)
 
     @staticmethod
     def _split_text_list(
@@ -90,5 +89,4 @@ class ChunkListGenerator:
                     word_count = 1
         if chunk:
             chunk_list.append(" ".join(chunk))
-        return chunk_list
-
+        return chunk_list