Add object and chunk base impl and test cases

This commit is contained in:
liyaxing
2023-09-11 00:46:56 +08:00
parent e8e3812d28
commit 36476eb5b4
16 changed files with 385 additions and 125 deletions
+20 -10
View File
@@ -15,12 +15,19 @@ root = logging.getLogger()
root.setLevel(logging.DEBUG)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
handler.setFormatter(formatter)
root.addHandler(handler)
from knowledge import ChunkTracker, ChunkID, HashValue, PositionType, KnowledgeStore, ChunkListGenerator
from knowledge import (
ChunkTracker,
ChunkID,
HashValue,
PositionType,
KnowledgeStore,
ChunkListWriter,
)
import asyncio
import unittest
@@ -28,29 +35,32 @@ import unittest
class TestChunk(unittest.TestCase):
def test_chunk_tracker(self):
tracker = KnowledgeStore().get_chunk_tracker()
hash = HashValue.hash_data("1234567890".encode("utf-8"));
hash = HashValue.hash_data("1234567890".encode("utf-8"))
cid = ChunkID.new_chunk_id(hash)
print(cid)
tracker.add_position(cid, "/tmp/1", PositionType.File)
ret = tracker.get_position(cid)
print(ret[0])
tracker.remove_position(cid)
ret = tracker.get_position(cid)
self.assertEqual(ret, None)
def test_chunk(self):
gen = ChunkListGenerator(KnowledgeStore().get_chunk_store(), KnowledgeStore().get_chunk_tracker())
gen = ChunkListWriter(
KnowledgeStore().get_chunk_store(), KnowledgeStore().get_chunk_tracker()
)
gen.create_chunk_list_from_file("H:/test", 1024 * 1024, True)
# Read the file
text_file = "H:/test.txt"
with open(text_file, 'r', encoding = "utf-8") as file:
with open(text_file, "r", encoding="utf-8") as file:
text = file.read()
gen.create_chunk_list_from_text(text, 1024)
if __name__ == "__main__":
unittest.main()