Add chunk relate impl and test case

src/knowledge/__init__.py

@@ -1,2 +1,3 @@
-from .object import EmailObject, ImageObject, TextChunkObject, ObjectID
+from .object import *
 from .vector import *
+from .data import *

src/knowledge/data/__init__.py

@@ -0,0 +1,2 @@
+from .chunk import ChunkID, PositionType, PositionFileRange
+from .tracker import ChunkTracker

src/knowledge/data/chunk.py

@@ -0,0 +1,43 @@
+from enum import Enum
+from ..object import ObjectID
+
+ChunkID = ObjectID
+
+class PositionType(Enum):
+    Unknown = 1
+    Device = 2
+    File = 3
+    FileRange = 4
+    ChunkManager = 5
+
+
+class PositionFileRange:
+    def __init__(self, path: str, range_begin: int, range_end: int):
+        self.path = path
+        self.range_begin = range_begin
+        self.range_end = range_end
+
+    def encode(self):
+        return f"{self.range_begin}:{self.range_end}:{self.path}"
+
+    @staticmethod
+    def decode(value: str):
+        parts = value.split(":")
+        if len(parts) < 3:
+            raise ValueError("Invalid input string")
+
+        try:
+            range_begin = int(parts[0])
+            range_end = int(parts[1])
+        except ValueError as e:
+            raise ValueError("Invalid range_begin or range_end string") from e
+
+        path = ":".join(parts[2:])
+        return PositionFileRange(path, range_begin, range_end)
+
+    def __str__(self):
+        return self.encode()
+
+    @staticmethod
+    def from_string(value: str):
+        return PositionFileRange.decode(value)

src/knowledge/data/tracker.py

@@ -0,0 +1,77 @@
+import sqlite3
+import time
+import logging
+import os
+from .chunk import  ChunkID, PositionType, PositionFileRange
+
+
+class ChunkTracker:
+    _instance = None
+    
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            directory = os.path.join(
+                os.path.dirname(__file__), "../../../rootfs/data/chunk/"
+            )
+            
+            if not os.path.exists(directory):
+                os.mkdir(directory)
+            file = os.path.join(directory, "chunk_tracker.db")
+            logging.info(f"will init chunk tracker, db={file}")
+            cls._instance.__singleton_init__(file)
+            
+        return cls._instance
+
+    def __singleton_init__(self, db_path: str):
+        logging.info(f"will init chunk tracker, db={db_path}")
+
+        self.conn = sqlite3.connect(db_path)
+        self.cursor = self.conn.cursor()
+        self.cursor.execute(
+            """
+            CREATE TABLE IF NOT EXISTS chunks (
+                id TEXT NOT NULL,
+                pos TEXT NOT NULL,
+                pos_type TINYINT NOT NULL,
+                insert_time UNSIGNED BIG INT NOT NULL,
+                update_time UNSIGNED BIG INT NOT NULL,
+                flags INTEGER DEFAULT 0,
+                PRIMARY KEY(id, pos, pos_type)
+            )
+        """
+        )
+        self.conn.commit()
+
+    def add_position(self, chunk_id: ChunkID, position: str, position_type: PositionType):
+        logging.debug(f"add chunk position: {chunk_id}, {position}, {position_type}")
+
+        insert_time = update_time = int(time.time())
+        self.cursor.execute(
+            """
+            INSERT OR REPLACE INTO chunks (id, pos, pos_type, insert_time, update_time)
+            VALUES (?, ?, ?, ?, ?)
+        """,
+            (str(chunk_id), position, position_type.value, insert_time, update_time,),
+        )
+        self.conn.commit()
+
+    def remove_position(self, chunk_id: ChunkID):
+        logging.info(f"remove chunk position: {chunk_id}")
+
+        self.cursor.execute(
+            """
+            DELETE FROM chunks WHERE id = ?
+        """,
+            (str(chunk_id),),
+        )
+        self.conn.commit()
+
+    def get_position(self, chunk_id: ChunkID) -> (str, PositionType):
+        self.cursor.execute(
+            """
+            SELECT pos, pos_type FROM chunks WHERE id = ?
+        """,
+            (str(chunk_id),),
+        )
+        return self.cursor.fetchone()

test/test_chunk.py

@@ -0,0 +1,36 @@
+import sys
+import os
+
+dir_path = os.path.dirname(os.path.realpath(__file__))
+print(dir_path)
+
+sys.path.append("{}/../src/".format(dir_path))
+print(sys.path)
+
+from knowledge import ChunkTracker, ChunkID, HashValue, PositionType
+
+
+import asyncio
+import unittest
+
+
+class TestChunk(unittest.TestCase):
+    def test_chunk_tracker(self):
+        tracker = ChunkTracker()
+        
+        hash = HashValue.hash_data("1234567890".encode("utf-8"));
+        cid = ChunkID.new_chunk_id(hash)
+        print(cid)
+            
+        tracker.add_position(cid, "/tmp/1", PositionType.File)
+        ret = tracker.get_position(cid)
+        print(ret[0])
+        
+        tracker.remove_position(cid)
+        ret = tracker.get_position(cid)
+        self.assertEqual(ret, None)
+    
+
+
+if __name__ == "__main__":
+    unittest.main()