Add object and chunk base impl and test cases

src/knowledge/object/__init__.py

@@ -0,0 +1,5 @@
+from .object import EmailObject, ImageObject, TextChunkObject, ObjectID
+from .blob import FileBlobStorage
+from .hash import HashValue, hash_data
+from .object_store import ObjectStore
+from .object_id import ObjectID, ObjectType

src/knowledge/object/blob.py

@@ -0,0 +1,54 @@
+import os
+import shutil
+from .object import ObjectID
+
+
+class FileBlobStorage:
+    def __init__(self, root):
+        self.root = root
+
+    def get_full_path(self, object_id: ObjectID, auto_create: bool = True):
+        if os.name == "nt":  # Windows
+            hash_str = object_id.to_base36()
+            len = 3
+        else:
+            hash_str = str(object_id)
+            len = 2
+
+        tmp, first = hash_str[:-len], hash_str[-len:]
+        second = tmp[-len:]
+
+        if os.name == "nt":  # Windows
+            if second in ["con", "aux", "nul", "prn"]:
+                second = tmp[-(len + 1) :]
+            if first in ["con", "aux", "nul", "prn"]:
+                first = f"{first}_"
+
+        path = os.path.join(self.root, first, second)
+        if auto_create and not os.path.exists(path):
+            os.makedirs(path)
+
+        path = os.path.join(path, hash_str)
+
+        return path
+
+    def write_sync(self, path: str, contents: bytes):
+        with open(path, "wb") as f:
+            f.write(contents)
+
+    def put(self, object_id: ObjectID, contents: bytes):
+        full_path = self.get_full_path(object_id)
+        self.write_sync(full_path, contents)
+
+    def get(self, object_id: ObjectID) -> bytes:
+        full_path = self.get_full_path(object_id)
+        with open(full_path, "rb") as f:
+            return f.read()
+
+    def delete(self, object_id: ObjectID):
+        full_path = self.get_full_path(object_id)
+        os.remove(full_path)
+
+    def exists(self, object_id: ObjectID) -> bool:
+        full_path = self.get_full_path(object_id)
+        return os.path.exists(full_path)

src/knowledge/object/hash.py

@@ -0,0 +1,42 @@
+import hashlib
+import base58
+import base36
+
+class HashValue:
+    def __init__(self, value: bytes):
+        assert len(value) == 32, "HashValue must be 32 bytes long"
+        self.value = value
+
+    def __str__(self) -> str:
+        return self.to_base58()
+        
+    @staticmethod
+    def hash_data(data):
+        return hash_data(data)
+
+    def to_base58(self):
+        return base58.b58encode(self.value).decode()
+
+    @staticmethod
+    def from_base58(s):
+        return HashValue(base58.b58decode(s))
+
+    def to_base36(self):
+        # Convert the bytes to int before encoding
+        num = int.from_bytes(self.value, 'big')
+        return base36.dumps(num)
+
+    @staticmethod
+    def from_base36(s):
+        # Decode to int and then convert to bytes
+        num = base36.loads(s)
+        return HashValue(num.to_bytes((num.bit_length() + 7) // 8, 'big'))
+    
+    
+HASH_VALUE_LEN = 32
+
+
+def hash_data(data: bytes):
+    sha256 = hashlib.sha256()
+    sha256.update(data)
+    return HashValue(sha256.digest())

src/knowledge/object/object.py

@@ -0,0 +1,79 @@
+
+# define a object type enum
+from abc import ABC, abstractmethod
+from enum import Enum
+from .object_id import ObjectID
+import hashlib
+import json
+import pickle
+
+class KnowledgeObject(ABC):
+    def __init__(self, object_type: int, desc: dict = {}, body: dict = {}):
+        self.desc = desc
+        self.body = body
+        self.object_type = object_type
+
+    def get_object_type(self):
+        return self.object_type
+
+    def object_id(self) -> ObjectID:
+        return self.calculate_id()
+    
+    def set_desc_with_key_value(self, key, value):
+        self.desc[key] = value
+
+    def get_desc_with_key(self, key):
+        return self.desc.get(key)
+
+    def get_desc(self) -> dict:
+        return self.desc
+    
+    def set_body_with_key_value(self, key, value):
+        self.body[key] = value
+
+    def get_body_with_key(self, key):
+        return self.body.get(key)
+
+    def get_body(self) -> dict:
+        return self.body
+    
+    def calculate_id(self):
+        # Convert the object_type and desc to string and compute the SHA256 hash
+        data = json.dumps({"object_type": self.object_type, "desc": self.desc})
+        sha256 = hashlib.sha256()
+        sha256.update(data.encode())
+        return ObjectID(sha256.digest())
+
+    def encode(self) -> bytes:
+        return pickle.dumps(self)
+
+    @staticmethod
+    def decode(data: bytes):
+        return pickle.loads(data)
+    
+
+        
+# define a text chunk class
+class TextChunkObject(KnowledgeObject): # pylint: disable=too-few-public-methods
+    def __init__(self, text: str):
+        super().__init__(ObjectType.TextChunk)
+        self.text = text
+
+
+# define a image class
+class ImageObject(KnowledgeObject): # pylint: disable=too-few-public-methods
+    def __init__(self, meta, path):
+        super().__init__(ObjectType.Image)
+        self.meta = meta
+        self.path = path
+
+
+# define a email class
+class EmailObject(KnowledgeObject): # pylint: disable=too-few-public-methods
+    def __init__(self, meta):
+        super().__init__(ObjectType.Email)
+        self.meta = meta
+        self.text = [ObjectID]
+        self.images = [ObjectID]
+
+    

src/knowledge/object/object_id.py

@@ -0,0 +1,53 @@
+# define a object type enum
+from abc import ABC, abstractmethod
+from enum import Enum
+from .hash import HashValue
+import base58
+import base36
+
+
+class ObjectType(Enum):
+    Chunk = 7
+    TextChunk = 100
+    Image = 101
+    Email = 102
+
+
+# define a object ID class to identify a object
+class ObjectID:  # pylint: disable=too-few-public-methods
+    def __init__(self, object_type: ObjectType, value: bytes):
+        assert len(value) == 32, "ObjectID must be 32 bytes long"
+        self.object_type = object_type
+        self.value = value
+
+    def __str__(self):
+        return self.to_base58()
+
+    def get_object_type(self):
+        return self.object_type
+    
+    def to_base58(self):
+        return base58.b58encode(self.value).decode()
+
+    @staticmethod
+    def from_base58(s):
+        return ObjectID(base58.b58decode(s))
+
+    def to_base36(self):
+        # Convert the bytes to int before encoding
+        num = int.from_bytes(self.value, "big")
+        return base36.dumps(num)
+
+    @staticmethod
+    def from_base36(s):
+        # Decode to int and then convert to bytes
+        num = base36.loads(s)
+        return ObjectID(num.to_bytes((num.bit_length() + 7) // 8, "big"))
+
+    @staticmethod
+    def new_chunk_id(chunk_hash: HashValue):
+        return ObjectID(ObjectType.Chunk, chunk_hash.value)
+    
+    @staticmethod
+    def hash_data(data: bytes):
+        return ObjectID.new_chunk_id(HashValue.hash_data(data))

src/knowledge/object/object_store.py

@@ -0,0 +1,24 @@
+import os
+import logging
+from .blob import FileBlobStorage
+from .object_id import ObjectID
+
+
+class ObjectStore:
+    def __init__(self, root_dir: str):
+        logging.info(f"will init object blob store, root_dir={root_dir}")
+
+        blob_dir = os.path.join(root_dir, "blob")
+        if not os.path.exists(blob_dir):
+            logging.info(f"will create blob dir: {blob_dir}")
+            os.makedirs(blob_dir)
+        self.blob = FileBlobStorage(blob_dir)
+
+    def put_object(self, object_id: ObjectID, contents: bytes):
+        self.blob.put(object_id, contents)
+
+    def get_object(self, object_id: ObjectID) -> bytes:
+        return self.blob.get(object_id)
+
+    def delete_object(self, object_id: ObjectID):
+        self.blob.delete(object_id)