Refactor the code directory structure to better suit the current complexity
This commit is contained in:
Liu Zhicong
@@ -0,0 +1,6 @@
|
||||
from .object import KnowledgeObject
|
||||
from .blob import FileBlobStorage
|
||||
from .hash import HashValue, hash_data
|
||||
from .relation import ObjectRelationStore
|
||||
from .object_store import ObjectStore
|
||||
from .object_id import ObjectID, ObjectType
|
||||
@@ -0,0 +1,63 @@
|
||||
import os
|
||||
import shutil
|
||||
from .object import ObjectID
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FileBlobStorage:
|
||||
def __init__(self, root):
|
||||
self.root = root
|
||||
|
||||
def get_full_path(self, object_id: ObjectID, auto_create: bool = True):
|
||||
if os.name == "nt": # Windows
|
||||
hash_str = object_id.to_base36()
|
||||
len = 3
|
||||
else:
|
||||
hash_str = str(object_id)
|
||||
len = 2
|
||||
|
||||
tmp, first = hash_str[:-len], hash_str[-len:]
|
||||
second = tmp[-len:]
|
||||
|
||||
if os.name == "nt": # Windows
|
||||
if second in ["con", "aux", "nul", "prn"]:
|
||||
second = tmp[-(len + 1) :]
|
||||
if first in ["con", "aux", "nul", "prn"]:
|
||||
first = f"{first}_"
|
||||
|
||||
path = os.path.join(self.root, first, second)
|
||||
if auto_create and not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
|
||||
path = os.path.join(path, hash_str)
|
||||
|
||||
return path
|
||||
|
||||
def write_sync(self, path: str, contents: bytes):
|
||||
with open(path, "wb") as f:
|
||||
f.write(contents)
|
||||
|
||||
def put(self, object_id: ObjectID, contents: bytes):
|
||||
full_path = self.get_full_path(object_id)
|
||||
if os.path.exists(full_path):
|
||||
logger.warning(f"will replace object: {object_id}")
|
||||
|
||||
self.write_sync(full_path, contents)
|
||||
|
||||
def get(self, object_id: ObjectID) -> bytes:
|
||||
full_path = self.get_full_path(object_id)
|
||||
if not os.path.exists(full_path):
|
||||
return None
|
||||
|
||||
with open(full_path, "rb") as f:
|
||||
return f.read()
|
||||
|
||||
def delete(self, object_id: ObjectID):
|
||||
full_path = self.get_full_path(object_id)
|
||||
if os.path.exists(full_path):
|
||||
os.remove(full_path)
|
||||
|
||||
def exists(self, object_id: ObjectID) -> bool:
|
||||
full_path = self.get_full_path(object_id)
|
||||
return os.path.exists(full_path)
|
||||
@@ -0,0 +1,42 @@
|
||||
import hashlib
|
||||
import base58
|
||||
import base36
|
||||
|
||||
class HashValue:
|
||||
def __init__(self, value: bytes):
|
||||
assert len(value) == 32, "HashValue must be 32 bytes long"
|
||||
self.value = value
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.to_base58()
|
||||
|
||||
@staticmethod
|
||||
def hash_data(data):
|
||||
return hash_data(data)
|
||||
|
||||
def to_base58(self):
|
||||
return base58.b58encode(self.value).decode()
|
||||
|
||||
@staticmethod
|
||||
def from_base58(s):
|
||||
return HashValue(base58.b58decode(s))
|
||||
|
||||
def to_base36(self):
|
||||
# Convert the bytes to int before encoding
|
||||
num = int.from_bytes(self.value, 'big')
|
||||
return base36.dumps(num)
|
||||
|
||||
@staticmethod
|
||||
def from_base36(s):
|
||||
# Decode to int and then convert to bytes
|
||||
num = base36.loads(s)
|
||||
return HashValue(num.to_bytes((num.bit_length() + 7) // 8, 'big'))
|
||||
|
||||
|
||||
HASH_VALUE_LEN = 32
|
||||
|
||||
|
||||
def hash_data(data: bytes):
|
||||
sha256 = hashlib.sha256()
|
||||
sha256.update(data)
|
||||
return HashValue(sha256.digest())
|
||||
@@ -0,0 +1,78 @@
|
||||
# define a object type enum
|
||||
from __future__ import annotations
|
||||
from abc import ABC, abstractmethod
|
||||
from enum import Enum
|
||||
|
||||
from .object_id import ObjectID, ObjectType
|
||||
import hashlib
|
||||
import json
|
||||
import pickle
|
||||
from typing import Any
|
||||
|
||||
|
||||
class ObjectEnhancedJSONEncoder(json.JSONEncoder):
|
||||
def default(self, o: Any) -> Any:
|
||||
if isinstance(o, ObjectID):
|
||||
return o.to_base58()
|
||||
|
||||
return super().default(o)
|
||||
|
||||
|
||||
class KnowledgeObject(ABC):
|
||||
def __init__(self, object_type: ObjectType, desc: dict = {}, body: dict = {}):
|
||||
self.desc = desc
|
||||
self.body = body
|
||||
self.object_type = object_type
|
||||
|
||||
def get_object_type(self) -> ObjectType:
|
||||
return self.object_type
|
||||
|
||||
def object_id(self) -> ObjectID:
|
||||
return self.calculate_id()
|
||||
|
||||
def set_desc_with_key_value(self, key, value):
|
||||
self.desc[key] = value
|
||||
|
||||
def get_desc_with_key(self, key):
|
||||
return self.desc.get(key)
|
||||
|
||||
def get_desc(self) -> dict:
|
||||
return self.desc
|
||||
|
||||
def set_body_with_key_value(self, key, value):
|
||||
self.body[key] = value
|
||||
|
||||
def get_body_with_key(self, key):
|
||||
return self.body.get(key)
|
||||
|
||||
def get_body(self) -> dict:
|
||||
return self.body
|
||||
|
||||
def get_summary(self) -> str:
|
||||
return self.desc.get("summary")
|
||||
|
||||
# def get_articl_catelog(self) -> str:
|
||||
# assert self.object_type == ObjectType.Document
|
||||
# return self.desc.get("catelog")
|
||||
|
||||
# def get_article_full_content(self) -> str:
|
||||
# assert self.object_type == ObjectType.Document
|
||||
# return self.body
|
||||
|
||||
def calculate_id(self):
|
||||
# Convert the object_type and desc to string and compute the SHA256 hash
|
||||
data = json.dumps(
|
||||
{"object_type": self.object_type, "desc": self.desc},
|
||||
cls=ObjectEnhancedJSONEncoder,
|
||||
)
|
||||
sha256 = hashlib.sha256()
|
||||
sha256.update(data.encode())
|
||||
hash_bytes = sha256.digest()
|
||||
return ObjectID(bytes([self.object_type]) + hash_bytes[1:])
|
||||
|
||||
def encode(self) -> bytes:
|
||||
return pickle.dumps(self)
|
||||
|
||||
# @staticmethod
|
||||
# def decode(data: bytes) -> "ImageObject":
|
||||
# return pickle.loads(data)
|
||||
@@ -0,0 +1,69 @@
|
||||
# define a object type enum
|
||||
from abc import ABC, abstractmethod
|
||||
from enum import IntEnum
|
||||
from .hash import HashValue
|
||||
import base58
|
||||
import base36
|
||||
|
||||
|
||||
class ObjectType(IntEnum):
|
||||
Chunk = 7
|
||||
Image = 101
|
||||
Video = 102
|
||||
Document = 103
|
||||
RichText = 104
|
||||
Email = 105
|
||||
UserDef = 200
|
||||
|
||||
def is_user_def(self) -> bool:
|
||||
return self.value >= 200
|
||||
|
||||
def get_user_def_type_code(self):
|
||||
return (self.value - 200) if self.is_user_def() else None
|
||||
|
||||
@classmethod
|
||||
def from_user_def_type_code(cls, value):
|
||||
return value + 200
|
||||
|
||||
|
||||
# define a object ID class to identify a object
|
||||
class ObjectID: # pylint: disable=too-few-public-methods
|
||||
def __init__(self, value: bytes):
|
||||
assert len(value) == 32, "ObjectID must be 32 bytes long"
|
||||
self.value = value
|
||||
|
||||
def __str__(self):
|
||||
return self.to_base58()
|
||||
|
||||
def to_base58(self):
|
||||
return base58.b58encode(self.value).decode()
|
||||
|
||||
@staticmethod
|
||||
def from_base58(s):
|
||||
return ObjectID(base58.b58decode(s))
|
||||
|
||||
def to_base36(self):
|
||||
# Convert the bytes to int before encoding
|
||||
num = int.from_bytes(self.value, "big")
|
||||
return base36.dumps(num)
|
||||
|
||||
@staticmethod
|
||||
def from_base36(s):
|
||||
# Decode to int and then convert to bytes
|
||||
num = base36.loads(s)
|
||||
return ObjectID(num.to_bytes((num.bit_length() + 7) // 8, "big"))
|
||||
|
||||
@staticmethod
|
||||
def new_chunk_id(chunk_hash: HashValue):
|
||||
assert len(chunk_hash.value) == 32, "ObjectID must be 32 bytes long"
|
||||
return ObjectID(bytes([ObjectType.Chunk]) + chunk_hash.value[1:])
|
||||
|
||||
def get_object_type(self) -> ObjectType:
|
||||
return ObjectType(self.value[0])
|
||||
|
||||
@staticmethod
|
||||
def hash_data(data: bytes):
|
||||
return ObjectID.new_chunk_id(HashValue.hash_data(data))
|
||||
|
||||
def __eq__(self, other) -> bool:
|
||||
return self.value == other.value
|
||||
@@ -0,0 +1,25 @@
|
||||
import os
|
||||
import logging
|
||||
from .blob import FileBlobStorage
|
||||
from .object_id import ObjectID
|
||||
|
||||
|
||||
class ObjectStore:
|
||||
def __init__(self, root_dir: str):
|
||||
logging.info(f"will init object blob store, root_dir={root_dir}")
|
||||
|
||||
blob_dir = os.path.join(root_dir, "blob")
|
||||
if not os.path.exists(blob_dir):
|
||||
logging.info(f"will create blob dir: {blob_dir}")
|
||||
os.makedirs(blob_dir)
|
||||
self.blob = FileBlobStorage(blob_dir)
|
||||
|
||||
def put_object(self, object_id: ObjectID, contents: bytes):
|
||||
logging.info(f"will put object: {object_id}")
|
||||
self.blob.put(object_id, contents)
|
||||
|
||||
def get_object(self, object_id: ObjectID) -> bytes:
|
||||
return self.blob.get(object_id)
|
||||
|
||||
def delete_object(self, object_id: ObjectID):
|
||||
self.blob.delete(object_id)
|
||||
@@ -0,0 +1,104 @@
|
||||
# define a relation store class
|
||||
from .object_id import ObjectID
|
||||
import sqlite3
|
||||
from typing import List, Tuple, Optional
|
||||
import logging
|
||||
import os
|
||||
from enum import IntEnum
|
||||
|
||||
|
||||
class ObjectRelationType(IntEnum):
|
||||
Parent = 1
|
||||
|
||||
|
||||
class ObjectRelationStore:
|
||||
def __init__(self, root_dir: str):
|
||||
if not os.path.exists(root_dir):
|
||||
os.makedirs(root_dir)
|
||||
file = os.path.join(root_dir, "relation.db")
|
||||
logging.info(f"will init object relation store, db={file}")
|
||||
|
||||
self.conn = sqlite3.connect(file)
|
||||
self.cursor = self.conn.cursor()
|
||||
self.cursor.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS relations (
|
||||
object_id TEXT,
|
||||
assoc_id TEXT,
|
||||
relation_type TEXT,
|
||||
PRIMARY KEY (object_id, assoc_id, relation_type)
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
def add_relation(
|
||||
self,
|
||||
object_id: ObjectID,
|
||||
assoc_id: ObjectID,
|
||||
relation_type: ObjectRelationType = ObjectRelationType.Parent,
|
||||
):
|
||||
if relation_type == None:
|
||||
relation_type = ObjectRelationType.Parent
|
||||
|
||||
self.cursor.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO relations (object_id, assoc_id, relation_type)
|
||||
VALUES (?, ?, ?)
|
||||
""",
|
||||
(str(object_id), str(assoc_id), relation_type.value),
|
||||
)
|
||||
self.conn.commit()
|
||||
|
||||
def get_related_objects(
|
||||
self, object_id: ObjectID, relation_type: Optional[ObjectRelationType] = None
|
||||
) -> List[ObjectID]:
|
||||
if relation_type:
|
||||
self.cursor.execute(
|
||||
"""
|
||||
SELECT assoc_id FROM relations WHERE object_id = ? AND relation_type = ?
|
||||
""",
|
||||
(str(object_id), relation_type.value),
|
||||
)
|
||||
else:
|
||||
self.cursor.execute(
|
||||
"""
|
||||
SELECT assoc_id FROM relations WHERE object_id = ?
|
||||
""",
|
||||
(str(object_id),),
|
||||
)
|
||||
return [ObjectID.from_base58(row[0]) for row in self.cursor.fetchall()]
|
||||
|
||||
def get_related_root_objects(
|
||||
self, object_id: ObjectID, relation_type: Optional[ObjectRelationType] = None
|
||||
) -> List[ObjectID]:
|
||||
root_objects = []
|
||||
related_objects = self.get_related_objects(object_id, relation_type)
|
||||
history = []
|
||||
history.append(object_id)
|
||||
|
||||
while related_objects:
|
||||
for obj in related_objects:
|
||||
next_related_objects = self.get_related_objects(obj, relation_type)
|
||||
if not next_related_objects:
|
||||
if obj not in root_objects:
|
||||
root_objects.append(obj)
|
||||
else:
|
||||
for related_object in next_related_objects:
|
||||
if obj not in history:
|
||||
related_objects.append(related_object)
|
||||
else:
|
||||
logging.warning(
|
||||
f"loop detected: {obj} <-> {related_object}"
|
||||
)
|
||||
related_objects = next_related_objects
|
||||
|
||||
return root_objects
|
||||
|
||||
def delete_relation(self, object_id: ObjectID):
|
||||
self.cursor.execute(
|
||||
"""
|
||||
DELETE FROM relations WHERE object_id = ?
|
||||
""",
|
||||
(str(object_id),),
|
||||
)
|
||||
self.conn.commit()
|
||||
Reference in New Issue
Block a user