impl text splitter with tiktoken

2023-09-26 14:43:56 +08:00
parent 8b862c0075
commit 411ab61880
3 changed files with 147 additions and 32 deletions
@@ -50,11 +50,7 @@ class DocumentObjectBuilder:
        return self

    def build(self) -> DocumentObject:
-        chunk_list = KnowledgeStore().get_chunk_list_writer().create_chunk_list_from_text(
-            self.text,
-            1024 * 4,
-            ".?!\n"
-        )
+        chunk_list = KnowledgeStore().get_chunk_list_writer().create_chunk_list_from_text(self.text)
        doc = DocumentObject(self.meta, self.tags, chunk_list)
        doc_id = doc.calculate_id()