impl text splitter with tiktoken

2023-09-26 14:43:56 +08:00
parent 8b862c0075
commit 411ab61880
3 changed files with 147 additions and 32 deletions
@@ -59,7 +59,7 @@ class TestChunk(unittest.TestCase):
        with open(text_file, "r", encoding="utf-8") as file:
            text = file.read()

-        gen.create_chunk_list_from_text(text, 1024)
+        gen.create_chunk_list_from_text(text)


 if __name__ == "__main__":