default text embedding with local node

This commit is contained in:
tsukasa
2023-09-26 16:37:30 +08:00
parent 411ab61880
commit cf2d917404
3 changed files with 14 additions and 14 deletions
+2 -3
View File
@@ -3,14 +3,14 @@ import hashlib
import re
import tiktoken
import logging
from typing import Tuple, List
from typing import Callable, Iterable, Optional, Tuple, List
from .chunk_store import ChunkStore
from .chunk import ChunkID, PositionFileRange, PositionType
from ..object import HashValue
from .tracker import ChunkTracker
from .chunk_list import ChunkList
def _join_docs(self, docs: List[str], separator: str) -> Optional[str]:
def _join_docs(docs: List[str], separator: str) -> Optional[str]:
text = separator.join(docs)
text = text.strip()
if text == "":
@@ -19,7 +19,6 @@ def _join_docs(self, docs: List[str], separator: str) -> Optional[str]:
return text
def _merge_splits(
self,
splits: Iterable[str],
separator: str,
chunk_size: int,