ewilliams28/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-08-13 01:02:25 -06:00
Code Issues Packages Projects Releases Wiki Activity

refac

Browse Source
This commit is contained in:
Timothy Jaeryang Baek
2026-07-24 01:13:04 -04:00
parent ff11ff5a3e
commit 33cf3fbb7f
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/open_webui/routers/retrieval.py
+4 -1
View File
@@ -127,6 +127,8 @@ from sqlalchemy.ext.asyncio import AsyncSession
log = logging.getLogger(__name__)
TIKTOKEN_DISALLOWED_SPECIAL = ()
##########################################
#
# Utility functions
@@ -1582,7 +1584,7 @@ def get_splitter_length_function(
) -> Callable[[str], int]:
if config.TEXT_SPLITTER == 'token':
encoding = tiktoken.get_encoding(str(config.TIKTOKEN_ENCODING_NAME))
return lambda text: len(encoding.encode(text, disallowed_special=()))
return lambda text: len(encoding.encode(text, disallowed_special=TIKTOKEN_DISALLOWED_SPECIAL))
if config.TEXT_SPLITTER == 'token_transformers':
tokenizer = get_transformers_tokenizer(request, config)
@@ -1689,6 +1691,7 @@ def save_docs_to_vector_db(
chunk_size=config.CHUNK_SIZE,
chunk_overlap=config.CHUNK_OVERLAP,
add_start_index=True,
disallowed_special=TIKTOKEN_DISALLOWED_SPECIAL,
)
docs = text_splitter.split_documents(docs)
elif config.TEXT_SPLITTER == 'token_transformers':