From 33cf3fbb7f017ab1b79dce5c5ca4d4e1c3092844 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Fri, 24 Jul 2026 01:13:04 -0400 Subject: [PATCH] refac --- backend/open_webui/routers/retrieval.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index 2b9ec9d956..5ca0e464dd 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -127,6 +127,8 @@ from sqlalchemy.ext.asyncio import AsyncSession log = logging.getLogger(__name__) +TIKTOKEN_DISALLOWED_SPECIAL = () + ########################################## # # Utility functions @@ -1582,7 +1584,7 @@ def get_splitter_length_function( ) -> Callable[[str], int]: if config.TEXT_SPLITTER == 'token': encoding = tiktoken.get_encoding(str(config.TIKTOKEN_ENCODING_NAME)) - return lambda text: len(encoding.encode(text, disallowed_special=())) + return lambda text: len(encoding.encode(text, disallowed_special=TIKTOKEN_DISALLOWED_SPECIAL)) if config.TEXT_SPLITTER == 'token_transformers': tokenizer = get_transformers_tokenizer(request, config) @@ -1689,6 +1691,7 @@ def save_docs_to_vector_db( chunk_size=config.CHUNK_SIZE, chunk_overlap=config.CHUNK_OVERLAP, add_start_index=True, + disallowed_special=TIKTOKEN_DISALLOWED_SPECIAL, ) docs = text_splitter.split_documents(docs) elif config.TEXT_SPLITTER == 'token_transformers':