diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index d406f09c8b..4bb2fbca88 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -1579,12 +1579,21 @@ def get_transformers_tokenizer(request: Request, config: RetrievalConfig): if not os.path.exists(tokenizer_model) and '/' not in tokenizer_model: tokenizer_model = f'sentence-transformers/{tokenizer_model}' - return AutoTokenizer.from_pretrained( + cache_dir = os.getenv('SENTENCE_TRANSFORMERS_HOME') or os.getenv('HF_HUB_CACHE') + local_files_only = not RAG_EMBEDDING_MODEL_AUTO_UPDATE + tokenizer_key = (tokenizer_model, cache_dir, local_files_only) + cached_tokenizer = getattr(request.app.state, 'transformers_tokenizer', None) + if cached_tokenizer and cached_tokenizer[0] == tokenizer_key: + return cached_tokenizer[1] + + tokenizer = AutoTokenizer.from_pretrained( tokenizer_model, - cache_dir=os.getenv('SENTENCE_TRANSFORMERS_HOME') or os.getenv('HF_HUB_CACHE'), + cache_dir=cache_dir, trust_remote_code=RAG_EMBEDDING_MODEL_TRUST_REMOTE_CODE, - local_files_only=not RAG_EMBEDDING_MODEL_AUTO_UPDATE, + local_files_only=local_files_only, ) + request.app.state.transformers_tokenizer = (tokenizer_key, tokenizer) + return tokenizer tokenizer = getattr(getattr(request.app.state, 'ef', None), 'tokenizer', None) if tokenizer is not None: @@ -1952,7 +1961,7 @@ async def process_file( ] text_content = ' '.join([doc.page_content for doc in docs]) - log.debug(f'text_content: {text_content}') + log.debug('text_content: %s', text_content) await Files.update_file_data_by_id( file.id, {'content': text_content}, @@ -2094,7 +2103,7 @@ async def process_text( ) ] text_content = form_data.content - log.debug(f'text_content: {text_content}') + log.debug('text_content: %s', text_content) config = await get_retrieval_config() result = await run_in_threadpool(save_docs_to_vector_db, request, docs, collection_name, config, user=user) @@ -2131,7 +2140,7 @@ async def process_web( config = await get_retrieval_config() try: content, docs = await get_content_from_url(request, form_data.url) - log.debug(f'text_content: {content}') + log.debug('text_content: %s', content) if process: collection_name = form_data.collection_name @@ -2645,8 +2654,9 @@ async def process_web_search(request: Request, form_data: SearchForm, user=Depen urls = [ doc.metadata.get('source') for doc in docs if doc.metadata.get('source') ] # only keep the urls returned by the loader + url_set = set(urls) result_items = [ - dict(item) for item in result_items if item.link in urls + dict(item) for item in result_items if item.link in url_set ] # only keep the search results that have been loaded if config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL: