diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index af7234bb2c..00fc1b36f3 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -3491,6 +3491,12 @@ YANDEX_WEB_SEARCH_CONFIG = PersistentConfig( os.environ.get("YANDEX_WEB_SEARCH_CONFIG", ""), ) +YOUCOM_API_KEY = PersistentConfig( + "YOUCOM_API_KEY", + "rag.web.search.youcom_api_key", + os.environ.get("YOUCOM_API_KEY", ""), +) + #################################### # Images #################################### diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index 693a611266..57660fe9e4 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -360,6 +360,7 @@ from open_webui.config import ( YANDEX_WEB_SEARCH_URL, YANDEX_WEB_SEARCH_API_KEY, YANDEX_WEB_SEARCH_CONFIG, + YOUCOM_API_KEY, # WebUI WEBUI_AUTH, WEBUI_NAME, @@ -1067,6 +1068,7 @@ app.state.config.EXTERNAL_WEB_LOADER_API_KEY = EXTERNAL_WEB_LOADER_API_KEY app.state.config.YANDEX_WEB_SEARCH_URL = YANDEX_WEB_SEARCH_URL app.state.config.YANDEX_WEB_SEARCH_API_KEY = YANDEX_WEB_SEARCH_API_KEY app.state.config.YANDEX_WEB_SEARCH_CONFIG = YANDEX_WEB_SEARCH_CONFIG +app.state.config.YOUCOM_API_KEY = YOUCOM_API_KEY app.state.config.PLAYWRIGHT_WS_URL = PLAYWRIGHT_WS_URL diff --git a/backend/open_webui/retrieval/web/ydc.py b/backend/open_webui/retrieval/web/ydc.py new file mode 100644 index 0000000000..21d725a895 --- /dev/null +++ b/backend/open_webui/retrieval/web/ydc.py @@ -0,0 +1,73 @@ +import logging +from typing import Optional, List + +import requests +from open_webui.retrieval.web.main import SearchResult, get_filtered_results + +log = logging.getLogger(__name__) + + +def search_youcom( + api_key: str, + query: str, + count: int, + filter_list: Optional[List[str]] = None, + language: str = "EN", +) -> List[SearchResult]: + """Search using You.com's YDC Index API and return the results as a list of SearchResult objects. + + Args: + api_key (str): A You.com API key + query (str): The query to search for + count (int): Maximum number of results to return + filter_list (list[str], optional): Domain filter list + language (str): Language code for search results (default: "EN") + """ + url = "https://ydc-index.io/v1/search" + headers = { + "Accept": "application/json", + "X-API-KEY": api_key, + } + params = { + "query": query, + "count": count, + "language": language, + } + + response = requests.get(url, headers=headers, params=params) + response.raise_for_status() + + json_response = response.json() + results = json_response.get("results", {}).get("web", []) + + if filter_list: + results = get_filtered_results(results, filter_list) + + return [ + SearchResult( + link=result["url"], + title=result.get("title"), + snippet=_build_snippet(result), + ) + for result in results[:count] + ] + + +def _build_snippet(result: dict) -> str: + """Combine the description and snippets list into a single string. + + The You.com API returns a short ``description`` plus a ``snippets`` + list with richer passages. Merging them gives downstream retrieval + (embedding, BM25, bypass-loader context) the most content to work with. + """ + parts: list[str] = [] + + description = result.get("description") + if description: + parts.append(description) + + snippets = result.get("snippets") + if snippets and isinstance(snippets, list): + parts.extend(snippets) + + return "\n\n".join(parts) diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index 9630462613..2c8de8c8df 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -77,6 +77,7 @@ from open_webui.retrieval.web.sougou import search_sougou from open_webui.retrieval.web.firecrawl import search_firecrawl from open_webui.retrieval.web.external import search_external from open_webui.retrieval.web.yandex import search_yandex +from open_webui.retrieval.web.ydc import search_youcom from open_webui.retrieval.utils import ( get_content_from_url, @@ -591,6 +592,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)): "YANDEX_WEB_SEARCH_URL": request.app.state.config.YANDEX_WEB_SEARCH_URL, "YANDEX_WEB_SEARCH_API_KEY": request.app.state.config.YANDEX_WEB_SEARCH_API_KEY, "YANDEX_WEB_SEARCH_CONFIG": request.app.state.config.YANDEX_WEB_SEARCH_CONFIG, + "YOUCOM_API_KEY": request.app.state.config.YOUCOM_API_KEY, }, } @@ -657,6 +659,7 @@ class WebConfig(BaseModel): YANDEX_WEB_SEARCH_URL: Optional[str] = None YANDEX_WEB_SEARCH_API_KEY: Optional[str] = None YANDEX_WEB_SEARCH_CONFIG: Optional[str] = None + YOUCOM_API_KEY: Optional[str] = None class ConfigForm(BaseModel): @@ -1213,6 +1216,9 @@ async def update_rag_config( request.app.state.config.YANDEX_WEB_SEARCH_CONFIG = ( form_data.web.YANDEX_WEB_SEARCH_CONFIG ) + request.app.state.config.YOUCOM_API_KEY = ( + form_data.web.YOUCOM_API_KEY + ) return { "status": True, @@ -1340,6 +1346,7 @@ async def update_rag_config( "YANDEX_WEB_SEARCH_URL": request.app.state.config.YANDEX_WEB_SEARCH_URL, "YANDEX_WEB_SEARCH_API_KEY": request.app.state.config.YANDEX_WEB_SEARCH_API_KEY, "YANDEX_WEB_SEARCH_CONFIG": request.app.state.config.YANDEX_WEB_SEARCH_CONFIG, + "YOUCOM_API_KEY": request.app.state.config.YOUCOM_API_KEY, }, } @@ -2299,6 +2306,13 @@ def search_web( request.app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST, user=user, ) + elif engine == "youcom": + return search_youcom( + request.app.state.config.YOUCOM_API_KEY, + query, + request.app.state.config.WEB_SEARCH_RESULT_COUNT, + request.app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST, + ) else: raise Exception("No search engine API key found in environment variables") diff --git a/src/lib/components/admin/Settings/WebSearch.svelte b/src/lib/components/admin/Settings/WebSearch.svelte index 4778b69ec5..4d5d167c98 100644 --- a/src/lib/components/admin/Settings/WebSearch.svelte +++ b/src/lib/components/admin/Settings/WebSearch.svelte @@ -37,7 +37,8 @@ 'sougou', 'firecrawl', 'external', - 'yandex' + 'yandex', + 'youcom' ]; let webLoaderEngines = ['playwright', 'firecrawl', 'tavily', 'external']; @@ -786,6 +787,19 @@ + {:else if webConfig.WEB_SEARCH_ENGINE === 'youcom'} +
+
+
+ {$i18n.t('You.com API Key')} +
+ + +
+
{/if} {#if webConfig.WEB_SEARCH_ENGINE === 'duckduckgo'}