diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index 8765ccd617..2d9cb5de8b 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -1179,6 +1179,12 @@ PERPLEXITY_SEARCH_CONTEXT_USAGE = os.getenv('PERPLEXITY_SEARCH_CONTEXT_USAGE', ' PERPLEXITY_SEARCH_API_URL = os.getenv('PERPLEXITY_SEARCH_API_URL', 'https://api.perplexity.ai/search') +MICROSOFT_WEB_IQ_API_BASE_URL = os.getenv('MICROSOFT_WEB_IQ_API_BASE_URL', 'https://api.microsoft.ai/v3') + +MICROSOFT_WEB_IQ_API_KEY = os.getenv('MICROSOFT_WEB_IQ_API_KEY', '') + +MICROSOFT_WEB_IQ_LANGUAGE = os.getenv('MICROSOFT_WEB_IQ_LANGUAGE', 'en') + SOUGOU_API_SID = os.getenv('SOUGOU_API_SID', '') SOUGOU_API_SK = os.getenv('SOUGOU_API_SK', '') @@ -2805,6 +2811,9 @@ DEFAULT_CONFIG = { 'rag.web.search.perplexity_model': PERPLEXITY_MODEL, 'rag.web.search.perplexity_search_context_usage': PERPLEXITY_SEARCH_CONTEXT_USAGE, 'rag.web.search.perplexity_search_api_url': PERPLEXITY_SEARCH_API_URL, + 'rag.web.search.microsoft_web_iq_api_base_url': MICROSOFT_WEB_IQ_API_BASE_URL, + 'rag.web.search.microsoft_web_iq_api_key': MICROSOFT_WEB_IQ_API_KEY, + 'rag.web.search.microsoft_web_iq_language': MICROSOFT_WEB_IQ_LANGUAGE, 'rag.web.search.sougou_api_sid': SOUGOU_API_SID, 'rag.web.search.sougou_api_sk': SOUGOU_API_SK, 'rag.web.search.tavily_api_key': TAVILY_API_KEY, diff --git a/backend/open_webui/retrieval/loaders/microsoft_web_iq.py b/backend/open_webui/retrieval/loaders/microsoft_web_iq.py new file mode 100644 index 0000000000..fdf10e6332 --- /dev/null +++ b/backend/open_webui/retrieval/loaders/microsoft_web_iq.py @@ -0,0 +1,110 @@ +import logging +import time +from collections.abc import Iterator +from typing import Any +from urllib.parse import urlparse + +import requests +from langchain_core.document_loaders import BaseLoader +from langchain_core.documents import Document + +log = logging.getLogger(__name__) + +DEFAULT_MICROSOFT_WEB_IQ_API_BASE_URL = 'https://api.microsoft.ai/v3' +MICROSOFT_BROWSE_RETRY_STATUS_CODES = {202, 429, 500, 502, 503, 504} +MICROSOFT_BROWSE_MAX_RETRIES = 2 + + +class MicrosoftWebIQLoader(BaseLoader): + def __init__( + self, + urls: str | list[str], + api_base_url: str, + api_key: str, + language: str = 'en', + verify_ssl: bool = True, + timeout: Any = None, + continue_on_failure: bool = True, + ) -> None: + self.urls = urls if isinstance(urls, list) else [urls] + self.api_base_url = (api_base_url or DEFAULT_MICROSOFT_WEB_IQ_API_BASE_URL).rstrip('/') + self.api_key = api_key + self.language = language + self.verify_ssl = verify_ssl + self.timeout = timeout + self.continue_on_failure = continue_on_failure + + def lazy_load(self) -> Iterator[Document]: + for url in self.urls: + try: + doc = self._browse_url(url) + if doc is not None: + yield doc + except Exception as e: + if self.continue_on_failure: + log.warning(f'Error browsing {url} with Microsoft Web IQ: {e}') + else: + raise e + + def _browse_url(self, url: str) -> Document | None: + headers = { + 'host': urlparse(self.api_base_url).netloc or 'api.microsoft.ai', + 'x-apikey': self.api_key, + 'content-type': 'application/json', + } + payload = { + 'url': url, + 'contentFormat': 'markdown', + 'liveCrawl': 'fallback', + 'renderDynamicPages': True, + 'language': self.language, + } + try: + request_timeout = float(self.timeout) + except (TypeError, ValueError): + request_timeout = 60 + request_timeout = request_timeout if request_timeout > 0 else 60 + + data: dict[str, Any] = {} + for attempt in range(MICROSOFT_BROWSE_MAX_RETRIES + 1): + response = requests.post( + f'{self.api_base_url}/browse', + json=payload, + headers=headers, + timeout=request_timeout, + verify=self.verify_ssl, + ) + + if response.status_code in MICROSOFT_BROWSE_RETRY_STATUS_CODES and attempt < MICROSOFT_BROWSE_MAX_RETRIES: + try: + body = response.json() + except Exception: + body = {} + retry_after = body.get('retryAfter') if isinstance(body, dict) else None + retry_after = retry_after or response.headers.get('Retry-After') + try: + delay = min(10.0, max(0.0, float(str(retry_after).rstrip('s')))) + except (TypeError, ValueError): + delay = min(8.0, float(2**attempt)) + log.warning( + 'Microsoft Browse %s returned HTTP %s; retrying in %.1fs', + url, + response.status_code, + delay, + ) + time.sleep(delay) + continue + + response.raise_for_status() + data = response.json() + break + + content = data.get('content') or '' + if not isinstance(content, str) or not content.strip(): + return None + + metadata = {'source': data.get('url') or url} + if data.get('title'): + metadata['title'] = data['title'] + + return Document(page_content=content, metadata=metadata) diff --git a/backend/open_webui/retrieval/web/microsoft_web_iq.py b/backend/open_webui/retrieval/web/microsoft_web_iq.py new file mode 100644 index 0000000000..c4188b684c --- /dev/null +++ b/backend/open_webui/retrieval/web/microsoft_web_iq.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import logging +from urllib.parse import urlparse + +import requests +from open_webui.retrieval.web.main import SearchResult, get_filtered_results +from open_webui.utils.headers import include_user_info_headers + +log = logging.getLogger(__name__) + +DEFAULT_MICROSOFT_WEB_IQ_API_BASE_URL = 'https://api.microsoft.ai/v3' + + +def search_microsoft_web_iq( + api_base_url: str, + api_key: str, + query: str, + count: int, + filter_list: list[str | None] | None = None, + language: str = 'en', + user=None, +) -> list[SearchResult]: + try: + api_base_url = (api_base_url or DEFAULT_MICROSOFT_WEB_IQ_API_BASE_URL).rstrip('/') + headers = { + 'host': urlparse(api_base_url).netloc or 'api.microsoft.ai', + 'x-apikey': api_key, + 'content-type': 'application/json', + } + if user is not None: + headers = include_user_info_headers(headers, user) + + response = requests.post( + f'{api_base_url}/search/web', + json={ + 'query': query, + 'maxResults': count, + 'language': language, + 'contentFormat': 'passage', + }, + headers=headers, + ) + response.raise_for_status() + + results = response.json().get('webResults', []) + if filter_list: + results = get_filtered_results(results, filter_list) + + return [ + SearchResult( + link=result['url'], + title=result.get('title'), + snippet=result.get('content'), + ) + for result in results + ] + except Exception as e: + log.error(f'Error searching with Microsoft Web IQ API: {e}') + return [] diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index 327d7e2738..cb3c32f238 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -36,6 +36,9 @@ from open_webui.config import ( FIRECRAWL_API_BASE_URL, FIRECRAWL_API_KEY, FIRECRAWL_TIMEOUT, + MICROSOFT_WEB_IQ_API_BASE_URL, + MICROSOFT_WEB_IQ_API_KEY, + MICROSOFT_WEB_IQ_LANGUAGE, PLAYWRIGHT_TIMEOUT, PLAYWRIGHT_WS_URL, TAVILY_API_KEY, @@ -52,6 +55,7 @@ from open_webui.env import ( USER_AGENT, ) from open_webui.retrieval.loaders.external_web import ExternalWebLoader +from open_webui.retrieval.loaders.microsoft_web_iq import MicrosoftWebIQLoader from open_webui.retrieval.loaders.tavily import TavilyLoader from open_webui.retrieval.web.firecrawl import scrape_firecrawl_url from open_webui.utils.misc import is_host_allowed @@ -356,6 +360,7 @@ class SafeTavilyLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): def __init__( self, web_paths: Union[str, List[str]], + api_base_url: str, api_key: str, extract_depth: Literal['basic', 'advanced'] = 'basic', continue_on_failure: bool = True, @@ -389,6 +394,7 @@ class SafeTavilyLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): # Store parameters for creating TavilyLoader instances self.web_paths = web_paths if isinstance(web_paths, list) else [web_paths] + self.api_base_url = api_base_url self.api_key = api_key self.extract_depth = extract_depth self.continue_on_failure = continue_on_failure @@ -464,6 +470,67 @@ class SafeTavilyLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): raise e +class SafeMicrosoftWebIQLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): + def __init__( + self, + web_paths: Union[str, List[str]], + api_key: str, + language: str = 'en', + verify_ssl: bool = True, + trust_env: bool = False, + requests_per_second: Optional[float] = None, + continue_on_failure: bool = True, + timeout: Optional[int] = None, + ): + self.web_paths = web_paths if isinstance(web_paths, list) else [web_paths] + self.api_key = api_key + self.language = language + self.verify_ssl = verify_ssl + self.trust_env = trust_env + self.requests_per_second = requests_per_second + self.last_request_time = None + self.continue_on_failure = continue_on_failure + self.timeout = timeout + + def lazy_load(self) -> Iterator[Document]: + valid_urls = [] + for url in self.web_paths: + try: + self._safe_process_url_sync(url) + valid_urls.append(url) + except Exception as e: + log.warning(f'SSL verification failed for {url}: {str(e)}') + if not self.continue_on_failure: + raise e + if not valid_urls: + if self.continue_on_failure: + log.warning('No valid URLs to process after SSL verification') + return + raise ValueError('No valid URLs to process after SSL verification') + + loader = MicrosoftWebIQLoader( + urls=valid_urls, + api_base_url=self.api_base_url, + api_key=self.api_key, + language=self.language, + verify_ssl=self.verify_ssl, + timeout=self.timeout, + continue_on_failure=self.continue_on_failure, + ) + yield from loader.lazy_load() + + async def alazy_load(self) -> AsyncIterator[Document]: + try: + docs = await run_in_threadpool(lambda: list(self.lazy_load())) + for doc in docs: + yield doc + except Exception as e: + if self.continue_on_failure: + log.warning(f'Error browsing URLs with Microsoft Web IQ: {e}') + else: + raise e + + class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessingMixin): """Load HTML pages safely with Playwright, supporting SSL verification, rate limiting, and remote browser connection. @@ -813,6 +880,17 @@ def get_web_loader( web_loader_args['api_key'] = TAVILY_API_KEY web_loader_args['extract_depth'] = TAVILY_EXTRACT_DEPTH + if WEB_LOADER_ENGINE == 'microsoft_web_iq': + WebLoaderClass = SafeMicrosoftWebIQLoader + web_loader_args['api_base_url'] = MICROSOFT_WEB_IQ_API_BASE_URL + web_loader_args['api_key'] = MICROSOFT_WEB_IQ_API_KEY + web_loader_args['language'] = MICROSOFT_WEB_IQ_LANGUAGE + if WEB_LOADER_TIMEOUT: + try: + web_loader_args['timeout'] = int(WEB_LOADER_TIMEOUT) + except ValueError: + pass + if WEB_LOADER_ENGINE == 'external': WebLoaderClass = ExternalWebLoader web_loader_args['external_url'] = EXTERNAL_WEB_LOADER_URL @@ -831,5 +909,5 @@ def get_web_loader( else: raise ValueError( f'Invalid WEB_LOADER_ENGINE: {WEB_LOADER_ENGINE}. ' - "Please set it to 'safe_web', 'playwright', 'firecrawl', or 'tavily'." + "Please set it to 'safe_web', 'playwright', 'firecrawl', 'tavily', 'external', or 'microsoft_web_iq'." ) diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index 41c9de51a8..38cddbe3bd 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -95,6 +95,7 @@ from open_webui.retrieval.web.kagi import search_kagi # Web search engines from open_webui.retrieval.web.main import SearchResult +from open_webui.retrieval.web.microsoft_web_iq import search_microsoft_web_iq from open_webui.retrieval.web.mojeek import search_mojeek from open_webui.retrieval.web.ollama import search_ollama_cloud from open_webui.retrieval.web.perplexity import search_perplexity @@ -317,6 +318,9 @@ RETRIEVAL_CONFIG_KEYS = { 'MINERU_API_URL': 'rag.mineru_api_url', 'MINERU_FILE_EXTENSIONS': 'rag.mineru_file_extensions', 'MINERU_PARAMS': 'rag.mineru_params', + 'MICROSOFT_WEB_IQ_API_BASE_URL': 'rag.web.search.microsoft_web_iq_api_base_url', + 'MICROSOFT_WEB_IQ_API_KEY': 'rag.web.search.microsoft_web_iq_api_key', + 'MICROSOFT_WEB_IQ_LANGUAGE': 'rag.web.search.microsoft_web_iq_language', 'MISTRAL_OCR_API_BASE_URL': 'rag.mistral_ocr_api_base_url', 'MISTRAL_OCR_API_KEY': 'rag.mistral_ocr_api_key', 'MOJEEK_SEARCH_API_KEY': 'rag.web.search.mojeek_search_api_key', @@ -724,6 +728,9 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)): 'PERPLEXITY_MODEL': config.PERPLEXITY_MODEL, 'PERPLEXITY_SEARCH_CONTEXT_USAGE': config.PERPLEXITY_SEARCH_CONTEXT_USAGE, 'PERPLEXITY_SEARCH_API_URL': config.PERPLEXITY_SEARCH_API_URL, + 'MICROSOFT_WEB_IQ_API_BASE_URL': config.MICROSOFT_WEB_IQ_API_BASE_URL, + 'MICROSOFT_WEB_IQ_API_KEY': config.MICROSOFT_WEB_IQ_API_KEY, + 'MICROSOFT_WEB_IQ_LANGUAGE': config.MICROSOFT_WEB_IQ_LANGUAGE, 'SOUGOU_API_SID': config.SOUGOU_API_SID, 'SOUGOU_API_SK': config.SOUGOU_API_SK, 'WEB_LOADER_ENGINE': config.WEB_LOADER_ENGINE, @@ -797,6 +804,9 @@ class WebConfig(BaseModel): PERPLEXITY_MODEL: str | None = None PERPLEXITY_SEARCH_CONTEXT_USAGE: str | None = None PERPLEXITY_SEARCH_API_URL: str | None = None + MICROSOFT_WEB_IQ_API_BASE_URL: str | None = None + MICROSOFT_WEB_IQ_API_KEY: str | None = None + MICROSOFT_WEB_IQ_LANGUAGE: str | None = None SOUGOU_API_SID: str | None = None SOUGOU_API_SK: str | None = None WEB_LOADER_ENGINE: str | None = None @@ -1298,6 +1308,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend config.PERPLEXITY_MODEL = form_data.web.PERPLEXITY_MODEL config.PERPLEXITY_SEARCH_CONTEXT_USAGE = form_data.web.PERPLEXITY_SEARCH_CONTEXT_USAGE config.PERPLEXITY_SEARCH_API_URL = form_data.web.PERPLEXITY_SEARCH_API_URL + config.MICROSOFT_WEB_IQ_API_BASE_URL = form_data.web.MICROSOFT_WEB_IQ_API_BASE_URL + config.MICROSOFT_WEB_IQ_API_KEY = form_data.web.MICROSOFT_WEB_IQ_API_KEY + config.MICROSOFT_WEB_IQ_LANGUAGE = form_data.web.MICROSOFT_WEB_IQ_LANGUAGE config.SOUGOU_API_SID = form_data.web.SOUGOU_API_SID config.SOUGOU_API_SK = form_data.web.SOUGOU_API_SK @@ -1326,6 +1339,8 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend config.LINKUP_API_KEY = form_data.web.LINKUP_API_KEY config.LINKUP_SEARCH_PARAMS = form_data.web.LINKUP_SEARCH_PARAMS + await config.save() + return { 'status': True, # RAG settings @@ -1438,6 +1453,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend 'PERPLEXITY_MODEL': config.PERPLEXITY_MODEL, 'PERPLEXITY_SEARCH_CONTEXT_USAGE': config.PERPLEXITY_SEARCH_CONTEXT_USAGE, 'PERPLEXITY_SEARCH_API_URL': config.PERPLEXITY_SEARCH_API_URL, + 'MICROSOFT_WEB_IQ_API_BASE_URL': config.MICROSOFT_WEB_IQ_API_BASE_URL, + 'MICROSOFT_WEB_IQ_API_KEY': config.MICROSOFT_WEB_IQ_API_KEY, + 'MICROSOFT_WEB_IQ_LANGUAGE': config.MICROSOFT_WEB_IQ_LANGUAGE, 'SOUGOU_API_SID': config.SOUGOU_API_SID, 'SOUGOU_API_SK': config.SOUGOU_API_SK, 'WEB_LOADER_ENGINE': config.WEB_LOADER_ENGINE, @@ -2394,6 +2412,20 @@ async def search_web(request: Request, engine: str, query: str, user=None) -> li model=config.PERPLEXITY_MODEL, search_context_usage=config.PERPLEXITY_SEARCH_CONTEXT_USAGE, ) + elif engine == 'microsoft_web_iq': + if config.MICROSOFT_WEB_IQ_API_KEY: + return await asyncio.to_thread( + search_microsoft_web_iq, + config.MICROSOFT_WEB_IQ_API_BASE_URL, + config.MICROSOFT_WEB_IQ_API_KEY, + query, + config.WEB_SEARCH_RESULT_COUNT, + config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config.MICROSOFT_WEB_IQ_LANGUAGE, + user, + ) + else: + raise Exception('No MICROSOFT_WEB_IQ_API_KEY found in environment variables') elif engine == 'sougou': if config.SOUGOU_API_SID and config.SOUGOU_API_SK: return await asyncio.to_thread( diff --git a/src/lib/components/admin/Settings/WebSearch.svelte b/src/lib/components/admin/Settings/WebSearch.svelte index 38938d6a72..fdd94da220 100644 --- a/src/lib/components/admin/Settings/WebSearch.svelte +++ b/src/lib/components/admin/Settings/WebSearch.svelte @@ -36,6 +36,7 @@ 'bing', 'exa', 'perplexity', + 'microsoft_web_iq', 'sougou', 'firecrawl', 'external', @@ -43,7 +44,7 @@ 'youcom', 'linkup' ]; - let webLoaderEngines = ['playwright', 'firecrawl', 'tavily', 'external']; + let webLoaderEngines = ['playwright', 'firecrawl', 'tavily', 'microsoft_web_iq', 'external']; let webConfig = null; @@ -714,6 +715,51 @@ + {:else if webConfig.WEB_SEARCH_ENGINE === 'microsoft_web_iq'} +
+
+
+ {$i18n.t('Microsoft Web IQ API Base URL')} +
+ +
+
+ +
+
+
+ +
+
+ {$i18n.t('Microsoft Web IQ API Key')} +
+ + +
+ +
+
+ {$i18n.t('Language')} +
+ + +
+
{:else if webConfig.WEB_SEARCH_ENGINE === 'sougou'}
@@ -1211,6 +1257,53 @@
{/if}
+ {:else if webConfig.WEB_LOADER_ENGINE === 'microsoft_web_iq'} +
+ {#if webConfig.WEB_SEARCH_ENGINE !== 'microsoft_web_iq'} +
+
+ {$i18n.t('Microsoft Web IQ API Base URL')} +
+ +
+
+ +
+
+
+ +
+
+ {$i18n.t('Microsoft Web IQ API Key')} +
+ + +
+ +
+
+ {$i18n.t('Language')} +
+ + +
+ {/if} +
{:else if webConfig.WEB_LOADER_ENGINE === 'external'}