ewilliams28/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-08-13 01:02:25 -06:00
Code Issues Packages Projects Releases Wiki Activity

refac

Browse Source
This commit is contained in:
Timothy Jaeryang Baek
2026-06-29 01:52:07 -05:00
parent b7934e9182
commit ce4a323f43
6 changed files with 384 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/open_webui/config.py
+9
View File
@@ -1179,6 +1179,12 @@ PERPLEXITY_SEARCH_CONTEXT_USAGE = os.getenv('PERPLEXITY_SEARCH_CONTEXT_USAGE', '
PERPLEXITY_SEARCH_API_URL = os.getenv('PERPLEXITY_SEARCH_API_URL', 'https://api.perplexity.ai/search')
MICROSOFT_WEB_IQ_API_BASE_URL = os.getenv('MICROSOFT_WEB_IQ_API_BASE_URL', 'https://api.microsoft.ai/v3')
MICROSOFT_WEB_IQ_API_KEY = os.getenv('MICROSOFT_WEB_IQ_API_KEY', '')
MICROSOFT_WEB_IQ_LANGUAGE = os.getenv('MICROSOFT_WEB_IQ_LANGUAGE', 'en')
SOUGOU_API_SID = os.getenv('SOUGOU_API_SID', '')
SOUGOU_API_SK = os.getenv('SOUGOU_API_SK', '')
@@ -2805,6 +2811,9 @@ DEFAULT_CONFIG = {
'rag.web.search.perplexity_model': PERPLEXITY_MODEL,
'rag.web.search.perplexity_search_context_usage': PERPLEXITY_SEARCH_CONTEXT_USAGE,
'rag.web.search.perplexity_search_api_url': PERPLEXITY_SEARCH_API_URL,
'rag.web.search.microsoft_web_iq_api_base_url': MICROSOFT_WEB_IQ_API_BASE_URL,
'rag.web.search.microsoft_web_iq_api_key': MICROSOFT_WEB_IQ_API_KEY,
'rag.web.search.microsoft_web_iq_language': MICROSOFT_WEB_IQ_LANGUAGE,
'rag.web.search.sougou_api_sid': SOUGOU_API_SID,
'rag.web.search.sougou_api_sk': SOUGOU_API_SK,
'rag.web.search.tavily_api_key': TAVILY_API_KEY,
backend/open_webui/retrieval/loaders/microsoft_web_iq.py
+110
View File
@@ -0,0 +1,110 @@
import logging
import time
from collections.abc import Iterator
from typing import Any
from urllib.parse import urlparse
import requests
from langchain_core.document_loaders import BaseLoader
from langchain_core.documents import Document
log = logging.getLogger(__name__)
DEFAULT_MICROSOFT_WEB_IQ_API_BASE_URL = 'https://api.microsoft.ai/v3'
MICROSOFT_BROWSE_RETRY_STATUS_CODES = {202, 429, 500, 502, 503, 504}
MICROSOFT_BROWSE_MAX_RETRIES = 2
class MicrosoftWebIQLoader(BaseLoader):
def __init__(
self,
urls: str | list[str],
api_base_url: str,
api_key: str,
language: str = 'en',
verify_ssl: bool = True,
timeout: Any = None,
continue_on_failure: bool = True,
) -> None:
self.urls = urls if isinstance(urls, list) else [urls]
self.api_base_url = (api_base_url or DEFAULT_MICROSOFT_WEB_IQ_API_BASE_URL).rstrip('/')
self.api_key = api_key
self.language = language
self.verify_ssl = verify_ssl
self.timeout = timeout
self.continue_on_failure = continue_on_failure
def lazy_load(self) -> Iterator[Document]:
for url in self.urls:
try:
doc = self._browse_url(url)
if doc is not None:
yield doc
except Exception as e:
if self.continue_on_failure:
log.warning(f'Error browsing {url} with Microsoft Web IQ: {e}')
else:
raise e
def _browse_url(self, url: str) -> Document | None:
headers = {
'host': urlparse(self.api_base_url).netloc or 'api.microsoft.ai',
'x-apikey': self.api_key,
'content-type': 'application/json',
}
payload = {
'url': url,
'contentFormat': 'markdown',
'liveCrawl': 'fallback',
'renderDynamicPages': True,
'language': self.language,
}
try:
request_timeout = float(self.timeout)
except (TypeError, ValueError):
request_timeout = 60
request_timeout = request_timeout if request_timeout > 0 else 60
data: dict[str, Any] = {}
for attempt in range(MICROSOFT_BROWSE_MAX_RETRIES + 1):
response = requests.post(
f'{self.api_base_url}/browse',
json=payload,
headers=headers,
timeout=request_timeout,
verify=self.verify_ssl,
)
if response.status_code in MICROSOFT_BROWSE_RETRY_STATUS_CODES and attempt < MICROSOFT_BROWSE_MAX_RETRIES:
try:
body = response.json()
except Exception:
body = {}
retry_after = body.get('retryAfter') if isinstance(body, dict) else None
retry_after = retry_after or response.headers.get('Retry-After')
try:
delay = min(10.0, max(0.0, float(str(retry_after).rstrip('s'))))
except (TypeError, ValueError):
delay = min(8.0, float(2**attempt))
log.warning(
'Microsoft Browse %s returned HTTP %s; retrying in %.1fs',
url,
response.status_code,
delay,
)
time.sleep(delay)
continue
response.raise_for_status()
data = response.json()
break
content = data.get('content') or ''
if not isinstance(content, str) or not content.strip():
return None
metadata = {'source': data.get('url') or url}
if data.get('title'):
metadata['title'] = data['title']
return Document(page_content=content, metadata=metadata)
backend/open_webui/retrieval/web/microsoft_web_iq.py
+60
View File
@@ -0,0 +1,60 @@
from __future__ import annotations
import logging
from urllib.parse import urlparse
import requests
from open_webui.retrieval.web.main import SearchResult, get_filtered_results
from open_webui.utils.headers import include_user_info_headers
log = logging.getLogger(__name__)
DEFAULT_MICROSOFT_WEB_IQ_API_BASE_URL = 'https://api.microsoft.ai/v3'
def search_microsoft_web_iq(
api_base_url: str,
api_key: str,
query: str,
count: int,
filter_list: list[str | None] | None = None,
language: str = 'en',
user=None,
) -> list[SearchResult]:
try:
api_base_url = (api_base_url or DEFAULT_MICROSOFT_WEB_IQ_API_BASE_URL).rstrip('/')
headers = {
'host': urlparse(api_base_url).netloc or 'api.microsoft.ai',
'x-apikey': api_key,
'content-type': 'application/json',
}
if user is not None:
headers = include_user_info_headers(headers, user)
response = requests.post(
f'{api_base_url}/search/web',
json={
'query': query,
'maxResults': count,
'language': language,
'contentFormat': 'passage',
},
headers=headers,
)
response.raise_for_status()
results = response.json().get('webResults', [])
if filter_list:
results = get_filtered_results(results, filter_list)
return [
SearchResult(
link=result['url'],
title=result.get('title'),
snippet=result.get('content'),
)
for result in results
]
except Exception as e:
log.error(f'Error searching with Microsoft Web IQ API: {e}')
return []
backend/open_webui/retrieval/web/utils.py
+79 -1
View File
@@ -36,6 +36,9 @@ from open_webui.config import (
FIRECRAWL_API_BASE_URL,
FIRECRAWL_API_KEY,
FIRECRAWL_TIMEOUT,
MICROSOFT_WEB_IQ_API_BASE_URL,
MICROSOFT_WEB_IQ_API_KEY,
MICROSOFT_WEB_IQ_LANGUAGE,
PLAYWRIGHT_TIMEOUT,
PLAYWRIGHT_WS_URL,
TAVILY_API_KEY,
@@ -52,6 +55,7 @@ from open_webui.env import (
USER_AGENT,
)
from open_webui.retrieval.loaders.external_web import ExternalWebLoader
from open_webui.retrieval.loaders.microsoft_web_iq import MicrosoftWebIQLoader
from open_webui.retrieval.loaders.tavily import TavilyLoader
from open_webui.retrieval.web.firecrawl import scrape_firecrawl_url
from open_webui.utils.misc import is_host_allowed
@@ -356,6 +360,7 @@ class SafeTavilyLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
def __init__(
self,
web_paths: Union[str, List[str]],
api_base_url: str,
api_key: str,
extract_depth: Literal['basic', 'advanced'] = 'basic',
continue_on_failure: bool = True,
@@ -389,6 +394,7 @@ class SafeTavilyLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
# Store parameters for creating TavilyLoader instances
self.web_paths = web_paths if isinstance(web_paths, list) else [web_paths]
self.api_base_url = api_base_url
self.api_key = api_key
self.extract_depth = extract_depth
self.continue_on_failure = continue_on_failure
@@ -464,6 +470,67 @@ class SafeTavilyLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
raise e
class SafeMicrosoftWebIQLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
def __init__(
self,
web_paths: Union[str, List[str]],
api_key: str,
language: str = 'en',
verify_ssl: bool = True,
trust_env: bool = False,
requests_per_second: Optional[float] = None,
continue_on_failure: bool = True,
timeout: Optional[int] = None,
):
self.web_paths = web_paths if isinstance(web_paths, list) else [web_paths]
self.api_key = api_key
self.language = language
self.verify_ssl = verify_ssl
self.trust_env = trust_env
self.requests_per_second = requests_per_second
self.last_request_time = None
self.continue_on_failure = continue_on_failure
self.timeout = timeout
def lazy_load(self) -> Iterator[Document]:
valid_urls = []
for url in self.web_paths:
try:
self._safe_process_url_sync(url)
valid_urls.append(url)
except Exception as e:
log.warning(f'SSL verification failed for {url}: {str(e)}')
if not self.continue_on_failure:
raise e
if not valid_urls:
if self.continue_on_failure:
log.warning('No valid URLs to process after SSL verification')
return
raise ValueError('No valid URLs to process after SSL verification')
loader = MicrosoftWebIQLoader(
urls=valid_urls,
api_base_url=self.api_base_url,
api_key=self.api_key,
language=self.language,
verify_ssl=self.verify_ssl,
timeout=self.timeout,
continue_on_failure=self.continue_on_failure,
)
yield from loader.lazy_load()
async def alazy_load(self) -> AsyncIterator[Document]:
try:
docs = await run_in_threadpool(lambda: list(self.lazy_load()))
for doc in docs:
yield doc
except Exception as e:
if self.continue_on_failure:
log.warning(f'Error browsing URLs with Microsoft Web IQ: {e}')
else:
raise e
class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessingMixin):
"""Load HTML pages safely with Playwright, supporting SSL verification, rate limiting, and remote browser connection.
@@ -813,6 +880,17 @@ def get_web_loader(
web_loader_args['api_key'] = TAVILY_API_KEY
web_loader_args['extract_depth'] = TAVILY_EXTRACT_DEPTH
if WEB_LOADER_ENGINE == 'microsoft_web_iq':
WebLoaderClass = SafeMicrosoftWebIQLoader
web_loader_args['api_base_url'] = MICROSOFT_WEB_IQ_API_BASE_URL
web_loader_args['api_key'] = MICROSOFT_WEB_IQ_API_KEY
web_loader_args['language'] = MICROSOFT_WEB_IQ_LANGUAGE
if WEB_LOADER_TIMEOUT:
try:
web_loader_args['timeout'] = int(WEB_LOADER_TIMEOUT)
except ValueError:
pass
if WEB_LOADER_ENGINE == 'external':
WebLoaderClass = ExternalWebLoader
web_loader_args['external_url'] = EXTERNAL_WEB_LOADER_URL
@@ -831,5 +909,5 @@ def get_web_loader(
else:
raise ValueError(
f'Invalid WEB_LOADER_ENGINE: {WEB_LOADER_ENGINE}. '
"Please set it to 'safe_web', 'playwright', 'firecrawl', or 'tavily'."
"Please set it to 'safe_web', 'playwright', 'firecrawl', 'tavily', 'external', or 'microsoft_web_iq'."
)
backend/open_webui/routers/retrieval.py
+32
View File
@@ -95,6 +95,7 @@ from open_webui.retrieval.web.kagi import search_kagi
# Web search engines
from open_webui.retrieval.web.main import SearchResult
from open_webui.retrieval.web.microsoft_web_iq import search_microsoft_web_iq
from open_webui.retrieval.web.mojeek import search_mojeek
from open_webui.retrieval.web.ollama import search_ollama_cloud
from open_webui.retrieval.web.perplexity import search_perplexity
@@ -317,6 +318,9 @@ RETRIEVAL_CONFIG_KEYS = {
'MINERU_API_URL': 'rag.mineru_api_url',
'MINERU_FILE_EXTENSIONS': 'rag.mineru_file_extensions',
'MINERU_PARAMS': 'rag.mineru_params',
'MICROSOFT_WEB_IQ_API_BASE_URL': 'rag.web.search.microsoft_web_iq_api_base_url',
'MICROSOFT_WEB_IQ_API_KEY': 'rag.web.search.microsoft_web_iq_api_key',
'MICROSOFT_WEB_IQ_LANGUAGE': 'rag.web.search.microsoft_web_iq_language',
'MISTRAL_OCR_API_BASE_URL': 'rag.mistral_ocr_api_base_url',
'MISTRAL_OCR_API_KEY': 'rag.mistral_ocr_api_key',
'MOJEEK_SEARCH_API_KEY': 'rag.web.search.mojeek_search_api_key',
@@ -724,6 +728,9 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
'PERPLEXITY_MODEL': config.PERPLEXITY_MODEL,
'PERPLEXITY_SEARCH_CONTEXT_USAGE': config.PERPLEXITY_SEARCH_CONTEXT_USAGE,
'PERPLEXITY_SEARCH_API_URL': config.PERPLEXITY_SEARCH_API_URL,
'MICROSOFT_WEB_IQ_API_BASE_URL': config.MICROSOFT_WEB_IQ_API_BASE_URL,
'MICROSOFT_WEB_IQ_API_KEY': config.MICROSOFT_WEB_IQ_API_KEY,
'MICROSOFT_WEB_IQ_LANGUAGE': config.MICROSOFT_WEB_IQ_LANGUAGE,
'SOUGOU_API_SID': config.SOUGOU_API_SID,
'SOUGOU_API_SK': config.SOUGOU_API_SK,
'WEB_LOADER_ENGINE': config.WEB_LOADER_ENGINE,
@@ -797,6 +804,9 @@ class WebConfig(BaseModel):
PERPLEXITY_MODEL: str | None = None
PERPLEXITY_SEARCH_CONTEXT_USAGE: str | None = None
PERPLEXITY_SEARCH_API_URL: str | None = None
MICROSOFT_WEB_IQ_API_BASE_URL: str | None = None
MICROSOFT_WEB_IQ_API_KEY: str | None = None
MICROSOFT_WEB_IQ_LANGUAGE: str | None = None
SOUGOU_API_SID: str | None = None
SOUGOU_API_SK: str | None = None
WEB_LOADER_ENGINE: str | None = None
@@ -1298,6 +1308,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
config.PERPLEXITY_MODEL = form_data.web.PERPLEXITY_MODEL
config.PERPLEXITY_SEARCH_CONTEXT_USAGE = form_data.web.PERPLEXITY_SEARCH_CONTEXT_USAGE
config.PERPLEXITY_SEARCH_API_URL = form_data.web.PERPLEXITY_SEARCH_API_URL
config.MICROSOFT_WEB_IQ_API_BASE_URL = form_data.web.MICROSOFT_WEB_IQ_API_BASE_URL
config.MICROSOFT_WEB_IQ_API_KEY = form_data.web.MICROSOFT_WEB_IQ_API_KEY
config.MICROSOFT_WEB_IQ_LANGUAGE = form_data.web.MICROSOFT_WEB_IQ_LANGUAGE
config.SOUGOU_API_SID = form_data.web.SOUGOU_API_SID
config.SOUGOU_API_SK = form_data.web.SOUGOU_API_SK
@@ -1326,6 +1339,8 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
config.LINKUP_API_KEY = form_data.web.LINKUP_API_KEY
config.LINKUP_SEARCH_PARAMS = form_data.web.LINKUP_SEARCH_PARAMS
await config.save()
return {
'status': True,
# RAG settings
@@ -1438,6 +1453,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
'PERPLEXITY_MODEL': config.PERPLEXITY_MODEL,
'PERPLEXITY_SEARCH_CONTEXT_USAGE': config.PERPLEXITY_SEARCH_CONTEXT_USAGE,
'PERPLEXITY_SEARCH_API_URL': config.PERPLEXITY_SEARCH_API_URL,
'MICROSOFT_WEB_IQ_API_BASE_URL': config.MICROSOFT_WEB_IQ_API_BASE_URL,
'MICROSOFT_WEB_IQ_API_KEY': config.MICROSOFT_WEB_IQ_API_KEY,
'MICROSOFT_WEB_IQ_LANGUAGE': config.MICROSOFT_WEB_IQ_LANGUAGE,
'SOUGOU_API_SID': config.SOUGOU_API_SID,
'SOUGOU_API_SK': config.SOUGOU_API_SK,
'WEB_LOADER_ENGINE': config.WEB_LOADER_ENGINE,
@@ -2394,6 +2412,20 @@ async def search_web(request: Request, engine: str, query: str, user=None) -> li
model=config.PERPLEXITY_MODEL,
search_context_usage=config.PERPLEXITY_SEARCH_CONTEXT_USAGE,
)
elif engine == 'microsoft_web_iq':
if config.MICROSOFT_WEB_IQ_API_KEY:
return await asyncio.to_thread(
search_microsoft_web_iq,
config.MICROSOFT_WEB_IQ_API_BASE_URL,
config.MICROSOFT_WEB_IQ_API_KEY,
query,
config.WEB_SEARCH_RESULT_COUNT,
config.WEB_SEARCH_DOMAIN_FILTER_LIST,
config.MICROSOFT_WEB_IQ_LANGUAGE,
user,
)
else:
raise Exception('No MICROSOFT_WEB_IQ_API_KEY found in environment variables')
elif engine == 'sougou':
if config.SOUGOU_API_SID and config.SOUGOU_API_SK:
return await asyncio.to_thread(
src/lib/components/admin/Settings/WebSearch.svelte
+94 -1
View File
@@ -36,6 +36,7 @@
'bing',
'exa',
'perplexity',
'microsoft_web_iq',
'sougou',
'firecrawl',
'external',
@@ -43,7 +44,7 @@
'youcom',
'linkup'
];
let webLoaderEngines = ['playwright', 'firecrawl', 'tavily', 'external'];
let webLoaderEngines = ['playwright', 'firecrawl', 'tavily', 'microsoft_web_iq', 'external'];
let webConfig = null;
@@ -714,6 +715,51 @@
</select>
</div>
</div>
{:else if webConfig.WEB_SEARCH_ENGINE === 'microsoft_web_iq'}
<div class="mb-2.5 flex w-full flex-col">
<div>
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Microsoft Web IQ API Base URL')}
</div>
<div class="flex w-full">
<div class="flex-1">
<input
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
type="text"
placeholder={$i18n.t('Enter Microsoft Web IQ API Base URL')}
bind:value={webConfig.MICROSOFT_WEB_IQ_API_BASE_URL}
autocomplete="off"
/>
</div>
</div>
</div>
<div class="mt-2">
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Microsoft Web IQ API Key')}
</div>
<SensitiveInput
placeholder={$i18n.t('Enter Microsoft Web IQ API Key')}
bind:value={webConfig.MICROSOFT_WEB_IQ_API_KEY}
/>
</div>
<div class="mt-2">
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Language')}
</div>
<input
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
type="text"
placeholder={$i18n.t('Enter language')}
bind:value={webConfig.MICROSOFT_WEB_IQ_LANGUAGE}
autocomplete="off"
/>
</div>
</div>
{:else if webConfig.WEB_SEARCH_ENGINE === 'sougou'}
<div class="mb-2.5 flex w-full flex-col">
<div>
@@ -1211,6 +1257,53 @@
</div>
{/if}
</div>
{:else if webConfig.WEB_LOADER_ENGINE === 'microsoft_web_iq'}
<div class="mb-2.5 flex w-full flex-col">
{#if webConfig.WEB_SEARCH_ENGINE !== 'microsoft_web_iq'}
<div>
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Microsoft Web IQ API Base URL')}
</div>
<div class="flex w-full">
<div class="flex-1">
<input
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
type="text"
placeholder={$i18n.t('Enter Microsoft Web IQ API Base URL')}
bind:value={webConfig.MICROSOFT_WEB_IQ_API_BASE_URL}
autocomplete="off"
/>
</div>
</div>
</div>
<div class="mt-2">
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Microsoft Web IQ API Key')}
</div>
<SensitiveInput
placeholder={$i18n.t('Enter Microsoft Web IQ API Key')}
bind:value={webConfig.MICROSOFT_WEB_IQ_API_KEY}
/>
</div>
<div class="mt-2">
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Language')}
</div>
<input
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
type="text"
placeholder={$i18n.t('Enter language')}
bind:value={webConfig.MICROSOFT_WEB_IQ_LANGUAGE}
autocomplete="off"
/>
</div>
{/if}
</div>
{:else if webConfig.WEB_LOADER_ENGINE === 'external'}
<div class="mb-2.5 flex w-full flex-col">
<div>