ewilliams28/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-08-13 01:02:25 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix: close Playwright pages and browser on failure in SafePlaywrightURLLoader (#27526)

Browse Source 
`SafePlaywrightURLLoader` opened a new Playwright page for every URL and never closed it, and it only closed the browser after the URL loop finished normally. Pages therefore piled up for the whole batch, and any early exit (a raised error with `continue_on_failure=False`, or the caller abandoning/cancelling the generator mid-search) skipped `browser.close()` entirely.

With `PLAYWRIGHT_WS_URL` pointing at a remote Playwright server this leaks sessions on that server: navigation and route timeouts on slow or bot-protected pages leave pages and browser connections open until the server is restarted, which degrades every later web search.

Both `lazy_load()` and `alazy_load()` now scope the page to the per-URL loop body and the browser to the whole loop using their context managers, so each page is closed as soon as its URL is done and the browser is closed on success, on failure, and on cancellation. Closing a page also disposes the context implicitly created by `new_page()`. Exception handling is unchanged: a close error raised while `continue_on_failure` is set is still caught, logged, and the loop continues.

Fixes #25880
This commit is contained in:
Classic298
2026-07-26 23:35:13 +02:00
committed by GitHub
parent 225e238856
commit 94b1b7e6b6
1 changed files with 34 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/open_webui/retrieval/web/utils.py
+34 -34
View File
@@ -669,24 +669,24 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing
else:
browser = p.chromium.launch(headless=self.headless, proxy=self.proxy)
for url in self.urls:
try:
self._safe_process_url_sync(url)
page = browser.new_page()
page.route('**/*', self._intercept_navigation_sync)
response = page.goto(url, timeout=self.playwright_timeout)
if response is None:
raise ValueError(f'page.goto() returned None for url {url}')
with browser:
for url in self.urls:
try:
self._safe_process_url_sync(url)
with browser.new_page() as page:
page.route('**/*', self._intercept_navigation_sync)
response = page.goto(url, timeout=self.playwright_timeout)
if response is None:
raise ValueError(f'page.goto() returned None for url {url}')
text = self.evaluator.evaluate(page, browser, response)
metadata = {'source': url}
yield Document(page_content=text, metadata=metadata)
except Exception as e:
if self.continue_on_failure:
log.exception(f'Error loading {url}: {e}')
continue
raise e
browser.close()
text = self.evaluator.evaluate(page, browser, response)
metadata = {'source': url}
yield Document(page_content=text, metadata=metadata)
except Exception as e:
if self.continue_on_failure:
log.exception(f'Error loading {url}: {e}')
continue
raise e
async def alazy_load(self) -> AsyncIterator[Document]:
"""Safely load URLs asynchronously with support for remote browser."""
@@ -699,24 +699,24 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing
else:
browser = await p.chromium.launch(headless=self.headless, proxy=self.proxy)
for url in self.urls:
try:
await self._safe_process_url(url)
page = await browser.new_page()
await page.route('**/*', self._intercept_navigation)
response = await page.goto(url, timeout=self.playwright_timeout)
if response is None:
raise ValueError(f'page.goto() returned None for url {url}')
async with browser:
for url in self.urls:
try:
await self._safe_process_url(url)
async with await browser.new_page() as page:
await page.route('**/*', self._intercept_navigation)
response = await page.goto(url, timeout=self.playwright_timeout)
if response is None:
raise ValueError(f'page.goto() returned None for url {url}')
text = await self.evaluator.evaluate_async(page, browser, response)
metadata = {'source': url}
yield Document(page_content=text, metadata=metadata)
except Exception as e:
if self.continue_on_failure:
log.exception(f'Error loading {url}: {e}')
continue
raise e
await browser.close()
text = await self.evaluator.evaluate_async(page, browser, response)
metadata = {'source': url}
yield Document(page_content=text, metadata=metadata)
except Exception as e:
if self.continue_on_failure:
log.exception(f'Error loading {url}: {e}')
continue
raise e
class SafeWebBaseLoader(WebBaseLoader):