From 94b1b7e6b66a1fe5de0a57d7e4ed62adecf156d2 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Sun, 26 Jul 2026 23:35:13 +0200 Subject: [PATCH] fix: close Playwright pages and browser on failure in SafePlaywrightURLLoader (#27526) `SafePlaywrightURLLoader` opened a new Playwright page for every URL and never closed it, and it only closed the browser after the URL loop finished normally. Pages therefore piled up for the whole batch, and any early exit (a raised error with `continue_on_failure=False`, or the caller abandoning/cancelling the generator mid-search) skipped `browser.close()` entirely. With `PLAYWRIGHT_WS_URL` pointing at a remote Playwright server this leaks sessions on that server: navigation and route timeouts on slow or bot-protected pages leave pages and browser connections open until the server is restarted, which degrades every later web search. Both `lazy_load()` and `alazy_load()` now scope the page to the per-URL loop body and the browser to the whole loop using their context managers, so each page is closed as soon as its URL is done and the browser is closed on success, on failure, and on cancellation. Closing a page also disposes the context implicitly created by `new_page()`. Exception handling is unchanged: a close error raised while `continue_on_failure` is set is still caught, logged, and the loop continues. Fixes #25880 --- backend/open_webui/retrieval/web/utils.py | 68 +++++++++++------------ 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index 22fe7d98ae..4222aa71be 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -669,24 +669,24 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing else: browser = p.chromium.launch(headless=self.headless, proxy=self.proxy) - for url in self.urls: - try: - self._safe_process_url_sync(url) - page = browser.new_page() - page.route('**/*', self._intercept_navigation_sync) - response = page.goto(url, timeout=self.playwright_timeout) - if response is None: - raise ValueError(f'page.goto() returned None for url {url}') + with browser: + for url in self.urls: + try: + self._safe_process_url_sync(url) + with browser.new_page() as page: + page.route('**/*', self._intercept_navigation_sync) + response = page.goto(url, timeout=self.playwright_timeout) + if response is None: + raise ValueError(f'page.goto() returned None for url {url}') - text = self.evaluator.evaluate(page, browser, response) - metadata = {'source': url} - yield Document(page_content=text, metadata=metadata) - except Exception as e: - if self.continue_on_failure: - log.exception(f'Error loading {url}: {e}') - continue - raise e - browser.close() + text = self.evaluator.evaluate(page, browser, response) + metadata = {'source': url} + yield Document(page_content=text, metadata=metadata) + except Exception as e: + if self.continue_on_failure: + log.exception(f'Error loading {url}: {e}') + continue + raise e async def alazy_load(self) -> AsyncIterator[Document]: """Safely load URLs asynchronously with support for remote browser.""" @@ -699,24 +699,24 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing else: browser = await p.chromium.launch(headless=self.headless, proxy=self.proxy) - for url in self.urls: - try: - await self._safe_process_url(url) - page = await browser.new_page() - await page.route('**/*', self._intercept_navigation) - response = await page.goto(url, timeout=self.playwright_timeout) - if response is None: - raise ValueError(f'page.goto() returned None for url {url}') + async with browser: + for url in self.urls: + try: + await self._safe_process_url(url) + async with await browser.new_page() as page: + await page.route('**/*', self._intercept_navigation) + response = await page.goto(url, timeout=self.playwright_timeout) + if response is None: + raise ValueError(f'page.goto() returned None for url {url}') - text = await self.evaluator.evaluate_async(page, browser, response) - metadata = {'source': url} - yield Document(page_content=text, metadata=metadata) - except Exception as e: - if self.continue_on_failure: - log.exception(f'Error loading {url}: {e}') - continue - raise e - await browser.close() + text = await self.evaluator.evaluate_async(page, browser, response) + metadata = {'source': url} + yield Document(page_content=text, metadata=metadata) + except Exception as e: + if self.continue_on_failure: + log.exception(f'Error loading {url}: {e}') + continue + raise e class SafeWebBaseLoader(WebBaseLoader):