diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index 22fe7d98ae..4222aa71be 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -669,24 +669,24 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing else: browser = p.chromium.launch(headless=self.headless, proxy=self.proxy) - for url in self.urls: - try: - self._safe_process_url_sync(url) - page = browser.new_page() - page.route('**/*', self._intercept_navigation_sync) - response = page.goto(url, timeout=self.playwright_timeout) - if response is None: - raise ValueError(f'page.goto() returned None for url {url}') + with browser: + for url in self.urls: + try: + self._safe_process_url_sync(url) + with browser.new_page() as page: + page.route('**/*', self._intercept_navigation_sync) + response = page.goto(url, timeout=self.playwright_timeout) + if response is None: + raise ValueError(f'page.goto() returned None for url {url}') - text = self.evaluator.evaluate(page, browser, response) - metadata = {'source': url} - yield Document(page_content=text, metadata=metadata) - except Exception as e: - if self.continue_on_failure: - log.exception(f'Error loading {url}: {e}') - continue - raise e - browser.close() + text = self.evaluator.evaluate(page, browser, response) + metadata = {'source': url} + yield Document(page_content=text, metadata=metadata) + except Exception as e: + if self.continue_on_failure: + log.exception(f'Error loading {url}: {e}') + continue + raise e async def alazy_load(self) -> AsyncIterator[Document]: """Safely load URLs asynchronously with support for remote browser.""" @@ -699,24 +699,24 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing else: browser = await p.chromium.launch(headless=self.headless, proxy=self.proxy) - for url in self.urls: - try: - await self._safe_process_url(url) - page = await browser.new_page() - await page.route('**/*', self._intercept_navigation) - response = await page.goto(url, timeout=self.playwright_timeout) - if response is None: - raise ValueError(f'page.goto() returned None for url {url}') + async with browser: + for url in self.urls: + try: + await self._safe_process_url(url) + async with await browser.new_page() as page: + await page.route('**/*', self._intercept_navigation) + response = await page.goto(url, timeout=self.playwright_timeout) + if response is None: + raise ValueError(f'page.goto() returned None for url {url}') - text = await self.evaluator.evaluate_async(page, browser, response) - metadata = {'source': url} - yield Document(page_content=text, metadata=metadata) - except Exception as e: - if self.continue_on_failure: - log.exception(f'Error loading {url}: {e}') - continue - raise e - await browser.close() + text = await self.evaluator.evaluate_async(page, browser, response) + metadata = {'source': url} + yield Document(page_content=text, metadata=metadata) + except Exception as e: + if self.continue_on_failure: + log.exception(f'Error loading {url}: {e}') + continue + raise e class SafeWebBaseLoader(WebBaseLoader):