From 6f8221df58b17334233ac6bfe069b8f837f677d6 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Mon, 29 Jun 2026 11:59:29 -0500 Subject: [PATCH] refac --- backend/open_webui/retrieval/web/utils.py | 43 ++++++++++------- src/lib/components/chat/MessageInput.svelte | 52 +++++++++++---------- 2 files changed, 55 insertions(+), 40 deletions(-) diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index cb3c32f238..0cb10eb9a6 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -326,8 +326,9 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): self.params = params or {} def lazy_load(self) -> Iterator[Document]: - try: - for url in self.web_paths: + for url in self.web_paths: + try: + self._sync_wait_for_rate_limit() doc = scrape_firecrawl_url( self.api_url, self.api_key, @@ -338,22 +339,32 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): ) if doc is not None: yield doc - except Exception as e: - if self.continue_on_failure: - log.warning(f'Error extracting content from URLs with Firecrawl: {e}') - else: - raise e + except Exception as e: + if self.continue_on_failure: + log.warning(f'Error extracting content from {url} with Firecrawl: {e}') + continue + raise async def alazy_load(self): - try: - docs = await run_in_threadpool(lambda: list(self.lazy_load())) - for doc in docs: - yield doc - except Exception as e: - if self.continue_on_failure: - log.warning(f'Error extracting content from URLs with Firecrawl: {e}') - else: - raise e + for url in self.web_paths: + try: + await self._wait_for_rate_limit() + doc = await run_in_threadpool( + scrape_firecrawl_url, + self.api_url, + self.api_key, + url, + verify_ssl=self.verify_ssl, + timeout=self.timeout, + params=self.params, + ) + if doc is not None: + yield doc + except Exception as e: + if self.continue_on_failure: + log.warning(f'Error extracting content from {url} with Firecrawl: {e}') + continue + raise class SafeTavilyLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): diff --git a/src/lib/components/chat/MessageInput.svelte b/src/lib/components/chat/MessageInput.svelte index 58a695b9a0..46cd0af3a7 100644 --- a/src/lib/components/chat/MessageInput.svelte +++ b/src/lib/components/chat/MessageInput.svelte @@ -1684,7 +1684,7 @@
-
+
@@ -1749,9 +1749,12 @@ {#if showWebSearchButton || showImageGenerationButton || showCodeInterpreterButton || showToolsButton || showSkillsButton || (toggleFilters && toggleFilters.length > 0)}
+ {/if} +
+ {#if showWebSearchButton || showImageGenerationButton || showCodeInterpreterButton || showToolsButton || showSkillsButton || (toggleFilters && toggleFilters.length > 0)} - {/if} + {/if} - {#if selectedModelIds.length === 1 && $models.find((m) => m.id === selectedModelIds[0])?.has_user_valves} -
- - - -
- {/if} + {#if selectedModelIds.length === 1 && $models.find((m) => m.id === selectedModelIds[0])?.has_user_valves} +
+ + + +
+ {/if} -
+
{#if (selectedToolIds ?? []).length > 0} {/each}
+