From 121f2404eeffdb45d3599020713c58d0a84338b2 Mon Sep 17 00:00:00 2001 From: G30 <50341825+silentoplayz@users.noreply.github.com> Date: Mon, 10 Aug 2026 21:18:04 -0400 Subject: [PATCH] fix(retrieval): report why a URL could not be read instead of blaming the knowledge base (#28362) Fetching a URL and saving it were reported as one thing. Everything from reading the URL to writing the vector database sat inside a single try, whose handler blamed the knowledge base, so a page that could not be fetched, parsed or resolved was reported as a knowledge base error even though nothing had reached the knowledge base yet. Reading the URL now has its own handler that names the URL, and the knowledge base message is left to the step that actually touches it. When YouTube refused a transcript the reason was discarded earlier still: the loader caught the error, logged it, and returned an empty document list, so the empty result failed downstream and even the salvageable explanation was gone before a message was produced. The loader now raises YoutubeTranscriptError carrying a readable reason, mapped from the transcript library's own exception types. Blocked requests mention that a proxy can be configured, and disabled, age restricted, unavailable and missing language cases each say what actually happened. URLs that attach successfully are unaffected. --- .../open_webui/retrieval/loaders/youtube.py | 36 ++++++++++++++++--- backend/open_webui/routers/retrieval.py | 19 +++++++++- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/backend/open_webui/retrieval/loaders/youtube.py b/backend/open_webui/retrieval/loaders/youtube.py index 61659d7bc1..a1ee8d1128 100644 --- a/backend/open_webui/retrieval/loaders/youtube.py +++ b/backend/open_webui/retrieval/loaders/youtube.py @@ -18,6 +18,32 @@ ALLOWED_NETLOCS = { } +class YoutubeTranscriptError(Exception): + """A YouTube transcript could not be retrieved.""" + + +def _transcript_error_message(error: Exception, video_id: str) -> str: + name = type(error).__name__ + + if name in {'RequestBlocked', 'IpBlocked'}: + return ( + f'YouTube blocked the transcript request for {video_id} from this server. ' + 'This usually means the server address is rate limited or belongs to a cloud ' + 'provider. A proxy for these requests can be configured under Admin Settings, ' + 'Web Search, Youtube Proxy URL.' + ) + if name == 'TranscriptsDisabled': + return f'Transcripts are disabled for the YouTube video {video_id}.' + if name == 'AgeRestricted': + return f'The YouTube video {video_id} is age restricted, so its transcript cannot be retrieved.' + if name in {'VideoUnavailable', 'VideoUnplayable', 'InvalidVideoId'}: + return f'The YouTube video {video_id} is unavailable.' + if name == 'PoTokenRequired': + return f'YouTube requires additional verification to return the transcript for {video_id}.' + + return f'Could not retrieve a transcript for the YouTube video {video_id}.' + + def _parse_video_id(url: str) -> Optional[str]: """Parse a YouTube URL and return the video ID if valid, otherwise None.""" parsed_url = urlparse(url) @@ -98,8 +124,8 @@ class YoutubeLoader: try: transcript_list = transcript_api.list(self.video_id) except Exception as e: - log.warning(f'Loading YouTube transcript failed: {e}') - return [] + log.warning('Loading YouTube transcript failed: %s', e) + raise YoutubeTranscriptError(_transcript_error_message(e, self.video_id)) from e # Try each language in order of priority for lang in self.language: @@ -139,14 +165,16 @@ class YoutubeLoader: continue except Exception as e: log.info("Error finding transcript for language '%s'", lang) - raise e + raise YoutubeTranscriptError(_transcript_error_message(e, self.video_id)) from e # If we get here, all languages failed languages_tried = ', '.join(self.language) log.warning( f'No transcript found for any of the specified languages: {languages_tried}. Verify if the video has transcripts, add more languages if needed.' ) - raise NoTranscriptFound(self.video_id, self.language, list(transcript_list)) + raise YoutubeTranscriptError( + f'No transcript found for the YouTube video {self.video_id} in these languages: {languages_tried}.' + ) async def aload(self) -> Generator[Document, None, None]: """Asynchronously load YouTube transcripts into `Document` objects.""" diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index 0128a5eb9a..4c55589b90 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -66,7 +66,7 @@ from open_webui.models.knowledge import Knowledges from open_webui.models.config import Config # Document loaders -from open_webui.retrieval.loaders.youtube import YoutubeLoader +from open_webui.retrieval.loaders.youtube import YoutubeLoader, YoutubeTranscriptError from open_webui.retrieval.utils import ( build_loader_from_config, get_loader_config, @@ -2336,8 +2336,25 @@ async def process_web( user=Depends(get_verified_user), ): config = await get_retrieval_config() + try: content, docs = await get_content_from_url(request, form_data.url) + except HTTPException: + raise + except YoutubeTranscriptError as e: + log.warning('YouTube transcript unavailable for %s: %s', form_data.url, e) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(e), + ) + except Exception as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ERROR_MESSAGES.DEFAULT(e, f'Could not read content from {form_data.url}'), + ) + + try: log.debug('text_content: %s', content) if process: