fix(retrieval): report why a URL could not be read instead of blaming the knowledge base (#28362)

Fetching a URL and saving it were reported as one thing. Everything from
reading the URL to writing the vector database sat inside a single try,
whose handler blamed the knowledge base, so a page that could not be
fetched, parsed or resolved was reported as a knowledge base error even
though nothing had reached the knowledge base yet. Reading the URL now has
its own handler that names the URL, and the knowledge base message is left
to the step that actually touches it.

When YouTube refused a transcript the reason was discarded earlier still:
the loader caught the error, logged it, and returned an empty document
list, so the empty result failed downstream and even the salvageable
explanation was gone before a message was produced. The loader now raises
YoutubeTranscriptError carrying a readable reason, mapped from the
transcript library's own exception types. Blocked requests mention that a
proxy can be configured, and disabled, age restricted, unavailable and
missing language cases each say what actually happened.

URLs that attach successfully are unaffected.

backend/open_webui/retrieval/loaders/youtube.py

@@ -18,6 +18,32 @@ ALLOWED_NETLOCS = {
 }
 
 
+class YoutubeTranscriptError(Exception):
+    """A YouTube transcript could not be retrieved."""
+
+
+def _transcript_error_message(error: Exception, video_id: str) -> str:
+    name = type(error).__name__
+
+    if name in {'RequestBlocked', 'IpBlocked'}:
+        return (
+            f'YouTube blocked the transcript request for {video_id} from this server. '
+            'This usually means the server address is rate limited or belongs to a cloud '
+            'provider. A proxy for these requests can be configured under Admin Settings, '
+            'Web Search, Youtube Proxy URL.'
+        )
+    if name == 'TranscriptsDisabled':
+        return f'Transcripts are disabled for the YouTube video {video_id}.'
+    if name == 'AgeRestricted':
+        return f'The YouTube video {video_id} is age restricted, so its transcript cannot be retrieved.'
+    if name in {'VideoUnavailable', 'VideoUnplayable', 'InvalidVideoId'}:
+        return f'The YouTube video {video_id} is unavailable.'
+    if name == 'PoTokenRequired':
+        return f'YouTube requires additional verification to return the transcript for {video_id}.'
+
+    return f'Could not retrieve a transcript for the YouTube video {video_id}.'
+
+
 def _parse_video_id(url: str) -> Optional[str]:
     """Parse a YouTube URL and return the video ID if valid, otherwise None."""
     parsed_url = urlparse(url)
@@ -98,8 +124,8 @@ class YoutubeLoader:
         try:
             transcript_list = transcript_api.list(self.video_id)
         except Exception as e:
-            log.warning(f'Loading YouTube transcript failed: {e}')
-            return []
+            log.warning('Loading YouTube transcript failed: %s', e)
+            raise YoutubeTranscriptError(_transcript_error_message(e, self.video_id)) from e
 
         # Try each language in order of priority
         for lang in self.language:
@@ -139,14 +165,16 @@ class YoutubeLoader:
                 continue
             except Exception as e:
                 log.info("Error finding transcript for language '%s'", lang)
-                raise e
+                raise YoutubeTranscriptError(_transcript_error_message(e, self.video_id)) from e
 
         # If we get here, all languages failed
         languages_tried = ', '.join(self.language)
         log.warning(
             f'No transcript found for any of the specified languages: {languages_tried}. Verify if the video has transcripts, add more languages if needed.'
         )
-        raise NoTranscriptFound(self.video_id, self.language, list(transcript_list))
+        raise YoutubeTranscriptError(
+            f'No transcript found for the YouTube video {self.video_id} in these languages: {languages_tried}.'
+        )
 
     async def aload(self) -> Generator[Document, None, None]:
         """Asynchronously load YouTube transcripts into `Document` objects."""

backend/open_webui/routers/retrieval.py

@@ -66,7 +66,7 @@ from open_webui.models.knowledge import Knowledges
 from open_webui.models.config import Config
 
 # Document loaders
-from open_webui.retrieval.loaders.youtube import YoutubeLoader
+from open_webui.retrieval.loaders.youtube import YoutubeLoader, YoutubeTranscriptError
 from open_webui.retrieval.utils import (
     build_loader_from_config,
     get_loader_config,
@@ -2336,8 +2336,25 @@ async def process_web(
     user=Depends(get_verified_user),
 ):
     config = await get_retrieval_config()
+
     try:
         content, docs = await get_content_from_url(request, form_data.url)
+    except HTTPException:
+        raise
+    except YoutubeTranscriptError as e:
+        log.warning('YouTube transcript unavailable for %s: %s', form_data.url, e)
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=str(e),
+        )
+    except Exception as e:
+        log.exception(e)
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.DEFAULT(e, f'Could not read content from {form_data.url}'),
+        )
+
+    try:
         log.debug('text_content: %s', content)
 
         if process: