diff --git a/backend/open_webui/retrieval/loaders/main.py b/backend/open_webui/retrieval/loaders/main.py index f86f6fc2dc..218369fd93 100644 --- a/backend/open_webui/retrieval/loaders/main.py +++ b/backend/open_webui/retrieval/loaders/main.py @@ -303,13 +303,20 @@ class Loader: try: raw.decode('utf-8') return 'utf-8' - except UnicodeDecodeError: - pass + except UnicodeDecodeError as e: + first_non_utf8 = e.start # Use chardet as a hint, not as ground truth import chardet - detected = chardet.detect(raw) + # chardet is pure Python (~1.3s/MB), so sample around the first bad byte + window = 256 * 1024 + sample_start = max(0, first_non_utf8 - window // 2) + sample = raw[sample_start : sample_start + window] + detected = chardet.detect(sample) + # A stray byte can sit far from the real payload, leaving the sample with nothing to read + if len(sample.translate(None, delete=bytes(range(128)))) < 64 and len(sample) < len(raw): + detected = chardet.detect(raw) detected_enc = (detected.get('encoding') or '').lower().replace('-', '').replace('_', '') # Map chardet's detected encoding to the correct superset codec.