mirror of
https://github.com/open-webui/open-webui.git
synced 2026-08-28 08:25:01 -06:00
Classic298
2d18727ab8
Raising GLOBAL_LOG_LEVEL to WARNING buys quieter output but not less work: 241 INFO call sites interpolate their payload into an f-string before the logging call gets to drop it. The heaviest is get_doc, which logs every chunk id and metadata dict in a collection, so on the full-context retrieval path that is the entire knowledge base, once per chat request.
That one line at WARNING, CPython 3.12:
| knowledge base | payload | before | after |
| -------------- | ------- | -------- | ------- |
| top-k of 3 | 1.2 kB | 3.8 us | 0.07 us |
| 500 chunks | 201 kB | 583.6 us | 0.08 us |
| 5000 chunks | 2.0 MB | 5.8 ms | 0.15 us |
The lazy form log.info('query_doc:result %s %s', result.ids, result.metadatas) hands the payload to record.getMessage(), which the InterceptHandler only reaches once a record has passed the level check. Output at INFO is byte-identical. Two sites that already built their message eagerly, one str concat and one % operator, move to the same lazy form.
128 lines
4.3 KiB
Python
128 lines
4.3 KiB
Python
import base64
|
|
import logging
|
|
import os
|
|
import sys
|
|
from typing import List
|
|
|
|
import requests
|
|
from langchain_core.documents import Document
|
|
from open_webui.env import GLOBAL_LOG_LEVEL
|
|
|
|
logging.basicConfig(stream=sys.stdout, level=GLOBAL_LOG_LEVEL)
|
|
log = logging.getLogger(__name__)
|
|
|
|
PADDLEOCR_VL_IMAGE_EXTENSIONS = ['png', 'jpg', 'jpeg', 'bmp', 'tiff', 'webp']
|
|
PADDLEOCR_VL_SUPPORTED_EXTENSIONS = ['pdf'] + PADDLEOCR_VL_IMAGE_EXTENSIONS
|
|
|
|
|
|
class PaddleOCRVLLoader:
|
|
"""Loader that uses PaddleOCR-vl API to extract text from PDF/images."""
|
|
|
|
def __init__(
|
|
self,
|
|
api_url: str,
|
|
token: str,
|
|
file_path: str,
|
|
):
|
|
if not api_url or not token:
|
|
raise ValueError('PaddleOCR-vl API URL and Token are required.')
|
|
if not os.path.exists(file_path):
|
|
raise FileNotFoundError(f'File not found at {file_path}')
|
|
|
|
self.api_url = api_url.rstrip('/')
|
|
self.token = token
|
|
self.file_path = file_path
|
|
self.file_name = os.path.basename(file_path)
|
|
|
|
def load(self) -> List[Document]:
|
|
log.info('Processing with PaddleOCR-vl: %s', self.file_path)
|
|
|
|
try:
|
|
with open(self.file_path, 'rb') as file:
|
|
file_bytes = file.read()
|
|
file_data = base64.b64encode(file_bytes).decode('ascii')
|
|
except Exception as e:
|
|
log.error(f'Failed to read file {self.file_path}: {e}')
|
|
raise
|
|
|
|
headers = {'Authorization': f'token {self.token}', 'Content-Type': 'application/json'}
|
|
|
|
# Detect fileType based on file extension
|
|
ext = self.file_path.lower().split('.')[-1]
|
|
file_type = 1 if ext in PADDLEOCR_VL_IMAGE_EXTENSIONS else 0
|
|
|
|
payload = {
|
|
'file': file_data,
|
|
'fileType': file_type,
|
|
'useDocOrientationClassify': False,
|
|
'useDocUnwarping': False,
|
|
'useChartRecognition': False,
|
|
}
|
|
|
|
try:
|
|
response = requests.post(f'{self.api_url}/layout-parsing', json=payload, headers=headers)
|
|
response.raise_for_status()
|
|
|
|
result = response.json().get('result', {})
|
|
layout_results = result.get('layoutParsingResults', [])
|
|
|
|
documents = []
|
|
total_pages = len(layout_results)
|
|
skipped_pages = 0
|
|
|
|
for i, res in enumerate(layout_results):
|
|
markdown_text = res.get('markdown', {}).get('text', '')
|
|
|
|
if isinstance(markdown_text, str):
|
|
cleaned_content = markdown_text.strip()
|
|
else:
|
|
cleaned_content = str(markdown_text).strip()
|
|
|
|
if not cleaned_content:
|
|
skipped_pages += 1
|
|
continue
|
|
|
|
documents.append(
|
|
Document(
|
|
page_content=cleaned_content,
|
|
metadata={
|
|
'page': i,
|
|
'page_label': i + 1,
|
|
'total_pages': total_pages,
|
|
'file_name': self.file_name,
|
|
'processing_engine': 'paddleocr-vl',
|
|
},
|
|
)
|
|
)
|
|
|
|
if skipped_pages > 0:
|
|
log.info('PaddleOCR-vl: Processed %s pages, skipped %s empty pages.', len(documents), skipped_pages)
|
|
|
|
if not documents:
|
|
log.warning('No valid text content found by PaddleOCR-vl.')
|
|
return [
|
|
Document(
|
|
page_content='No valid text content found in document',
|
|
metadata={
|
|
'error': 'no_valid_pages',
|
|
'file_name': self.file_name,
|
|
'processing_engine': 'paddleocr-vl',
|
|
},
|
|
)
|
|
]
|
|
|
|
return documents
|
|
|
|
except Exception as e:
|
|
log.error(f'Error calling PaddleOCR-vl: {e}')
|
|
return [
|
|
Document(
|
|
page_content=f'Error during OCR processing: {e}',
|
|
metadata={
|
|
'error': 'processing_failed',
|
|
'file_name': self.file_name,
|
|
'processing_engine': 'paddleocr-vl',
|
|
},
|
|
)
|
|
]
|