ewilliams28/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-08-13 01:02:25 -06:00
Code Issues Packages Projects Releases Wiki Activity

refac

Browse Source
This commit is contained in:
Timothy Jaeryang Baek
2026-07-31 17:41:14 -04:00
parent 3becec6ccf
commit bb0f898b43
45 changed files with 552 additions and 552 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/open_webui/config.py
+38 -38
View File
@@ -1,7 +1,6 @@
from __future__ import annotations
import base64
import json
import logging
import os
import shutil
@@ -35,6 +34,7 @@ from open_webui.env import (
log,
)
from open_webui.models.config import Config
from open_webui.utils.json_codec import JSONCodec
async def seed_registered_defaults():
@@ -84,7 +84,7 @@ async def import_legacy_config_json():
if not os.path.exists(f'{DATA_DIR}/config.json'):
return
with open(f'{DATA_DIR}/config.json', 'r') as _f:
await Config.upsert(json.load(_f))
await Config.upsert(JSONCodec.loads(_f.read()))
os.rename(f'{DATA_DIR}/config.json', f'{DATA_DIR}/old_config.json')
@@ -299,12 +299,12 @@ OLLAMA_API_CONFIGS = {}
_ollama_api_configs = os.getenv('OLLAMA_API_CONFIGS', '')
if _ollama_api_configs:
try:
parsed = json.loads(_ollama_api_configs)
parsed = JSONCodec.loads(_ollama_api_configs)
if isinstance(parsed, dict):
OLLAMA_API_CONFIGS = parsed
else:
log.warning('OLLAMA_API_CONFIGS must be a JSON object, ignoring')
except (json.JSONDecodeError, TypeError):
except (JSONCodec.JSONDecodeError, TypeError):
log.warning('OLLAMA_API_CONFIGS is not valid JSON, ignoring')
####################################
@@ -346,12 +346,12 @@ OPENAI_API_CONFIGS = {}
_openai_api_configs = os.getenv('OPENAI_API_CONFIGS', '')
if _openai_api_configs:
try:
parsed = json.loads(_openai_api_configs)
parsed = JSONCodec.loads(_openai_api_configs)
if isinstance(parsed, dict):
OPENAI_API_CONFIGS = parsed
else:
log.warning('OPENAI_API_CONFIGS must be a JSON object, ignoring')
except (json.JSONDecodeError, TypeError):
except (JSONCodec.JSONDecodeError, TypeError):
log.warning('OPENAI_API_CONFIGS is not valid JSON, ignoring')
# Get the actual OpenAI API key based on the base URL
@@ -375,7 +375,7 @@ ENABLE_BASE_MODELS_CACHE = os.getenv('ENABLE_BASE_MODELS_CACHE', 'False').lower(
####################################
try:
tool_server_connections = json.loads(os.getenv('TOOL_SERVER_CONNECTIONS', '[]'))
tool_server_connections = JSONCodec.loads(os.getenv('TOOL_SERVER_CONNECTIONS', '[]'))
except Exception as e:
log.exception(f'Error loading TOOL_SERVER_CONNECTIONS: {e}')
tool_server_connections = []
@@ -389,12 +389,12 @@ OAUTH_CLIENT_TIMEOUT = os.getenv('OAUTH_CLIENT_TIMEOUT', '')
# TERMINAL_SERVER
####################################
terminal_server_connections = json.loads(os.getenv('TERMINAL_SERVER_CONNECTIONS', '[]'))
terminal_server_connections = JSONCodec.loads(os.getenv('TERMINAL_SERVER_CONNECTIONS', '[]'))
TERMINAL_SERVER_CONNECTIONS = terminal_server_connections
try:
TERMINAL_PROXY_HEADERS = json.loads(os.getenv('TERMINAL_PROXY_HEADERS', '{}'))
TERMINAL_PROXY_HEADERS = JSONCodec.loads(os.getenv('TERMINAL_PROXY_HEADERS', '{}'))
except Exception:
TERMINAL_PROXY_HEADERS = {}
@@ -900,8 +900,8 @@ MINERU_API_KEY = os.getenv('MINERU_API_KEY', '')
mineru_params = os.getenv('MINERU_PARAMS', '')
try:
mineru_params = json.loads(mineru_params)
except json.JSONDecodeError:
mineru_params = JSONCodec.loads(mineru_params)
except JSONCodec.JSONDecodeError:
mineru_params = {}
MINERU_PARAMS = mineru_params
@@ -914,8 +914,8 @@ EXTERNAL_DOCUMENT_LOADER_API_KEY = os.getenv('EXTERNAL_DOCUMENT_LOADER_API_KEY',
external_document_loader_headers = os.getenv('EXTERNAL_DOCUMENT_LOADER_HEADERS', '')
try:
external_document_loader_headers = json.loads(external_document_loader_headers)
except json.JSONDecodeError:
external_document_loader_headers = JSONCodec.loads(external_document_loader_headers)
except JSONCodec.JSONDecodeError:
external_document_loader_headers = {}
if not isinstance(external_document_loader_headers, dict):
external_document_loader_headers = {}
@@ -930,8 +930,8 @@ DOCLING_API_KEY = os.getenv('DOCLING_API_KEY', '')
docling_params = os.getenv('DOCLING_PARAMS', '')
try:
docling_params = json.loads(docling_params)
except json.JSONDecodeError:
docling_params = JSONCodec.loads(docling_params)
except JSONCodec.JSONDecodeError:
docling_params = {}
DOCLING_PARAMS = docling_params
@@ -1154,7 +1154,7 @@ WEB_SEARCH_RESULT_COUNT = int(os.getenv('WEB_SEARCH_RESULT_COUNT', '3'))
try:
web_search_domain_filter_list = json.loads(os.getenv('WEB_SEARCH_DOMAIN_FILTER_LIST', '[]'))
web_search_domain_filter_list = JSONCodec.loads(os.getenv('WEB_SEARCH_DOMAIN_FILTER_LIST', '[]'))
except Exception as e:
web_search_domain_filter_list = [
# "wikipedia.com",
@@ -1303,8 +1303,8 @@ LINKUP_API_KEY = os.getenv('LINKUP_API_KEY', '')
linkup_search_params = os.getenv('LINKUP_SEARCH_PARAMS', '')
try:
linkup_search_params = json.loads(linkup_search_params)
except json.JSONDecodeError:
linkup_search_params = JSONCodec.loads(linkup_search_params)
except JSONCodec.JSONDecodeError:
linkup_search_params = {}
LINKUP_SEARCH_PARAMS = linkup_search_params
@@ -1336,8 +1336,8 @@ AUTOMATIC1111_API_AUTH = os.getenv('AUTOMATIC1111_API_AUTH', '')
automatic1111_params = os.getenv('AUTOMATIC1111_PARAMS', '')
try:
automatic1111_params = json.loads(automatic1111_params)
except json.JSONDecodeError:
automatic1111_params = JSONCodec.loads(automatic1111_params)
except JSONCodec.JSONDecodeError:
automatic1111_params = {}
AUTOMATIC1111_PARAMS = automatic1111_params
@@ -1461,8 +1461,8 @@ COMFYUI_WORKFLOW = os.getenv('COMFYUI_WORKFLOW', COMFYUI_DEFAULT_WORKFLOW)
comfyui_workflow_nodes = os.getenv('COMFYUI_WORKFLOW_NODES', '')
try:
comfyui_workflow_nodes = json.loads(comfyui_workflow_nodes)
except json.JSONDecodeError:
comfyui_workflow_nodes = JSONCodec.loads(comfyui_workflow_nodes)
except JSONCodec.JSONDecodeError:
comfyui_workflow_nodes = []
COMFYUI_WORKFLOW_NODES = comfyui_workflow_nodes
@@ -1474,8 +1474,8 @@ IMAGES_OPENAI_API_KEY = os.getenv('IMAGES_OPENAI_API_KEY', OPENAI_API_KEY)
images_openai_params = os.getenv('IMAGES_OPENAI_PARAMS', '')
try:
images_openai_params = json.loads(images_openai_params)
except json.JSONDecodeError:
images_openai_params = JSONCodec.loads(images_openai_params)
except JSONCodec.JSONDecodeError:
images_openai_params = {}
@@ -1513,8 +1513,8 @@ IMAGES_EDIT_COMFYUI_WORKFLOW = os.getenv('IMAGES_EDIT_COMFYUI_WORKFLOW', '')
images_edit_comfyui_workflow_nodes = os.getenv('IMAGES_EDIT_COMFYUI_WORKFLOW_NODES', '')
try:
images_edit_comfyui_workflow_nodes = json.loads(images_edit_comfyui_workflow_nodes)
except json.JSONDecodeError:
images_edit_comfyui_workflow_nodes = JSONCodec.loads(images_edit_comfyui_workflow_nodes)
except JSONCodec.JSONDecodeError:
images_edit_comfyui_workflow_nodes = []
IMAGES_EDIT_COMFYUI_WORKFLOW_NODES = images_edit_comfyui_workflow_nodes
@@ -1588,8 +1588,8 @@ AUDIO_TTS_OPENAI_API_KEY = os.getenv('AUDIO_TTS_OPENAI_API_KEY', OPENAI_API_KEY)
audio_tts_openai_params = os.getenv('AUDIO_TTS_OPENAI_PARAMS', '')
try:
audio_tts_openai_params = json.loads(audio_tts_openai_params)
except json.JSONDecodeError:
audio_tts_openai_params = JSONCodec.loads(audio_tts_openai_params)
except JSONCodec.JSONDecodeError:
audio_tts_openai_params = {}
AUDIO_TTS_OPENAI_PARAMS = audio_tts_openai_params
@@ -1641,7 +1641,7 @@ DEFAULT_MODELS = os.getenv('DEFAULT_MODELS', None)
DEFAULT_PINNED_MODELS = os.getenv('DEFAULT_PINNED_MODELS', None)
try:
default_prompt_suggestions = json.loads(os.getenv('DEFAULT_PROMPT_SUGGESTIONS', '[]'))
default_prompt_suggestions = JSONCodec.loads(os.getenv('DEFAULT_PROMPT_SUGGESTIONS', '[]'))
except Exception as e:
log.exception(f'Error loading DEFAULT_PROMPT_SUGGESTIONS: {e}')
default_prompt_suggestions = []
@@ -1679,7 +1679,7 @@ if default_prompt_suggestions == []:
DEFAULT_PROMPT_SUGGESTIONS = default_prompt_suggestions
try:
model_order_list = json.loads(os.getenv('MODEL_ORDER_LIST', '[]'))
model_order_list = JSONCodec.loads(os.getenv('MODEL_ORDER_LIST', '[]'))
except Exception as e:
log.exception(f'Error loading MODEL_ORDER_LIST: {e}')
model_order_list = []
@@ -1687,7 +1687,7 @@ except Exception as e:
MODEL_ORDER_LIST = model_order_list
try:
default_model_metadata = json.loads(os.getenv('DEFAULT_MODEL_METADATA', '{}'))
default_model_metadata = JSONCodec.loads(os.getenv('DEFAULT_MODEL_METADATA', '{}'))
except Exception as e:
log.exception(f'Error loading DEFAULT_MODEL_METADATA: {e}')
default_model_metadata = {}
@@ -1695,7 +1695,7 @@ except Exception as e:
DEFAULT_MODEL_METADATA = default_model_metadata
try:
default_model_params = json.loads(os.getenv('DEFAULT_MODEL_PARAMS', '{}'))
default_model_params = JSONCodec.loads(os.getenv('DEFAULT_MODEL_PARAMS', '{}'))
except Exception as e:
log.exception(f'Error loading DEFAULT_MODEL_PARAMS: {e}')
default_model_params = {}
@@ -2038,7 +2038,7 @@ ENABLE_USER_STATUS = os.getenv('ENABLE_USER_STATUS', 'True').lower() == 'true'
ENABLE_EVALUATION_ARENA_MODELS = os.getenv('ENABLE_EVALUATION_ARENA_MODELS', 'True').lower() == 'true'
try:
evaluation_arena_models = json.loads(os.getenv('EVALUATION_ARENA_MODELS', '[]'))
evaluation_arena_models = JSONCodec.loads(os.getenv('EVALUATION_ARENA_MODELS', '[]'))
if not isinstance(evaluation_arena_models, list) or not all(
isinstance(model, dict) for model in evaluation_arena_models
):
@@ -2143,7 +2143,7 @@ class BannerModel(BaseModel):
try:
banners = json.loads(os.getenv('WEBUI_BANNERS', '[]'))
banners = JSONCodec.loads(os.getenv('WEBUI_BANNERS', '[]'))
banners = [BannerModel(**banner) for banner in banners]
except Exception as e:
log.exception(f'Error loading WEBUI_BANNERS: {e}')
@@ -2479,12 +2479,12 @@ GOOGLE_OAUTH_AUTHORIZE_PARAMS = {}
_google_oauth_authorize_params = os.getenv('GOOGLE_OAUTH_AUTHORIZE_PARAMS', '')
if _google_oauth_authorize_params:
try:
_parsed = json.loads(_google_oauth_authorize_params)
_parsed = JSONCodec.loads(_google_oauth_authorize_params)
if isinstance(_parsed, dict):
GOOGLE_OAUTH_AUTHORIZE_PARAMS = _parsed
else:
log.warning('GOOGLE_OAUTH_AUTHORIZE_PARAMS must be a JSON object, ignoring')
except (json.JSONDecodeError, TypeError):
except (JSONCodec.JSONDecodeError, TypeError):
log.warning('GOOGLE_OAUTH_AUTHORIZE_PARAMS is not valid JSON, ignoring')
MICROSOFT_CLIENT_ID = os.getenv('MICROSOFT_CLIENT_ID', '')
@@ -2599,12 +2599,12 @@ OAUTH_AUTHORIZE_PARAMS = {}
_oauth_authorize_params = os.getenv('OAUTH_AUTHORIZE_PARAMS', '')
if _oauth_authorize_params:
try:
_parsed = json.loads(_oauth_authorize_params)
_parsed = JSONCodec.loads(_oauth_authorize_params)
if isinstance(_parsed, dict):
OAUTH_AUTHORIZE_PARAMS = _parsed
else:
log.warning('OAUTH_AUTHORIZE_PARAMS must be a JSON object, ignoring')
except (json.JSONDecodeError, TypeError):
except (JSONCodec.JSONDecodeError, TypeError):
log.warning('OAUTH_AUTHORIZE_PARAMS is not valid JSON, ignoring')
backend/open_webui/functions.py
+7 -7
View File
@@ -1,6 +1,5 @@
import asyncio
import inspect
import json
import logging
import sys
from typing import AsyncGenerator, Generator, Iterator
@@ -29,6 +28,7 @@ from open_webui.socket.main import (
get_event_emitter,
)
from open_webui.utils.access_control import check_model_access
from open_webui.utils.json_codec import JSONCodec
from open_webui.utils.misc import (
add_or_update_system_message,
get_last_user_message,
@@ -170,7 +170,7 @@ async def generate_function_chat_completion(request, form_data, user, models: di
line = line.model_dump_json()
line = f'data: {line}'
if isinstance(line, dict):
line = f'data: {json.dumps(line)}'
line = f'data: {JSONCodec.dumps(line)}'
try:
line = line.decode('utf-8')
@@ -181,7 +181,7 @@ async def generate_function_chat_completion(request, form_data, user, models: di
return f'{line}\n\n'
else:
line = openai_chat_chunk_message_template(form_data['model'], line)
return f'data: {json.dumps(line)}\n\n'
return f'data: {JSONCodec.dumps(line)}\n\n'
def get_pipe_id(form_data: dict) -> str:
pipe_id = form_data['model']
@@ -308,17 +308,17 @@ async def generate_function_chat_completion(request, form_data, user, models: di
yield data
return
if isinstance(res, dict):
yield f'data: {json.dumps(res)}\n\n'
yield f'data: {JSONCodec.dumps(res)}\n\n'
return
except Exception as e:
log.error(f'Error: {e}')
yield f'data: {json.dumps({"error": {"detail": str(e)}})}\n\n'
yield f'data: {JSONCodec.dumps({"error": {"detail": str(e)}})}\n\n'
return
if isinstance(res, str):
message = openai_chat_chunk_message_template(form_data['model'], res)
yield f'data: {json.dumps(message)}\n\n'
yield f'data: {JSONCodec.dumps(message)}\n\n'
if isinstance(res, Iterator):
for line in res:
@@ -330,7 +330,7 @@ async def generate_function_chat_completion(request, form_data, user, models: di
finish_message = openai_chat_chunk_message_template(form_data['model'], '')
finish_message['choices'][0]['finish_reason'] = 'stop'
yield f'data: {json.dumps(finish_message)}\n\n'
yield f'data: {JSONCodec.dumps(finish_message)}\n\n'
yield 'data: [DONE]'
return StreamingResponse(stream_content(), media_type='text/event-stream')
backend/open_webui/internal/db.py
+5 -5
View File
@@ -1,6 +1,5 @@
from __future__ import annotations
import json
import logging
import os
import sys
@@ -28,6 +27,7 @@ from open_webui.env import (
ENABLE_DB_MIGRATIONS,
OPEN_WEBUI_DIR,
)
from open_webui.utils.json_codec import JSONCodec
from sqlalchemy import Dialect, MetaData, create_engine, event, types
from sqlalchemy.engine.url import make_url
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
@@ -124,18 +124,18 @@ class JSONField(types.TypeDecorator): # TEXT-backed JSON storage
"""Store arbitrary Python objects as JSON-encoded TEXT.
Used instead of native JSON columns for portability across SQLite and
PostgreSQL. Values are serialized with ``json.dumps`` on write and
deserialized with ``json.loads`` on read.
PostgreSQL. Values are serialized with ``JSONCodec.dumps`` on write and
deserialized with ``JSONCodec.loads`` on read.
"""
impl = types.UnicodeText
cache_ok = True
def process_bind_param(self, value: _T | None, dialect: Dialect) -> Any:
return json.dumps(value) if value is not None else None
return JSONCodec.dumps(value) if value is not None else None
def process_result_value(self, value: _T | None, dialect: Dialect) -> Any:
return json.loads(value) if value is not None else None
return JSONCodec.loads(value) if value is not None else None
def copy(self, **kwargs: Any) -> Self:
return JSONField(length=self.impl.length)
backend/open_webui/main.py
+10 -9
View File
@@ -1,7 +1,6 @@
from __future__ import annotations
import asyncio
import json
import logging
import mimetypes
import os
@@ -83,19 +82,19 @@ from open_webui.env import (
ENABLE_COMPRESSION_MIDDLEWARE,
ENABLE_CUSTOM_MODEL_FALLBACK,
ENABLE_EASTER_EGGS,
ENABLE_PLUGINS,
EXTERNAL_PWA_MANIFEST_URL,
# OAuth Back-Channel Logout
ENABLE_OAUTH_BACKCHANNEL_LOGOUT,
ENABLE_OTEL,
ENABLE_PLUGINS,
ENABLE_PUBLIC_ACTIVE_USERS_COUNT,
ENABLE_PYODIDE_FILE_PERSISTENCE,
# SCIM
ENABLE_SCIM,
ENABLE_SIGNUP_PASSWORD_CONFIRMATION,
ENABLE_STAR_SESSIONS_MIDDLEWARE,
ENABLE_PYODIDE_FILE_PERSISTENCE,
ENABLE_VERSION_UPDATE_CHECK,
ENABLE_WEBSOCKET_SUPPORT,
EXTERNAL_PWA_MANIFEST_URL,
GLOBAL_LOG_LEVEL,
INSTANCE_ID,
LICENSE_KEY,
@@ -121,12 +120,14 @@ from open_webui.env import (
from open_webui.events import (
EVENTS,
delete_event_webhook,
get_event_catalog as get_event_catalog_items,
get_event_webhooks,
migrate_legacy_webhook_config,
publish_event,
upsert_event_webhook,
)
from open_webui.events import (
get_event_catalog as get_event_catalog_items,
)
from open_webui.internal.db import engine, get_async_session
from open_webui.models.access_grants import AccessGrants
from open_webui.models.channels import Channels
@@ -154,8 +155,8 @@ from open_webui.routers import (
knowledge,
memories,
models,
notifications,
notes,
notifications,
ollama,
openai,
pipelines,
@@ -893,7 +894,7 @@ async def get_models(request: Request, refresh: bool = False, user=Depends(get_v
if log.isEnabledFor(logging.DEBUG):
log.debug(
f'/api/models returned filtered models accessible to the user: {json.dumps([model.get("id") for model in models])}'
f'/api/models returned filtered models accessible to the user: {JSONCodec.dumps([model.get("id") for model in models])}'
)
return {'data': models}
@@ -946,7 +947,7 @@ async def unload_model(request: Request, form_data: ModelUnloadForm, user=Depend
prefix_id = api_config.get('prefix_id', None)
actual_model = strip_provider_model_prefix(model_id, prefix_id)
payload = json.dumps({'model': actual_model, 'keep_alive': 0, 'prompt': ''})
payload = JSONCodec.dumps({'model': actual_model, 'keep_alive': 0, 'prompt': ''})
try:
timeout = aiohttp.ClientTimeout(total=30)
@@ -1568,7 +1569,7 @@ async def chat_completion(
# chat:tasks:cancel, unblocking the frontend.
if isinstance(response, JSONResponse) and response.status_code >= 400:
try:
error_body = json.loads(response.body.decode('utf-8', 'replace'))
error_body = JSONCodec.loads(response.body.decode('utf-8', 'replace'))
detail = error_body.get('error', error_body) if isinstance(error_body, dict) else error_body
if isinstance(detail, dict):
detail = detail.get('message', detail.get('detail', str(detail)))
backend/open_webui/models/chats.py
-1
View File
@@ -2,7 +2,6 @@
from __future__ import annotations
import json
import logging
import time
import uuid
backend/open_webui/models/oauth_sessions.py
+3 -3
View File
@@ -1,6 +1,5 @@
import base64
import hashlib
import json
import logging
import time
import uuid
@@ -9,6 +8,7 @@ from typing import List, Optional
from cryptography.fernet import Fernet
from open_webui.env import OAUTH_SESSION_TOKEN_ENCRYPTION_KEY
from open_webui.internal.db import Base, get_async_db_context
from open_webui.utils.json_codec import JSONCodec
from pydantic import BaseModel, ConfigDict
from sqlalchemy import BigInteger, Column, Index, String, Text, delete, select, update
from sqlalchemy.ext.asyncio import AsyncSession
@@ -85,7 +85,7 @@ class OAuthSessionTable:
def _encrypt_token(self, token) -> str:
"""Encrypt OAuth tokens for storage"""
try:
token_json = json.dumps(token)
token_json = JSONCodec.dumps(token)
encrypted = self.fernet.encrypt(token_json.encode()).decode()
return encrypted
except Exception as e:
@@ -96,7 +96,7 @@ class OAuthSessionTable:
"""Decrypt OAuth tokens from storage"""
try:
decrypted = self.fernet.decrypt(token.encode()).decode()
return json.loads(decrypted)
return JSONCodec.loads(decrypted)
except Exception as e:
log.error(f'Error decrypting tokens: {type(e).__name__}: {e}')
raise
backend/open_webui/models/prompts.py
+2 -2
View File
@@ -2,7 +2,6 @@
from __future__ import annotations
import json
import logging
import time
import uuid
@@ -15,6 +14,7 @@ from open_webui.models.access_grants import AccessGrantModel, AccessGrants
from open_webui.models.groups import Groups
from open_webui.models.prompt_history import PromptHistories
from open_webui.models.users import User, UserModel, UserResponse, Users
from open_webui.utils.json_codec import JSONCodec
from pydantic import BaseModel, ConfigDict, Field
from sqlalchemy import JSON, BigInteger, Boolean, Column, String, Text, cast, delete, func, or_, select, text, update
from sqlalchemy.ext.asyncio import AsyncSession
@@ -344,7 +344,7 @@ class PromptsTable:
else:
# Fallback: LIKE on serialised JSON text (ASCII-safe only)
tag_clause = func.lower(cast(Prompt.tags, String)).like(
f'%{json.dumps(tag_lower, ensure_ascii=False)}%'
f'%{JSONCodec.dumps(tag_lower, ensure_ascii=False)}%'
)
tag_lower = None
backend/open_webui/retrieval/loaders/datalab_marker.py
+3 -2
View File
@@ -7,6 +7,7 @@ from typing import List, Optional
import requests
from fastapi import HTTPException, status
from langchain_core.documents import Document
from open_webui.utils.json_codec import JSONCodec
log = logging.getLogger(__name__)
@@ -249,11 +250,11 @@ class DatalabMarkerLoader:
images = final_result.get('images', {})
if images:
metadata['image_count'] = len(images)
metadata['images'] = json.dumps(list(images.keys()))
metadata['images'] = JSONCodec.dumps(list(images.keys()))
for k, v in metadata.items():
if isinstance(v, (dict, list)):
metadata[k] = json.dumps(v)
metadata[k] = JSONCodec.dumps(v)
elif v is None:
metadata[k] = ''
backend/open_webui/retrieval/loaders/main.py
+3 -3
View File
@@ -1,5 +1,4 @@
import asyncio
import json
import logging
import sys
@@ -28,6 +27,7 @@ from open_webui.retrieval.loaders.mineru import MinerULoader
from open_webui.retrieval.loaders.mistral import MistralLoader
from open_webui.retrieval.loaders.paddleocr_vl import PADDLEOCR_VL_SUPPORTED_EXTENSIONS, PaddleOCRVLLoader
from open_webui.utils.headers import get_user_groups_for_custom_headers
from open_webui.utils.json_codec import JSONCodec
logging.basicConfig(stream=sys.stdout, level=GLOBAL_LOG_LEVEL)
log = logging.getLogger(__name__)
@@ -508,8 +508,8 @@ class Loader:
params = self.kwargs.get('DOCLING_PARAMS', {})
if not isinstance(params, dict):
try:
params = json.loads(params)
except json.JSONDecodeError:
params = JSONCodec.loads(params)
except JSONCodec.JSONDecodeError:
log.error('Invalid DOCLING_PARAMS format, expected JSON object')
params = {}
backend/open_webui/retrieval/vector/dbs/mariadb_vector.py
+4 -4
View File
@@ -5,7 +5,6 @@ NOTE: This vector database integration is community-supported and maintained on
from __future__ import annotations
import array
import json
import logging
import math
import re
@@ -30,6 +29,7 @@ from open_webui.retrieval.vector.main import (
VectorItem,
)
from open_webui.retrieval.vector.utils import process_metadata
from open_webui.utils.json_codec import JSONCodec
from sqlalchemy import create_engine
from sqlalchemy.pool import NullPool, QueuePool
@@ -72,7 +72,7 @@ def _safe_json(v: Any) -> Dict[str, Any]:
return {}
if isinstance(v, str):
try:
j = json.loads(v)
j = JSONCodec.loads(v)
return j if isinstance(j, dict) else {}
except Exception:
return {}
@@ -324,7 +324,7 @@ class MariaDBVectorClient(VectorDBBase):
emb,
collection_name,
item.get('text'),
json.dumps(meta),
JSONCodec.dumps(meta),
)
)
cur.executemany(sql, params)
@@ -367,7 +367,7 @@ class MariaDBVectorClient(VectorDBBase):
emb,
collection_name,
item.get('text'),
json.dumps(meta),
JSONCodec.dumps(meta),
)
)
cur.executemany(sql, params)
backend/open_webui/retrieval/vector/dbs/milvus.py
+2 -2
View File
@@ -2,7 +2,6 @@
NOTE: This vector database integration is community-supported and maintained on a best-effort basis.
"""
import json
import logging
from typing import Optional
@@ -25,6 +24,7 @@ from open_webui.retrieval.vector.main import (
VectorItem,
)
from open_webui.retrieval.vector.utils import process_metadata
from open_webui.utils.json_codec import JSONCodec
from pymilvus import DataType
from pymilvus import MilvusClient as Client
from pymilvus.exceptions import MilvusException
@@ -354,7 +354,7 @@ class MilvusClient(VectorDBBase):
ids=ids,
)
elif filter:
filter_string = ' && '.join([f'metadata["{key}"] == {json.dumps(value)}' for key, value in filter.items()])
filter_string = ' && '.join([f'metadata["{key}"] == {JSONCodec.dumps(value)}' for key, value in filter.items()])
log.info(
f'Deleting items by filter from {self.collection_prefix}_{collection_name}. Filter: {filter_string}'
)
backend/open_webui/retrieval/vector/dbs/pgvector.py
+3 -3
View File
@@ -1,4 +1,3 @@
import json
import logging
from typing import Any, Dict, List, Optional, Tuple
@@ -25,6 +24,7 @@ from open_webui.retrieval.vector.main import (
VectorItem,
)
from open_webui.retrieval.vector.utils import merge_hybrid_search_results, process_metadata
from open_webui.utils.json_codec import JSONCodec
from open_webui.utils.misc import sanitize_text_for_db
from pgvector.sqlalchemy import HALFVEC, Vector
from sqlalchemy import (
@@ -303,7 +303,7 @@ class PgvectorClient(VectorDBBase):
# Use raw SQL for BYTEA/pgcrypto
# Ensure metadata is converted to its JSON text representation
# Sanitize to strip null bytes / surrogates that PostgreSQL cannot store
json_metadata = sanitize_text_for_db(json.dumps(item['metadata']))
json_metadata = sanitize_text_for_db(JSONCodec.dumps(item['metadata']))
item_text = sanitize_text_for_db(item['text'])
self.session.execute(
text("""
@@ -354,7 +354,7 @@ class PgvectorClient(VectorDBBase):
for item in items:
vector = self.adjust_vector_length(item['vector'])
# Sanitize to strip null bytes / surrogates that PostgreSQL cannot store
json_metadata = sanitize_text_for_db(json.dumps(item['metadata']))
json_metadata = sanitize_text_for_db(JSONCodec.dumps(item['metadata']))
item_text = sanitize_text_for_db(item['text'])
self.session.execute(
text("""
backend/open_webui/retrieval/web/bocha.py
+2 -2
View File
@@ -1,9 +1,9 @@
import json
import logging
from typing import Optional
import requests
from open_webui.retrieval.web.main import SearchResult, get_filtered_results
from open_webui.utils.json_codec import JSONCodec
log = logging.getLogger(__name__)
@@ -41,7 +41,7 @@ def search_bocha(api_key: str, query: str, count: int, filter_list: Optional[lis
url = 'https://api.bochaai.com/v1/web-search?utm_source=ollama'
headers = {'Authorization': f'Bearer {api_key}', 'Content-Type': 'application/json'}
payload = json.dumps({'query': query, 'summary': True, 'freshness': 'noLimit', 'count': count})
payload = JSONCodec.dumps({'query': query, 'summary': True, 'freshness': 'noLimit', 'count': count})
response = requests.post(url, headers=headers, data=payload, timeout=5)
response.raise_for_status()
backend/open_webui/retrieval/web/serper.py
+2 -2
View File
@@ -1,9 +1,9 @@
from __future__ import annotations
import json
import logging
from open_webui.retrieval.web.main import SearchResult, get_filtered_results
from open_webui.utils.json_codec import JSONCodec
from open_webui.utils.session_pool import get_session
log = logging.getLogger(__name__)
@@ -23,7 +23,7 @@ async def search_serper(
headers = {'X-API-KEY': api_key, 'Content-Type': 'application/json'}
session = await get_session()
async with session.post(url, headers=headers, data=json.dumps({'q': query})) as response:
async with session.post(url, headers=headers, data=JSONCodec.dumps({'q': query})) as response:
response.raise_for_status()
payload = await response.json()
backend/open_webui/retrieval/web/sougou.py
+3 -3
View File
@@ -1,8 +1,8 @@
import json
import logging
from typing import List, Optional
from open_webui.retrieval.web.main import SearchResult, get_filtered_results
from open_webui.utils.json_codec import JSONCodec
log = logging.getLogger(__name__)
@@ -28,10 +28,10 @@ def search_sougou(
http_profile.endpoint = 'tms.tencentcloudapi.com'
client_profile = ClientProfile()
client_profile.http_profile = http_profile
params = json.dumps({'Query': query, 'Cnt': 20})
params = JSONCodec.dumps({'Query': query, 'Cnt': 20})
common_client = CommonClient('tms', '2020-12-29', cred, '', profile=client_profile)
results = [
json.loads(page) for page in common_client.call_json('SearchPro', json.loads(params))['Response']['Pages']
JSONCodec.loads(page) for page in common_client.call_json('SearchPro', JSONCodec.loads(params))['Response']['Pages']
]
sorted_results = sorted(results, key=lambda x: x.get('scour', 0.0), reverse=True)
if filter_list:
backend/open_webui/retrieval/web/yandex.py
+2 -2
View File
@@ -1,6 +1,5 @@
import base64
import io
import json
import logging
import os
from typing import List, Optional
@@ -12,6 +11,7 @@ from fastapi import Request
from open_webui.env import FORWARD_SESSION_INFO_HEADER_CHAT_ID
from open_webui.retrieval.web.main import SearchResult, get_filtered_results
from open_webui.utils.headers import include_user_info_headers
from open_webui.utils.json_codec import JSONCodec
log = logging.getLogger(__name__)
@@ -55,7 +55,7 @@ def search_yandex(
if chat_id:
headers[FORWARD_SESSION_INFO_HEADER_CHAT_ID] = str(chat_id)
payload = {} if yandex_search_config == '' else json.loads(yandex_search_config)
payload = {} if yandex_search_config == '' else JSONCodec.loads(yandex_search_config)
if type(payload.get('query', None)) != dict:
payload['query'] = {}
backend/open_webui/routers/audio.py
+17 -18
View File
@@ -5,7 +5,6 @@ import base64
import hashlib
import html
import io
import json
import logging
import mimetypes
import os
@@ -27,13 +26,6 @@ from fastapi import (
status,
)
from fastapi.responses import FileResponse
from pydantic import BaseModel
# pydub needs stdlib audioop (gone in 3.13); keep requires-python capped < 3.13
from pydub import AudioSegment
from pydub.silence import split_on_silence
from pydub.utils import mediainfo
from open_webui.config import (
CACHE_DIR,
ELEVENLABS_API_BASE_URL,
@@ -60,8 +52,15 @@ from open_webui.models.config import Config
from open_webui.utils.access_control import has_permission
from open_webui.utils.auth import get_admin_user, get_verified_user
from open_webui.utils.headers import include_user_info_headers
from open_webui.utils.json_codec import JSONCodec
from open_webui.utils.misc import strict_match_mime_type
from open_webui.utils.session_pool import get_session
from pydantic import BaseModel
# pydub needs stdlib audioop (gone in 3.13); keep requires-python capped < 3.13
from pydub import AudioSegment
from pydub.silence import split_on_silence
from pydub.utils import mediainfo
log = logging.getLogger(__name__)
router = APIRouter()
@@ -357,7 +356,7 @@ async def _write_tts_cache(
async with aiofiles.open(file_path, 'wb') as f:
await f.write(audio)
async with aiofiles.open(body_path, 'w') as f:
await f.write(json.dumps(payload))
await f.write(JSONCodec.dumps(payload))
async def _tts_openai(request, payload, file_path, file_body_path, user):
@@ -395,7 +394,7 @@ async def _tts_openai(request, payload, file_path, file_body_path, user):
await f.write(audio_data)
async with aiofiles.open(file_body_path, 'w') as f:
await f.write(json.dumps(payload))
await f.write(JSONCodec.dumps(payload))
return FileResponse(file_path)
except Exception as exc:
@@ -503,7 +502,7 @@ async def _tts_transformers(request, payload, file_path, file_body_path, user):
# Audio file already written by sf.write; just persist the request metadata.
async with aiofiles.open(file_body_path, 'w') as f:
await f.write(json.dumps(payload))
await f.write(JSONCodec.dumps(payload))
return FileResponse(file_path)
@@ -591,7 +590,7 @@ async def speech(request: Request, user=Depends(get_verified_user)):
return FileResponse(file_path)
try:
payload = json.loads(body)
payload = JSONCodec.loads(body)
except Exception as exc:
log.exception(exc)
raise HTTPException(status_code=400, detail='Invalid JSON payload')
@@ -639,7 +638,7 @@ async def _transcribe_whisper(request, file_path, languages, file_dir, id):
data = {'text': transcript.strip()}
async with aiofiles.open(os.path.join(file_dir, f'{id}.json'), 'w') as f:
await f.write(json.dumps(data))
await f.write(JSONCodec.dumps(data))
log.debug(data)
return data
@@ -702,7 +701,7 @@ async def _transcribe_openai(request, file_path, filename, languages, file_dir,
data = await r.json()
async with aiofiles.open(os.path.join(file_dir, f'{id}.json'), 'w') as f:
await f.write(json.dumps(data))
await f.write(JSONCodec.dumps(data))
return data
except Exception as e:
log.exception(e)
@@ -762,7 +761,7 @@ async def _transcribe_deepgram(request, file_path, languages, file_dir, id):
data = {'text': transcript}
async with aiofiles.open(os.path.join(file_dir, f'{id}.json'), 'w') as f:
await f.write(json.dumps(data))
await f.write(JSONCodec.dumps(data))
return data
except Exception as e:
@@ -828,7 +827,7 @@ async def _transcribe_azure(request, file_path, filename, file_dir, id):
raise HTTPException(status_code=400, detail='Azure API key and region are required for Azure STT')
# Build the transcription definition payload
definition = json.dumps(
definition = JSONCodec.dumps(
{'locales': locale_str.split(','), 'diarization': {'maxSpeakers': max_speakers, 'enabled': True}}
if locale_str
else {}
@@ -868,7 +867,7 @@ async def _transcribe_azure(request, file_path, filename, file_dir, id):
data = {'text': transcript}
async with aiofiles.open(os.path.join(file_dir, f'{id}.json'), 'w') as f:
await f.write(json.dumps(data))
await f.write(JSONCodec.dumps(data))
log.debug(data)
return data
@@ -1046,7 +1045,7 @@ async def _transcribe_mistral(request, file_path, filename, metadata, file_dir,
data = {'text': transcript}
async with aiofiles.open(os.path.join(file_dir, f'{id}.json'), 'w') as f:
await f.write(json.dumps(data))
await f.write(JSONCodec.dumps(data))
log.debug(data)
return data
backend/open_webui/routers/files.py
+6 -6
View File
@@ -1,7 +1,6 @@
import asyncio
import errno
import hashlib
import json
import logging
import os
import uuid
@@ -28,8 +27,8 @@ from open_webui.events import EVENTS, publish_event
from open_webui.internal.db import get_async_db_context, get_async_session
from open_webui.models.access_grants import AccessGrants
from open_webui.models.channels import Channels
from open_webui.models.config import Config
from open_webui.models.chats import Chats
from open_webui.models.config import Config
from open_webui.models.files import (
FileForm,
FileListResponse,
@@ -55,6 +54,7 @@ router = APIRouter()
from open_webui.utils.access_control.files import has_access_to_file
from open_webui.utils.json_codec import JSONCodec
############################
# Upload File
@@ -326,8 +326,8 @@ async def upload_file_handler(
if isinstance(metadata, str):
try:
metadata = json.loads(metadata)
except json.JSONDecodeError:
metadata = JSONCodec.loads(metadata)
except JSONCodec.JSONDecodeError:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=ERROR_MESSAGES.DEFAULT('Invalid metadata format'),
@@ -642,14 +642,14 @@ async def get_file_process_status(
if status == 'failed':
event['error'] = data.get('error')
yield f'data: {json.dumps(event)}\n\n'
yield f'data: {JSONCodec.dumps(event)}\n\n'
if status in ('completed', 'failed'):
break
else:
# Legacy
break
else:
yield f'data: {json.dumps({"status": "not_found"})}\n\n'
yield f'data: {JSONCodec.dumps({"status": "not_found"})}\n\n'
break
await asyncio.sleep(1)
backend/open_webui/routers/images.py
+4 -4
View File
@@ -3,7 +3,6 @@ from __future__ import annotations
import asyncio
import base64
import io
import json
import logging
import mimetypes
import re
@@ -17,7 +16,6 @@ import aiofiles
import aiohttp
from fastapi import APIRouter, Depends, HTTPException, Request, UploadFile
from fastapi.responses import FileResponse
from PIL import Image, ImageOps
from open_webui.config import (
CACHE_DIR,
ENABLE_OPENAI_IMAGE_EDIT_NORMALIZATION,
@@ -43,7 +41,9 @@ from open_webui.utils.images.comfyui import (
comfyui_edit_image,
comfyui_upload_image,
)
from open_webui.utils.json_codec import JSONCodec
from open_webui.utils.session_pool import get_session
from PIL import Image, ImageOps
from pydantic import BaseModel
from sqlalchemy.ext.asyncio import AsyncSession
@@ -389,7 +389,7 @@ async def get_models(request: Request, user=Depends(get_verified_user)):
) as r:
info = await r.json()
workflow = json.loads(image_config.COMFYUI_WORKFLOW)
workflow = JSONCodec.loads(image_config.COMFYUI_WORKFLOW)
model_node_id = None
for node in image_config.COMFYUI_WORKFLOW_NODES:
@@ -1008,7 +1008,7 @@ async def image_edits(
form = aiohttp.FormData()
for key, value in data.items():
if isinstance(value, dict):
form.add_field(key, json.dumps(value))
form.add_field(key, JSONCodec.dumps(value))
else:
form.add_field(key, str(value))
for param_name, (filename, file_obj, content_type_val) in files:
backend/open_webui/routers/knowledge.py
+3 -3
View File
@@ -2,7 +2,6 @@ from __future__ import annotations
import asyncio
import io
import json
import logging
import time
import uuid
@@ -31,8 +30,8 @@ from open_webui.models.knowledge import (
KnowledgeUserResponse,
)
from open_webui.models.models import ModelForm, Models
from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT
from open_webui.retrieval.external import retrieve_external_knowledge, retrieve_external_knowledge_for_connection
from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT
from open_webui.routers.retrieval import (
BatchProcessFilesForm,
ProcessFileForm,
@@ -43,6 +42,7 @@ from open_webui.storage.provider import Storage
from open_webui.utils.access_control import filter_allowed_access_grants, has_permission
from open_webui.utils.access_control.files import has_access_to_file
from open_webui.utils.auth import get_admin_user, get_verified_user
from open_webui.utils.json_codec import JSONCodec
from pydantic import BaseModel
from sqlalchemy.ext.asyncio import AsyncSession
@@ -1277,7 +1277,7 @@ async def get_pending_knowledge_files(
for _ in range(MAX_POLL_DURATION // 3):
pending = await Files.get_pending_files_for_knowledge(knowledge_id)
data = [f.model_dump() for f in pending]
yield f'data: {json.dumps(data)}\n\n'
yield f'data: {JSONCodec.dumps(data)}\n\n'
if len(pending) == 0:
break
await asyncio.sleep(3)
backend/open_webui/routers/ollama.py
+13 -15
View File
@@ -1,7 +1,6 @@
from __future__ import annotations
import asyncio
import json
import logging
import os
import random
@@ -16,12 +15,8 @@ import aiohttp
from aiocache import cached
from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, ConfigDict, validator
from sqlalchemy.ext.asyncio import AsyncSession
from open_webui.config import UPLOAD_DIR
from open_webui.constants import ERROR_MESSAGES
from open_webui.events import EVENTS, publish_event, publish_model_provider_request_failed
from open_webui.env import (
AIOHTTP_CLIENT_SESSION_SSL,
AIOHTTP_CLIENT_TIMEOUT_MODEL_LIST,
@@ -31,6 +26,7 @@ from open_webui.env import (
FORWARD_SESSION_INFO_HEADER_CHAT_ID,
MODELS_CACHE_TTL,
)
from open_webui.events import EVENTS, publish_event, publish_model_provider_request_failed
from open_webui.internal.db import get_async_session
from open_webui.models.access_grants import AccessGrants
from open_webui.models.config import Config
@@ -40,15 +36,17 @@ from open_webui.models.users import UserModel
from open_webui.utils.access_control import check_model_access
from open_webui.utils.auth import get_admin_user, get_verified_user
from open_webui.utils.headers import get_custom_headers, include_user_info_headers
from open_webui.utils.model_ids import strip_provider_model_prefix
from open_webui.utils.json_codec import JSONCodec
from open_webui.utils.misc import calculate_sha256
from open_webui.utils.model_ids import strip_provider_model_prefix
from open_webui.utils.payload import (
apply_model_params_to_body_ollama,
apply_model_params_to_body_openai,
apply_system_prompt_to_body,
)
from open_webui.utils.session_pool import cleanup_response, get_client_timeout, get_session, stream_wrapper
from pydantic import BaseModel, ConfigDict, validator
from sqlalchemy.ext.asyncio import AsyncSession
log = logging.getLogger(__name__)
@@ -619,7 +617,7 @@ async def unload_model(
try:
res = await send_request(
f'{url}/api/generate',
payload=json.dumps(payload),
payload=JSONCodec.dumps(payload),
key=key,
user=user,
)
@@ -657,7 +655,7 @@ async def pull_model(
# Admins may pull from any registry
return await send_request(
f'{url}/api/pull',
payload=json.dumps({**form_data, 'insecure': True}),
payload=JSONCodec.dumps({**form_data, 'insecure': True}),
key=get_api_key(url_idx, url, (await Config.get('ollama.api_configs', {}))),
user=user,
stream=True,
@@ -812,7 +810,7 @@ async def delete_model(
await send_request(
f'{url}/api/delete',
'DELETE',
payload=json.dumps(payload),
payload=JSONCodec.dumps(payload),
key=key,
user=user,
)
@@ -854,7 +852,7 @@ async def show_model_info(
return await send_request(
f'{url}/api/show',
payload=json.dumps(payload),
payload=JSONCodec.dumps(payload),
key=key,
user=user,
)
@@ -1581,7 +1579,7 @@ async def download_file_stream(
) as blob_resp:
if blob_resp.ok:
await asyncio.to_thread(os.remove, file_path)
yield f'data: {json.dumps({"done": done, "blob": f"sha256:{hashed}", "name": file_name})}\n\n'
yield f'data: {JSONCodec.dumps({"done": done, "blob": f"sha256:{hashed}", "name": file_name})}\n\n'
else:
raise RuntimeError('Ollama: Could not create blob, Please try again.')
@@ -1651,7 +1649,7 @@ async def upload_model(
while chunk := await f.read(chunk_size):
bytes_read += len(chunk)
progress = round(bytes_read / total_size * 100, 2)
event = json.dumps({'progress': progress, 'total': total_size, 'completed': bytes_read})
event = JSONCodec.dumps({'progress': progress, 'total': total_size, 'completed': bytes_read})
yield f'data: {event}\n\n'
session = await get_session()
@@ -1688,13 +1686,13 @@ async def upload_model(
async with session.post(
f'{ollama_url}/api/create',
headers={'Content-Type': 'application/json'},
data=json.dumps(create_payload),
data=JSONCodec.dumps(create_payload),
ssl=AIOHTTP_CLIENT_SESSION_SSL,
timeout=get_client_timeout(),
) as create_resp:
if create_resp.ok:
log.info('API SUCCESS!')
event = json.dumps(
event = JSONCodec.dumps(
{'done': True, 'blob': f'sha256:{file_hash}', 'name': filename, 'model_created': model}
)
yield f'data: {event}\n\n'
@@ -1703,6 +1701,6 @@ async def upload_model(
raise Exception(f'Failed to create model in Ollama. {resp_text}')
except Exception as exc:
yield f'data: {json.dumps({"error": str(exc)})}\n\n'
yield f'data: {JSONCodec.dumps({"error": str(exc)})}\n\n'
return StreamingResponse(file_process_stream(), media_type='text/event-stream')
backend/open_webui/routers/openai.py
+14 -15
View File
@@ -2,7 +2,6 @@ from __future__ import annotations
import asyncio
import hashlib
import json
import logging
import re
from typing import Optional
@@ -23,7 +22,6 @@ from open_webui.config import (
CACHE_DIR,
)
from open_webui.constants import ERROR_MESSAGES
from open_webui.events import EVENTS, publish_event, publish_model_provider_request_failed
from open_webui.env import (
AIOHTTP_CLIENT_SESSION_SSL,
AIOHTTP_CLIENT_TIMEOUT_MODEL_LIST,
@@ -33,6 +31,7 @@ from open_webui.env import (
FORWARD_SESSION_INFO_HEADER_CHAT_ID,
MODELS_CACHE_TTL,
)
from open_webui.events import EVENTS, publish_event, publish_model_provider_request_failed
from open_webui.internal.db import get_async_session
from open_webui.models.access_grants import AccessGrants
from open_webui.models.config import Config
@@ -44,11 +43,11 @@ from open_webui.utils.anthropic import get_anthropic_models, is_anthropic_url
from open_webui.utils.auth import get_admin_user, get_verified_user
from open_webui.utils.headers import get_custom_headers, include_user_info_headers
from open_webui.utils.json_codec import JSONCodec
from open_webui.utils.model_ids import strip_provider_model_prefix
from open_webui.utils.misc import (
convert_logit_bias_input_to_json,
stream_chunks_handler,
)
from open_webui.utils.model_ids import strip_provider_model_prefix
from open_webui.utils.payload import (
apply_model_params_to_body_openai,
apply_system_prompt_to_body,
@@ -353,7 +352,7 @@ async def count_anthropic_tokens(request: Request, form_data: dict, user: UserMo
response = await session.request(
method='POST',
url=request_url,
data=json.dumps(payload),
data=JSONCodec.dumps(payload),
headers=headers,
cookies=cookies,
ssl=AIOHTTP_CLIENT_SESSION_SSL,
@@ -502,7 +501,7 @@ async def speech(request: Request, user=Depends(get_verified_user)):
await f.write(chunk)
async with aiofiles.open(file_body_path, 'w') as f:
await f.write(json.dumps(json.loads(body.decode('utf-8'))))
await f.write(JSONCodec.dumps(JSONCodec.loads(body.decode('utf-8'))))
# Return the saved file
return FileResponse(file_path)
@@ -1286,7 +1285,7 @@ async def generate_chat_completion(
logit_bias = convert_logit_bias_input_to_json(payload['logit_bias'])
if logit_bias:
payload['logit_bias'] = json.loads(logit_bias)
payload['logit_bias'] = JSONCodec.loads(logit_bias)
headers, cookies = await get_headers_and_cookies(request, url, key, api_config, metadata, user=user)
@@ -1338,7 +1337,7 @@ async def generate_chat_completion(
if not is_streaming_request:
payload.pop('stream_options', None)
payload = json.dumps(payload)
payload = JSONCodec.dumps(payload)
r = None
streaming = False
@@ -1370,7 +1369,7 @@ async def generate_chat_completion(
error_body[:1000],
)
try:
error_json = json.loads(error_body)
error_json = JSONCodec.loads(error_body)
await publish_model_provider_request_failed(
request,
actor=user,
@@ -1382,7 +1381,7 @@ async def generate_chat_completion(
upstream_error=error_json,
)
return JSONResponse(status_code=r.status, content=error_json)
except json.JSONDecodeError:
except JSONCodec.JSONDecodeError:
await publish_model_provider_request_failed(
request,
actor=user,
@@ -1458,7 +1457,7 @@ async def embeddings(request: Request, form_data: dict, user):
"""
idx = 0
# Prepare payload/body
body = json.dumps(form_data)
body = JSONCodec.dumps(form_data)
# Find correct backend url/key based on model
model_id = form_data.get('model')
# Check if model is already in app state cache to avoid expensive get_all_models() call
@@ -1589,7 +1588,7 @@ async def responses(
# Enforce per-model access control
await check_model_access(user, await Models.get_model_by_id(model_id), BYPASS_MODEL_ACCESS_CONTROL)
body = json.dumps(payload)
body = JSONCodec.dumps(payload)
if model_id:
models = request.app.state.OPENAI_MODELS
@@ -1699,8 +1698,8 @@ async def proxy(path: str, request: Request, user=Depends(get_verified_user)):
payload = None
if body:
try:
payload = json.loads(body)
except (json.JSONDecodeError, ValueError):
payload = JSONCodec.loads(body)
except (JSONCodec.JSONDecodeError, ValueError):
payload = None
is_streaming_request = bool(payload.get('stream', False)) if isinstance(payload, dict) else False
@@ -1738,9 +1737,9 @@ async def proxy(path: str, request: Request, user=Depends(get_verified_user)):
api_version = api_config.get('api_version', '2023-03-15-preview')
headers['api-version'] = api_version
payload = json.loads(body)
payload = JSONCodec.loads(body)
url, payload = convert_to_azure_payload(url, payload, api_version)
body = json.dumps(payload).encode()
body = JSONCodec.dumps(payload).encode()
request_url = f'{url}/{path}?api-version={api_version}'
else:
backend/open_webui/routers/terminals.py
+4 -4
View File
@@ -13,12 +13,13 @@ import aiohttp
from fastapi import APIRouter, Depends, Request, Response, WebSocket
from fastapi.responses import JSONResponse, StreamingResponse
from open_webui.config import TERMINAL_PROXY_HEADERS
from open_webui.events import EVENTS, publish_event
from open_webui.env import AIOHTTP_CLIENT_SESSION_SSL
from open_webui.events import EVENTS, publish_event
from open_webui.models.config import Config
from open_webui.models.groups import Groups
from open_webui.utils.access_control import has_connection_access
from open_webui.utils.auth import get_verified_user
from open_webui.utils.json_codec import JSONCodec
from open_webui.utils.terminals import get_terminal_server_url
from open_webui.utils.tools import bearer_auth_header, normalize_bearer_token
from starlette.background import BackgroundTask
@@ -221,14 +222,13 @@ async def _resolve_authenticated_connection(ws: WebSocket, server_id: str):
with an appropriate error code.
"""
import asyncio
import json
from open_webui.utils.auth import get_verified_user_by_token
# First-message authentication
try:
raw = await asyncio.wait_for(ws.receive_text(), timeout=10.0)
payload = json.loads(raw)
payload = JSONCodec.loads(raw)
if payload.get('type') != 'auth':
await ws.close(code=4001, reason='Expected auth message')
return None
@@ -236,7 +236,7 @@ async def _resolve_authenticated_connection(ws: WebSocket, server_id: str):
if user is None:
await ws.close(code=4001, reason='Invalid token')
return None
except (asyncio.TimeoutError, json.JSONDecodeError):
except (asyncio.TimeoutError, JSONCodec.JSONDecodeError):
await ws.close(code=4001, reason='Auth timeout or invalid payload')
return None
except Exception:
backend/open_webui/storage/provider.py
+2 -2
View File
@@ -1,4 +1,3 @@
import json
import logging
import os
import re
@@ -33,6 +32,7 @@ from open_webui.config import (
UPLOAD_DIR,
)
from open_webui.constants import ERROR_MESSAGES
from open_webui.utils.json_codec import JSONCodec
log = logging.getLogger(__name__)
@@ -211,7 +211,7 @@ class GCSStorageProvider(StorageProvider):
if GOOGLE_APPLICATION_CREDENTIALS_JSON:
self.gcs_client = storage.Client.from_service_account_info(
info=json.loads(GOOGLE_APPLICATION_CREDENTIALS_JSON)
info=JSONCodec.loads(GOOGLE_APPLICATION_CREDENTIALS_JSON)
)
else:
# if no credentials json is provided, credentials will be picked up from the environment
backend/open_webui/tasks.py
+3 -3
View File
@@ -1,12 +1,12 @@
# tasks.py
import asyncio
import json
import logging
from uuid import uuid4
from redis.asyncio import Redis
from open_webui.env import REDIS_KEY_PREFIX
from open_webui.utils.json_codec import JSONCodec
log = logging.getLogger(__name__)
@@ -29,7 +29,7 @@ async def redis_task_command_listener(app):
if message['type'] != 'message':
continue
try:
command = json.loads(message['data'])
command = JSONCodec.loads(message['data'])
if command.get('action') == 'stop':
task_id = command.get('task_id')
local_task = tasks.get(task_id)
@@ -74,7 +74,7 @@ async def redis_list_item_tasks(redis: Redis, item_id: str) -> list[str]:
async def redis_send_command(redis: Redis, command: dict):
command_json = json.dumps(command)
command_json = JSONCodec.dumps(command)
# RedisCluster doesn't expose publish() directly, but the
# PUBLISH command broadcasts across all cluster nodes server-side.
if hasattr(redis, 'nodes_manager'):
backend/open_webui/tools/builtin.py
+286 -279
View File
File diff suppressed because it is too large Load Diff
backend/open_webui/utils/anthropic.py
+26 -26
View File
@@ -1,4 +1,3 @@
import json
import logging
import aiohttp
@@ -9,6 +8,7 @@ from open_webui.env import (
)
from open_webui.models.users import UserModel
from open_webui.utils.headers import include_user_info_headers
from open_webui.utils.json_codec import JSONCodec
log = logging.getLogger(__name__)
@@ -235,7 +235,7 @@ def convert_anthropic_to_openai_payload(
'function': {
'name': block.get('name', ''),
'arguments': (
json.dumps(block.get('input', {}))
JSONCodec.dumps(block.get('input', {}))
if isinstance(block.get('input'), dict)
else str(block.get('input', '{}'))
),
@@ -531,8 +531,8 @@ def convert_openai_to_anthropic_response(
for tool_call in tool_calls:
function = tool_call.get('function', {})
try:
tool_input = json.loads(function.get('arguments', '{}'))
except (json.JSONDecodeError, TypeError):
tool_input = JSONCodec.loads(function.get('arguments', '{}'))
except (JSONCodec.JSONDecodeError, TypeError):
tool_input = {}
content.append(
{
@@ -643,7 +643,7 @@ async def openai_stream_to_anthropic_stream(openai_stream_generator, model: str
'usage': {'input_tokens': input_tokens or 0, 'output_tokens': 0},
},
}
yield f'event: message_start\ndata: {json.dumps(message_start)}\n\n'.encode()
yield f'event: message_start\ndata: {JSONCodec.dumps(message_start)}\n\n'.encode()
try:
async for chunk in openai_stream_generator:
@@ -663,8 +663,8 @@ async def openai_stream_to_anthropic_stream(openai_stream_generator, model: str
continue
try:
data = json.loads(data_string)
except (json.JSONDecodeError, TypeError):
data = JSONCodec.loads(data_string)
except (JSONCodec.JSONDecodeError, TypeError):
continue
usage_data = data.get('usage')
@@ -730,7 +730,7 @@ async def openai_stream_to_anthropic_stream(openai_stream_generator, model: str
'index': current_block_index,
'content_block': {'type': 'thinking', 'thinking': ''},
}
yield f'event: content_block_start\ndata: {json.dumps(block_start)}\n\n'.encode()
yield f'event: content_block_start\ndata: {JSONCodec.dumps(block_start)}\n\n'.encode()
thinking_block_open = True
block_delta = {
@@ -738,7 +738,7 @@ async def openai_stream_to_anthropic_stream(openai_stream_generator, model: str
'index': current_block_index,
'delta': {'type': 'thinking_delta', 'thinking': reasoning_content},
}
yield f'event: content_block_delta\ndata: {json.dumps(block_delta)}\n\n'.encode()
yield f'event: content_block_delta\ndata: {JSONCodec.dumps(block_delta)}\n\n'.encode()
# --- Handle text content ---
# Anthropic expects text blocks before tool blocks, so skip
@@ -750,7 +750,7 @@ async def openai_stream_to_anthropic_stream(openai_stream_generator, model: str
'type': 'content_block_stop',
'index': current_block_index,
}
yield f'event: content_block_stop\ndata: {json.dumps(block_stop)}\n\n'.encode()
yield f'event: content_block_stop\ndata: {JSONCodec.dumps(block_stop)}\n\n'.encode()
thinking_block_open = False
current_block_index += 1
@@ -760,7 +760,7 @@ async def openai_stream_to_anthropic_stream(openai_stream_generator, model: str
'index': current_block_index,
'content_block': {'type': 'text', 'text': ''},
}
yield f'event: content_block_start\ndata: {json.dumps(block_start)}\n\n'.encode()
yield f'event: content_block_start\ndata: {JSONCodec.dumps(block_start)}\n\n'.encode()
text_block_open = True
block_delta = {
@@ -768,7 +768,7 @@ async def openai_stream_to_anthropic_stream(openai_stream_generator, model: str
'index': current_block_index,
'delta': {'type': 'text_delta', 'text': content},
}
yield f'event: content_block_delta\ndata: {json.dumps(block_delta)}\n\n'.encode()
yield f'event: content_block_delta\ndata: {JSONCodec.dumps(block_delta)}\n\n'.encode()
# --- Handle tool calls ---
# Some providers put tool_calls on the final message object
@@ -784,7 +784,7 @@ async def openai_stream_to_anthropic_stream(openai_stream_generator, model: str
'type': 'content_block_stop',
'index': current_block_index,
}
yield f'event: content_block_stop\ndata: {json.dumps(block_stop)}\n\n'.encode()
yield f'event: content_block_stop\ndata: {JSONCodec.dumps(block_stop)}\n\n'.encode()
thinking_block_open = False
current_block_index += 1
@@ -793,7 +793,7 @@ async def openai_stream_to_anthropic_stream(openai_stream_generator, model: str
'type': 'content_block_stop',
'index': current_block_index,
}
yield f'event: content_block_stop\ndata: {json.dumps(block_stop)}\n\n'.encode()
yield f'event: content_block_stop\ndata: {JSONCodec.dumps(block_stop)}\n\n'.encode()
text_block_open = False
current_block_index += 1
@@ -856,7 +856,7 @@ async def openai_stream_to_anthropic_stream(openai_stream_generator, model: str
'input': {},
},
}
yield f'event: content_block_start\ndata: {json.dumps(block_start)}\n\n'.encode()
yield f'event: content_block_start\ndata: {JSONCodec.dumps(block_start)}\n\n'.encode()
current_block_index += 1
# Buffer arguments and emit as input_json_delta
@@ -872,19 +872,19 @@ async def openai_stream_to_anthropic_stream(openai_stream_generator, model: str
'partial_json': arguments_chunk,
},
}
yield f'event: content_block_delta\ndata: {json.dumps(block_delta)}\n\n'.encode()
yield f'event: content_block_delta\ndata: {JSONCodec.dumps(block_delta)}\n\n'.encode()
# Close the block once arguments form complete JSON
if tool['started'] and not tool['stopped']:
try:
json.loads(tool['arguments'])
JSONCodec.loads(tool['arguments'])
tool['stopped'] = True
block_stop = {
'type': 'content_block_stop',
'index': tool['block_index'],
}
yield f'event: content_block_stop\ndata: {json.dumps(block_stop)}\n\n'.encode()
except (json.JSONDecodeError, ValueError):
yield f'event: content_block_stop\ndata: {JSONCodec.dumps(block_stop)}\n\n'.encode()
except (JSONCodec.JSONDecodeError, ValueError):
pass
# --- Handle finish reason ---
@@ -902,7 +902,7 @@ async def openai_stream_to_anthropic_stream(openai_stream_generator, model: str
# Close any open thinking block
if thinking_block_open:
block_stop = {'type': 'content_block_stop', 'index': current_block_index}
yield f'event: content_block_stop\ndata: {json.dumps(block_stop)}\n\n'.encode()
yield f'event: content_block_stop\ndata: {JSONCodec.dumps(block_stop)}\n\n'.encode()
current_block_index += 1
# Flush any tools that buffered arguments but never emitted a block
@@ -921,7 +921,7 @@ async def openai_stream_to_anthropic_stream(openai_stream_generator, model: str
'input': {},
},
}
yield f'event: content_block_start\ndata: {json.dumps(block_start)}\n\n'.encode()
yield f'event: content_block_start\ndata: {JSONCodec.dumps(block_start)}\n\n'.encode()
current_block_index += 1
if tool['arguments']:
@@ -933,18 +933,18 @@ async def openai_stream_to_anthropic_stream(openai_stream_generator, model: str
'partial_json': tool['arguments'],
},
}
yield f'event: content_block_delta\ndata: {json.dumps(block_delta)}\n\n'.encode()
yield f'event: content_block_delta\ndata: {JSONCodec.dumps(block_delta)}\n\n'.encode()
# Close any open text block
if text_block_open:
block_stop = {'type': 'content_block_stop', 'index': current_block_index}
yield f'event: content_block_stop\ndata: {json.dumps(block_stop)}\n\n'.encode()
yield f'event: content_block_stop\ndata: {JSONCodec.dumps(block_stop)}\n\n'.encode()
# Close any tool call blocks that are still open
for tool in tracked_tool_calls.values():
if tool['started'] and not tool['stopped']:
block_stop = {'type': 'content_block_stop', 'index': tool['block_index']}
yield f'event: content_block_stop\ndata: {json.dumps(block_stop)}\n\n'.encode()
yield f'event: content_block_stop\ndata: {JSONCodec.dumps(block_stop)}\n\n'.encode()
# Emit message_delta with stop reason
usage = {'output_tokens': output_tokens}
@@ -969,7 +969,7 @@ async def openai_stream_to_anthropic_stream(openai_stream_generator, model: str
},
'usage': usage,
}
yield f'event: message_delta\ndata: {json.dumps(message_delta)}\n\n'.encode()
yield f'event: message_delta\ndata: {JSONCodec.dumps(message_delta)}\n\n'.encode()
# Emit message_stop
yield f'event: message_stop\ndata: {json.dumps({"type": "message_stop"})}\n\n'.encode()
yield f'event: message_stop\ndata: {JSONCodec.dumps({"type": "message_stop"})}\n\n'.encode()
backend/open_webui/utils/auth.py
+3 -3
View File
@@ -4,7 +4,6 @@ import asyncio
import base64
import hashlib
import hmac
import json
import logging
import os
import uuid
@@ -40,6 +39,7 @@ from open_webui.models.auths import Auths
from open_webui.models.config import Config
from open_webui.models.users import Users
from open_webui.utils.access_control import has_permission
from open_webui.utils.json_codec import JSONCodec
from pytz import UTC
log = logging.getLogger(__name__)
@@ -143,13 +143,13 @@ def get_license_data(app, key):
ln, lt = nt(lb)
aesgcm = AESGCM(kb)
p = json.loads(aesgcm.decrypt(ln, lt, None))
p = JSONCodec.loads(aesgcm.decrypt(ln, lt, None))
pk.verify(base64.b64decode(p['s']), p['p'].encode())
pb = base64.b64decode(p['p'])
pn, pt = nt(pb)
data = json.loads(aesgcm.decrypt(pn, pt, None).decode())
data = JSONCodec.loads(aesgcm.decrypt(pn, pt, None).decode())
exp = data.get('exp')
if exp:
backend/open_webui/utils/chat.py
+3 -3
View File
@@ -1,5 +1,4 @@
import asyncio
import json
import logging
import random
import sys
@@ -32,6 +31,7 @@ from open_webui.utils.filter import (
get_filter_functions,
process_filter_functions,
)
from open_webui.utils.json_codec import JSONCodec
from open_webui.utils.models import check_model_access, get_all_models
from open_webui.utils.payload import convert_payload_openai_to_ollama
from open_webui.utils.response import (
@@ -108,7 +108,7 @@ async def generate_direct_chat_completion(
if 'done' in data and data['done']:
break # Stop streaming when 'done' is received
yield f'data: {json.dumps(data)}\n\n'
yield f'data: {JSONCodec.dumps(data)}\n\n'
elif isinstance(data, str):
if 'data:' in data:
yield f'{data}\n\n'
@@ -249,7 +249,7 @@ async def generate_chat_completion(
if form_data.get('stream') == True:
async def stream_wrapper(stream):
yield f'data: {json.dumps({"selected_model_id": selected_model_id})}\n\n'
yield f'data: {JSONCodec.dumps({"selected_model_id": selected_model_id})}\n\n'
async for chunk in stream:
yield chunk
backend/open_webui/utils/chat_variables.py
+5 -5
View File
@@ -1,9 +1,9 @@
from __future__ import annotations
import json
import re
from typing import Any
from open_webui.utils.json_codec import JSONCodec
CHAT_VARIABLE_KEY_RE = re.compile(r'^[a-z][a-z0-9_]*$')
CHAT_VARIABLE_ANY_RE = re.compile(r'{{\s*chat\.variables\.([^\s|}]+)(?:\s*\|\s*([^}]*))?\s*}}')
@@ -64,8 +64,8 @@ def parse_json_value(value: str) -> Any:
if re.match(r'^[\[{]', value):
try:
return json.loads(value)
except json.JSONDecodeError:
return JSONCodec.loads(value)
except JSONCodec.JSONDecodeError:
return value
return value
@@ -185,7 +185,7 @@ def validate_user_variables(variables: Any) -> dict[str, str]:
raise ChatVariablesError('User variables must be an object.')
try:
if len(json.dumps(variables)) > MAX_VARIABLES_JSON_LENGTH:
if len(JSONCodec.dumps(variables)) > MAX_VARIABLES_JSON_LENGTH:
raise ChatVariablesError('User variables are too large.')
except TypeError:
raise ChatVariablesError('User variables must be JSON serializable.')
@@ -214,7 +214,7 @@ def validate_chat_variables(
variables = normalize_chat_variables(variables)
try:
if len(json.dumps(variables)) > MAX_VARIABLES_JSON_LENGTH:
if len(JSONCodec.dumps(variables)) > MAX_VARIABLES_JSON_LENGTH:
raise ChatVariablesError('Chat variables are too large.')
except TypeError:
raise ChatVariablesError('Chat variables must be JSON serializable.')
backend/open_webui/utils/code_interpreter.py
+1 -2
View File
@@ -1,5 +1,4 @@
import asyncio
import json
import logging
import uuid
from typing import Optional
@@ -128,7 +127,7 @@ class JupyterCodeExecuter:
# send message
msg_id = uuid.uuid4().hex
await ws.send(
json.dumps(
JSONCodec.dumps(
{
'header': {
'msg_id': msg_id,
backend/open_webui/utils/context_compaction.py
+3 -3
View File
@@ -1,6 +1,5 @@
from __future__ import annotations
import json
import logging
from typing import Any
@@ -8,6 +7,7 @@ from fastapi.responses import JSONResponse
from open_webui.models.chats import Chats
from open_webui.models.config import Config
from open_webui.utils.chat_id import is_saved_chat_id
from open_webui.utils.json_codec import JSONCodec
from open_webui.utils.misc import get_content_from_message, get_last_user_message, get_message_list
from open_webui.utils.task import (
get_task_model_id,
@@ -429,7 +429,7 @@ def _response_text(response: Any) -> str:
if isinstance(response, JSONResponse):
try:
response = json.loads(response.body.decode('utf-8', 'replace'))
response = JSONCodec.loads(response.body.decode('utf-8', 'replace'))
except Exception:
return ''
@@ -477,7 +477,7 @@ def _estimate_tokens(value: Any) -> int:
if not isinstance(value, str):
try:
value = json.dumps(value, ensure_ascii=False)
value = JSONCodec.dumps(value, ensure_ascii=False)
except Exception:
value = str(value)
backend/open_webui/utils/images/comfyui.py
+4 -4
View File
@@ -1,4 +1,3 @@
import json
import logging
import random
import urllib.parse
@@ -6,6 +5,7 @@ from typing import Optional
import aiohttp
from open_webui.env import AIOHTTP_CLIENT_SESSION_SSL
from open_webui.utils.json_codec import JSONCodec
from open_webui.utils.session_pool import get_session
from pydantic import BaseModel
@@ -76,7 +76,7 @@ async def _ws_get_images(ws, workflow, client_id, base_url, api_key):
async for msg in ws:
if msg.type == aiohttp.WSMsgType.TEXT:
message = json.loads(msg.data)
message = JSONCodec.loads(msg.data)
if message['type'] == 'executing':
data = message['data']
if data['node'] is None and data['prompt_id'] == prompt_id:
@@ -188,7 +188,7 @@ def _apply_workflow_nodes(workflow, nodes, model, payload):
async def comfyui_create_image(model: str, payload: ComfyUICreateImageForm, client_id, base_url, api_key):
ws_url = base_url.replace('http://', 'ws://').replace('https://', 'wss://')
workflow = json.loads(payload.workflow.workflow)
workflow = JSONCodec.loads(payload.workflow.workflow)
_apply_workflow_nodes(workflow, payload.workflow.nodes, model, payload)
headers = {'Authorization': f'Bearer {api_key}'}
@@ -229,7 +229,7 @@ class ComfyUIEditImageForm(BaseModel):
async def comfyui_edit_image(model: str, payload: ComfyUIEditImageForm, client_id, base_url, api_key):
ws_url = base_url.replace('http://', 'ws://').replace('https://', 'wss://')
workflow = json.loads(payload.workflow.workflow)
workflow = JSONCodec.loads(payload.workflow.workflow)
_apply_workflow_nodes(workflow, payload.workflow.nodes, model, payload)
headers = {'Authorization': f'Bearer {api_key}'}
backend/open_webui/utils/logger.py
+2 -1
View File
@@ -19,6 +19,7 @@ from open_webui.env import (
LOG_FORMAT,
LOGURU_DIAGNOSE,
)
from open_webui.utils.json_codec import JSONCodec
if TYPE_CHECKING:
from loguru import Message, Record
@@ -34,7 +35,7 @@ def stdout_format(record: 'Record') -> str:
str: A formatted log string intended for stdout.
"""
if record['extra']:
record['extra']['extra_json'] = json.dumps(record['extra'])
record['extra']['extra_json'] = JSONCodec.dumps(record['extra'])
extra_format = ' - {extra[extra_json]}'
else:
extra_format = ''
backend/open_webui/utils/memory.py
+2 -3
View File
@@ -1,15 +1,14 @@
from __future__ import annotations
import asyncio
import json
import logging
import re
from typing import Any
from fastapi import HTTPException
from open_webui.models.config import Config
from open_webui.models.memories import Memories
from open_webui.utils.json_codec import JSONCodec
from open_webui.utils.misc import add_or_update_system_message, get_content_from_message
log = logging.getLogger(__name__)
@@ -593,7 +592,7 @@ Conversation:
return []
try:
parsed = json.loads(content[start : end + 1])
parsed = JSONCodec.loads(content[start : end + 1])
except Exception:
return []
backend/open_webui/utils/middleware.py
+16 -17
View File
@@ -40,6 +40,8 @@ from open_webui.env import (
GLOBAL_LOG_LEVEL,
RAG_SYSTEM_CONTEXT,
)
from open_webui.events import EVENTS, publish_event
from open_webui.models.access_grants import AccessGrants
from open_webui.models.chats import Chats
from open_webui.models.config import Config
from open_webui.models.folders import Folders
@@ -47,7 +49,6 @@ from open_webui.models.models import Models
from open_webui.models.notes import Notes
from open_webui.models.oauth_sessions import OAuthSessions
from open_webui.models.users import UserModel, Users
from open_webui.events import EVENTS, publish_event
from open_webui.retrieval.utils import get_sources_from_items
from open_webui.routers.images import (
CreateImageForm,
@@ -76,7 +77,6 @@ from open_webui.socket.main import (
get_event_emitter,
)
from open_webui.utils.access_control import has_connection_access, has_permission
from open_webui.models.access_grants import AccessGrants
from open_webui.utils.access_control.files import get_owner_accessible_folder_files
from open_webui.utils.access_control.folders import has_folder_access
from open_webui.utils.chat import generate_chat_completion
@@ -94,7 +94,6 @@ from open_webui.utils.filter import (
get_filter_functions,
process_filter_functions,
)
from open_webui.utils.json_codec import JSONCodec
from open_webui.utils.mcp.client import MCPClient
from open_webui.utils.memory import add_memory_context, review_memory_after_turn
@@ -242,7 +241,7 @@ def _split_tool_calls(
def split_json_objects(raw: str) -> list[str]:
if not isinstance(raw, str):
raw = '' if raw is None else json.dumps(raw)
raw = '' if raw is None else JSONCodec.dumps(raw)
decoder = json.JSONDecoder()
results = []
@@ -257,7 +256,7 @@ def _split_tool_calls(
_, end = decoder.raw_decode(raw, position)
results.append(raw[position:end].strip())
position = end
except json.JSONDecodeError:
except JSONCodec.JSONDecodeError:
return [raw]
return results or [raw]
@@ -267,7 +266,7 @@ def _split_tool_calls(
function = tool_call.setdefault('function', {})
arguments = function.get('arguments')
if not isinstance(arguments, str):
arguments = '' if arguments is None else json.dumps(arguments)
arguments = '' if arguments is None else JSONCodec.dumps(arguments)
function['arguments'] = arguments
split_arguments = split_json_objects(arguments)
@@ -302,7 +301,7 @@ def get_citation_source_from_tool_result(
try:
try:
tool_result = JSONCodec.loads(tool_result)
except (json.JSONDecodeError, TypeError):
except (JSONCodec.JSONDecodeError, TypeError):
pass # keep tool_result as-is (e.g. fetch_url returns plain text)
if isinstance(tool_result, dict) and 'error' in tool_result:
return []
@@ -1003,7 +1002,7 @@ async def process_tool_result(
if isinstance(text, str):
try:
text = JSONCodec.loads(text)
except json.JSONDecodeError:
except JSONCodec.JSONDecodeError:
pass
tool_response.append(text)
elif item.get('type') in ['image', 'audio']:
@@ -1031,7 +1030,7 @@ async def process_tool_result(
if isinstance(text, str) and text:
try:
text = JSONCodec.loads(text)
except json.JSONDecodeError:
except JSONCodec.JSONDecodeError:
pass
tool_response.append(text)
elif resource.get('blob'):
@@ -1106,7 +1105,7 @@ async def terminal_event_handler(
if isinstance(parsed, str):
try:
parsed = JSONCodec.loads(parsed)
except (json.JSONDecodeError, TypeError):
except (JSONCodec.JSONDecodeError, TypeError):
pass
if isinstance(parsed, dict) and parsed.get('exists') is False:
return
@@ -1198,7 +1197,7 @@ async def chat_completion_tools_handler(
sources = []
specs = [tool['spec'] for tool in tools.values()]
tools_specs = json.dumps(specs, ensure_ascii=False)
tools_specs = JSONCodec.dumps(specs, ensure_ascii=False)
tools_prompt_template = task_config.get('task.tools.prompt_template')
if tools_prompt_template != '':
@@ -1969,7 +1968,7 @@ def apply_params_to_form_data(form_data, model):
try:
# Attempt to parse the string as JSON
custom_params[key] = JSONCodec.loads(value)
except json.JSONDecodeError:
except JSONCodec.JSONDecodeError:
# If it fails, keep the original string
pass
@@ -3029,7 +3028,7 @@ def get_response_data(response):
if isinstance(response.body, bytes):
try:
response_data = JSONCodec.loads(response.body.decode('utf-8', 'replace'))
except json.JSONDecodeError:
except JSONCodec.JSONDecodeError:
response_data = {'error': {'detail': 'Invalid JSON response'}}
else:
response_data = response
@@ -4471,7 +4470,7 @@ async def streaming_chat_response_handler(response, ctx):
delta_tool_call['function']['arguments'] = (
''
if delta_arguments is None
else json.dumps(delta_arguments)
else JSONCodec.dumps(delta_arguments)
)
response_tool_calls.append(delta_tool_call)
else:
@@ -4486,7 +4485,7 @@ async def streaming_chat_response_handler(response, ctx):
if delta_arguments is not None:
if not isinstance(delta_arguments, str):
delta_arguments = json.dumps(delta_arguments)
delta_arguments = JSONCodec.dumps(delta_arguments)
current_response_tool_call.setdefault('function', {})
if not isinstance(
current_response_tool_call['function'].get('arguments'),
@@ -4874,7 +4873,7 @@ async def streaming_chat_response_handler(response, ctx):
'function': {
'name': item.get('name', ''),
'arguments': (
arguments if isinstance(arguments, str) else json.dumps(arguments)
arguments if isinstance(arguments, str) else JSONCodec.dumps(arguments)
),
},
}
@@ -4963,7 +4962,7 @@ async def streaming_chat_response_handler(response, ctx):
except Exception as e:
log.debug(e)
return None
tool_call.setdefault('function', {})['arguments'] = json.dumps(params)
tool_call.setdefault('function', {})['arguments'] = JSONCodec.dumps(params)
return params
async def execute_tool_call(tool_call):
backend/open_webui/utils/misc.py
+7 -7
View File
@@ -2,7 +2,6 @@ from __future__ import annotations
import collections.abc
import hashlib
import json
import logging
import re
import threading
@@ -15,6 +14,7 @@ from typing import Callable, Optional, Sequence, Union
import aiohttp
import mimeparse
from open_webui.env import CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE
from open_webui.utils.json_codec import JSONCodec
log = logging.getLogger(__name__)
SURROGATE_RE = re.compile('[\ud800-\udfff]')
@@ -355,7 +355,7 @@ def convert_output_to_messages(
arguments = item.get('arguments', '{}')
# Ensure arguments is always a JSON string
if not isinstance(arguments, str):
arguments = json.dumps(arguments)
arguments = JSONCodec.dumps(arguments)
pending_tool_calls.append(
{
'id': item.get('call_id', ''),
@@ -822,7 +822,7 @@ def sanitize_data_for_db(obj):
# json.dumps is implemented in C and much faster than a Python-level
# recursive walk over every leaf string.
try:
serialized = json.dumps(obj, ensure_ascii=False)
serialized = JSONCodec.dumps(obj, ensure_ascii=False)
if '\\u0000' not in serialized:
serialized.encode('utf-8')
return obj
@@ -854,7 +854,7 @@ def sanitize_metadata(metadata: dict) -> dict:
return None
# Last resort: try to see if it's serializable
try:
json.dumps(obj)
JSONCodec.dumps(obj)
return obj
except (TypeError, ValueError):
return None
@@ -864,7 +864,7 @@ def sanitize_metadata(metadata: dict) -> dict:
if isinstance(obj, (str, int, float, bool, type(None), dict, list)):
return True
try:
json.dumps(obj)
JSONCodec.dumps(obj)
return True
except (TypeError, ValueError):
return False
@@ -1018,7 +1018,7 @@ def convert_logit_bias_input_to_json(logit_bias_input) -> str | None:
return None
if isinstance(logit_bias_input, dict):
return json.dumps(logit_bias_input)
return JSONCodec.dumps(logit_bias_input)
logit_bias_pairs = logit_bias_input.split(',')
logit_bias_json = {}
@@ -1028,7 +1028,7 @@ def convert_logit_bias_input_to_json(logit_bias_input) -> str | None:
bias = int(bias.strip())
bias = 100 if bias > 100 else -100 if bias < -100 else bias
logit_bias_json[token] = bias
return json.dumps(logit_bias_json)
return JSONCodec.dumps(logit_bias_json)
def freeze(value):
backend/open_webui/utils/oauth.py
+7 -7
View File
@@ -2,7 +2,6 @@ import asyncio
import base64
import fnmatch
import hashlib
import json
import logging
import re
import sys
@@ -35,8 +34,8 @@ from mcp.shared.auth import (
)
from open_webui.config import (
DEFAULT_USER_ROLE,
ENABLE_OAUTH_GROUP_CREATION,
ENABLE_OAUTH,
ENABLE_OAUTH_GROUP_CREATION,
ENABLE_OAUTH_GROUP_MANAGEMENT,
ENABLE_OAUTH_ROLE_MANAGEMENT,
ENABLE_OAUTH_SIGNUP,
@@ -67,7 +66,6 @@ from open_webui.config import (
WEBHOOK_URL,
)
from open_webui.constants import ERROR_MESSAGES
from open_webui.events import EVENTS, publish_event
from open_webui.env import (
AIOHTTP_CLIENT_ALLOW_REDIRECTS,
AIOHTTP_CLIENT_SESSION_SSL,
@@ -79,6 +77,7 @@ from open_webui.env import (
WEBUI_AUTH_COOKIE_SAME_SITE,
WEBUI_AUTH_COOKIE_SECURE,
)
from open_webui.events import EVENTS, publish_event
from open_webui.models.auths import Auths
from open_webui.models.config import Config
from open_webui.models.groups import GroupForm, GroupModel, Groups, GroupUpdateForm
@@ -114,6 +113,7 @@ class OAuthClientInformationFull(OAuthClientMetadata):
from open_webui.env import GLOBAL_LOG_LEVEL
from open_webui.utils.json_codec import JSONCodec
logging.basicConfig(stream=sys.stdout, level=GLOBAL_LOG_LEVEL)
log = logging.getLogger(__name__)
@@ -265,7 +265,7 @@ except Exception as e:
def encrypt_data(data) -> str:
"""Encrypt data for storage"""
try:
data_json = json.dumps(data)
data_json = JSONCodec.dumps(data)
encrypted = FERNET.encrypt(data_json.encode()).decode()
return encrypted
except Exception as e:
@@ -277,7 +277,7 @@ def decrypt_data(data: str):
"""Decrypt data from storage"""
try:
decrypted = FERNET.decrypt(data.encode()).decode()
return json.loads(decrypted)
return JSONCodec.loads(decrypted)
except Exception as e:
log.error(f'Error decrypting data: {e}')
raise
@@ -968,7 +968,7 @@ class OAuthClientManager:
content_type = resp.headers.get('content-type', '')
if 'application/json' in content_type:
try:
payload = json.loads(response_text)
payload = JSONCodec.loads(response_text)
error = payload.get('error')
error_description = payload.get('error_description', '')
except Exception:
@@ -1556,7 +1556,7 @@ class OAuthManager:
oauth_claim = auth_config.OAUTH_GROUPS_CLAIM
try:
blocked_groups = json.loads(auth_config.OAUTH_BLOCKED_GROUPS)
blocked_groups = JSONCodec.loads(auth_config.OAUTH_BLOCKED_GROUPS)
except Exception as e:
log.exception(f'Error loading OAUTH_BLOCKED_GROUPS: {e}')
blocked_groups = []
backend/open_webui/utils/payload.py
+9 -9
View File
@@ -1,12 +1,12 @@
import json
from typing import Callable, Optional
from open_webui.utils.chat_variables import render_chat_variables, render_user_variables
from open_webui.utils.json_codec import JSONCodec
from open_webui.utils.misc import (
add_or_update_system_message,
deep_update,
replace_system_message_content,
)
from open_webui.utils.chat_variables import render_chat_variables, render_user_variables
from open_webui.utils.task import prompt_template, prompt_variables_template
@@ -115,8 +115,8 @@ def apply_model_params_to_body_openai(params: dict, form_data: dict) -> dict:
if isinstance(value, str):
try:
# Attempt to parse the string as JSON
custom_params[key] = json.loads(value)
except json.JSONDecodeError:
custom_params[key] = JSONCodec.loads(value)
except JSONCodec.JSONDecodeError:
# If it fails, keep the original string
pass
@@ -149,8 +149,8 @@ def apply_model_params_to_body_ollama(params: dict, form_data: dict) -> dict:
if isinstance(value, str):
try:
# Attempt to parse the string as JSON
custom_params[key] = json.loads(value)
except json.JSONDecodeError:
custom_params[key] = JSONCodec.loads(value)
except JSONCodec.JSONDecodeError:
# If it fails, keep the original string
pass
@@ -198,7 +198,7 @@ def apply_model_params_to_body_ollama(params: dict, form_data: dict) -> dict:
Parses a JSON string into a dictionary, handling potential JSONDecodeError.
"""
try:
return json.loads(value)
return JSONCodec.loads(value)
except Exception as e:
return value
@@ -253,7 +253,7 @@ def convert_messages_openai_to_ollama(messages: list[dict]) -> list[dict]:
'id': tool_call.get('id', None),
'function': {
'name': tool_call.get('function', {}).get('name', ''),
'arguments': json.loads(tool_call.get('function', {}).get('arguments', {})),
'arguments': JSONCodec.loads(tool_call.get('function', {}).get('arguments', {})),
},
}
ollama_tool_calls.append(ollama_tool_call)
@@ -336,7 +336,7 @@ def convert_payload_openai_to_ollama(openai_payload: dict) -> dict:
Parses a JSON string into a dictionary, handling potential JSONDecodeError.
"""
try:
return json.loads(value)
return JSONCodec.loads(value)
except Exception as e:
return value
backend/open_webui/utils/response.py
+1 -2
View File
@@ -1,4 +1,3 @@
import json
from numbers import Number
from uuid import uuid4
@@ -158,7 +157,7 @@ def convert_ollama_tool_call_to_openai(tool_calls: list) -> list:
'type': 'function',
'function': {
'name': function.get('name', ''),
'arguments': json.dumps(function.get('arguments', {})),
'arguments': JSONCodec.dumps(function.get('arguments', {})),
},
}
openai_tool_calls.append(openai_tool_call)
backend/open_webui/utils/subagents.py
+2 -2
View File
@@ -2,7 +2,6 @@ from __future__ import annotations
import asyncio
import copy
import json
import time
from datetime import timedelta
from uuid import uuid4
@@ -16,6 +15,7 @@ from open_webui.models.config import Config
from open_webui.models.users import UserModel, Users
from open_webui.tasks import create_task, has_active_tasks
from open_webui.utils.auth import create_token
from open_webui.utils.json_codec import JSONCodec
from open_webui.utils.misc import get_message_list
from sqlalchemy import select
from starlette.datastructures import Headers
@@ -647,7 +647,7 @@ async def delegate(
return f'Error: {exc}'
if background:
return json.dumps(
return JSONCodec.dumps(
{
'status': 'dispatched',
'delegation_id': delegation_id,
backend/open_webui/utils/timers.py
+4 -5
View File
@@ -4,7 +4,6 @@ from __future__ import annotations
import asyncio
import copy
import json
import logging
import re
import time
@@ -13,15 +12,15 @@ from typing import Literal
from uuid import uuid4
from fastapi import Request
from sqlalchemy import select
from starlette.datastructures import Headers
from open_webui.internal.db import get_async_db
from open_webui.models.chat_messages import ChatMessages
from open_webui.models.chats import Chat, ChatForm, Chats
from open_webui.models.users import UserModel, Users
from open_webui.tasks import has_active_tasks
from open_webui.utils.json_codec import JSONCodec
from open_webui.utils.misc import get_message_list
from sqlalchemy import select
from starlette.datastructures import Headers
log = logging.getLogger(__name__)
@@ -157,7 +156,7 @@ async def create_timer(
if not chat:
return 'Error: failed to create timer.'
return json.dumps(
return JSONCodec.dumps(
{
'status': 'set',
'at': datetime.fromtimestamp(due_at / 1_000_000_000, timezone.utc).isoformat().replace('+00:00', 'Z'),
backend/open_webui/utils/tools.py
+10 -10
View File
@@ -4,7 +4,6 @@ import asyncio
import base64
import copy
import inspect
import json
import logging
import os
import re
@@ -46,7 +45,6 @@ from open_webui.models.config import Config
from open_webui.models.groups import Groups
from open_webui.models.tools import Tools
from open_webui.models.users import UserModel
from open_webui.utils.chat_id import is_saved_chat_id
from open_webui.tools.builtin import (
add_memory,
calculate_timestamp,
@@ -65,8 +63,8 @@ from open_webui.tools.builtin import (
grep_chat_files,
grep_knowledge_files,
kb_exec,
list_chat_files,
list_automations,
list_chat_files,
list_knowledge,
list_knowledge_bases,
list_memories,
@@ -103,7 +101,9 @@ from open_webui.tools.builtin import (
write_note,
)
from open_webui.utils.access_control import has_access, has_connection_access, has_permission
from open_webui.utils.chat_id import is_saved_chat_id
from open_webui.utils.headers import get_custom_headers, include_user_info_headers
from open_webui.utils.json_codec import JSONCodec
from open_webui.utils.misc import is_string_allowed
from open_webui.utils.plugin import get_tool_contents_cache, get_tools_cache, load_tool_module_by_id
from open_webui.utils.terminals import get_terminal_server_url
@@ -1141,7 +1141,7 @@ async def set_tool_servers(request: Request):
try:
if request.app.state.redis is not None:
await request.app.state.redis.set(
f'{REDIS_KEY_PREFIX}:tool_servers', json.dumps(request.app.state.TOOL_SERVERS)
f'{REDIS_KEY_PREFIX}:tool_servers', JSONCodec.dumps(request.app.state.TOOL_SERVERS)
)
except Exception as e:
log.error(f'Error caching tool_servers to Redis: {e}')
@@ -1154,7 +1154,7 @@ async def get_tool_servers(request: Request):
tool_servers = []
if request.app.state.redis is not None:
try:
tool_servers = json.loads(await request.app.state.redis.get(f'{REDIS_KEY_PREFIX}:tool_servers'))
tool_servers = JSONCodec.loads(await request.app.state.redis.get(f'{REDIS_KEY_PREFIX}:tool_servers'))
request.app.state.TOOL_SERVERS = tool_servers
except Exception as e:
log.error(f'Error fetching tool_servers from Redis: {e}')
@@ -1286,7 +1286,7 @@ async def set_terminal_servers(request: Request):
if request.app.state.redis is not None:
await request.app.state.redis.set(
f'{REDIS_KEY_PREFIX}:terminal_servers', json.dumps(request.app.state.TERMINAL_SERVERS)
f'{REDIS_KEY_PREFIX}:terminal_servers', JSONCodec.dumps(request.app.state.TERMINAL_SERVERS)
)
return request.app.state.TERMINAL_SERVERS
@@ -1297,7 +1297,7 @@ async def get_terminal_servers(request: Request):
terminal_servers = []
if request.app.state.redis is not None:
try:
terminal_servers = json.loads(await request.app.state.redis.get(f'{REDIS_KEY_PREFIX}:terminal_servers'))
terminal_servers = JSONCodec.loads(await request.app.state.redis.get(f'{REDIS_KEY_PREFIX}:terminal_servers'))
request.app.state.TERMINAL_SERVERS = terminal_servers
except Exception as e:
log.error(f'Error fetching terminal_servers from Redis: {e}')
@@ -1434,8 +1434,8 @@ async def get_tool_server_data(url: str, headers: dict | None) -> dict[str, Any]
res = yaml.safe_load(text_content)
else:
try:
res = json.loads(text_content)
except json.JSONDecodeError:
res = JSONCodec.loads(text_content)
except JSONCodec.JSONDecodeError:
# Fall back to YAML for non-.yml URLs that aren't valid JSON
res = yaml.safe_load(text_content)
@@ -1491,7 +1491,7 @@ async def get_tool_servers_data(servers: list[dict[str, Any]]) -> list[dict[str,
# Use provided JSON spec
spec_json = None
try:
spec_json = json.loads(server.get('spec', ''))
spec_json = JSONCodec.loads(server.get('spec', ''))
except Exception as e:
log.error(f'Error parsing JSON spec for tool server {id}: {e}')
backend/open_webui/utils/valves.py
+4 -4
View File
@@ -1,11 +1,11 @@
import base64
import hashlib
import json
import logging
from functools import lru_cache
from cryptography.fernet import Fernet, InvalidToken
from open_webui.env import ENABLE_VALVE_ENCRYPTION, WEBUI_SECRET_KEY
from open_webui.utils.json_codec import JSONCodec
log = logging.getLogger(__name__)
@@ -21,7 +21,7 @@ def _fernet() -> Fernet:
def encrypt_valves(valves: dict) -> dict | str:
if not ENABLE_VALVE_ENCRYPTION:
return valves
return _fernet().encrypt(json.dumps(valves).encode()).decode()
return _fernet().encrypt(JSONCodec.dumps(valves).encode()).decode()
def decrypt_valves(valves) -> dict:
@@ -33,8 +33,8 @@ def decrypt_valves(valves) -> dict:
return {}
try:
decrypted = json.loads(_fernet().decrypt(valves.encode()).decode())
except (InvalidToken, json.JSONDecodeError) as e:
decrypted = JSONCodec.loads(_fernet().decrypt(valves.encode()).decode())
except (InvalidToken, JSONCodec.JSONDecodeError) as e:
log.warning('Failed to decrypt valves: %s', type(e).__name__)
return {}
backend/open_webui/utils/webhook.py
+2 -2
View File
@@ -1,5 +1,4 @@
import asyncio
import json
import logging
from open_webui.config import WEBUI_FAVICON_URL
@@ -9,6 +8,7 @@ from open_webui.env import (
VERSION,
)
from open_webui.retrieval.web.utils import get_ssrf_safe_session, validate_url
from open_webui.utils.json_codec import JSONCodec
log = logging.getLogger(__name__)
@@ -54,7 +54,7 @@ async def post_webhook(name: str, url: str, message: str, event_data: dict, desc
if isinstance(user_data, dict):
user_dict = user_data
else:
user_dict = json.loads(user_data)
user_dict = JSONCodec.loads(user_data)
facts = [{'name': key, 'value': value} for key, value in user_dict.items()]
if event_data.get('event'):
facts.insert(0, {'name': 'event', 'value': event_data.get('event')})