From 4fe2de78643c2213652190d2820f4e8d9f4f89cc Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Sat, 9 May 2026 01:13:16 +0900 Subject: [PATCH] refac --- backend/open_webui/main.py | 100 ++++++++++++++++++ backend/open_webui/routers/openai.py | 43 ++++++++ backend/open_webui/utils/models.py | 3 + src/lib/apis/index.ts | 33 ++++++ src/lib/components/AddConnectionModal.svelte | 32 +++--- .../chat/ModelSelector/ModelItem.svelte | 41 +++---- .../chat/ModelSelector/Selector.svelte | 3 +- src/routes/+layout.svelte | 3 +- 8 files changed, 222 insertions(+), 36 deletions(-) diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index af570af0af..ba503b40a9 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -1524,6 +1524,106 @@ async def get_base_models(request: Request, user=Depends(get_admin_user)): return {'data': models} +class ModelUnloadForm(BaseModel): + model: str + + +@app.post('/api/models/unload') +async def unload_model(request: Request, form_data: ModelUnloadForm, user=Depends(get_admin_user)): + """ + Unified model unload endpoint. + Resolves the provider that owns the model and calls its native unload mechanism. + Supports: Ollama (keep_alive=0) and llama.cpp (/models/unload). + """ + model_id = form_data.model + + # --- Ollama provider --- + ollama_models = getattr(request.app.state, 'OLLAMA_MODELS', None) or {} + if model_id in ollama_models: + url_indices = ollama_models[model_id].get('urls', []) + errors = [] + for idx in url_indices: + url = request.app.state.config.OLLAMA_BASE_URLS[idx] + api_config = request.app.state.config.OLLAMA_API_CONFIGS.get( + str(idx), + request.app.state.config.OLLAMA_API_CONFIGS.get(url, {}), + ) + key = api_config.get('key', None) + + prefix_id = api_config.get('prefix_id', None) + actual_model = model_id + if prefix_id and actual_model.startswith(f'{prefix_id}.'): + actual_model = actual_model[len(f'{prefix_id}.'):] + + payload = json.dumps({'model': actual_model, 'keep_alive': 0, 'prompt': ''}) + + try: + timeout = aiohttp.ClientTimeout(total=30) + async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: + headers = { + 'Content-Type': 'application/json', + **({"Authorization": f"Bearer {key}"} if key else {}), + } + async with session.post( + f'{url}/api/generate', + data=payload, + headers=headers, + ) as r: + if not r.ok: + errors.append({'url_idx': idx, 'error': await r.text()}) + except Exception as e: + log.exception(f'Failed to unload model on Ollama node {idx}: {e}') + errors.append({'url_idx': idx, 'error': str(e)}) + + if errors: + raise HTTPException( + status_code=500, + detail=f'Failed to unload model on {len(errors)} node(s): {errors}', + ) + return {'status': True} + + # --- OpenAI-compatible providers --- + openai_models = getattr(request.app.state, 'OPENAI_MODELS', None) or {} + if model_id in openai_models: + model_info = openai_models[model_id] + idx = model_info.get('urlIdx') + api_config = request.app.state.config.OPENAI_API_CONFIGS.get(str(idx), {}) + provider = api_config.get('provider', '') + base_url = request.app.state.config.OPENAI_API_BASE_URLS[idx] + key = request.app.state.config.OPENAI_API_KEYS[idx] if idx < len(request.app.state.config.OPENAI_API_KEYS) else '' + + if provider == 'llama.cpp': + root_url = base_url.rstrip('/').removesuffix('/v1') + try: + timeout = aiohttp.ClientTimeout(total=30) + async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: + headers = { + 'Content-Type': 'application/json', + **({"Authorization": f"Bearer {key}"} if key else {}), + } + async with session.post( + f'{root_url}/models/unload', + json={'model': model_id}, + headers=headers, + ) as r: + if not r.ok: + detail = await r.text() + raise HTTPException(status_code=r.status, detail=detail) + return await r.json() + except HTTPException: + raise + except Exception as e: + log.exception(f'Failed to unload model via llama.cpp: {e}') + raise HTTPException(status_code=500, detail=str(e)) + else: + raise HTTPException( + status_code=400, + detail=f'Provider "{provider or "default"}" does not support model unloading', + ) + + raise HTTPException(status_code=404, detail=f'Model "{model_id}" not found') + + ################################## # Embeddings ################################## diff --git a/backend/open_webui/routers/openai.py b/backend/open_webui/routers/openai.py index 6f8c0f81bf..ab2eec9527 100644 --- a/backend/open_webui/routers/openai.py +++ b/backend/open_webui/routers/openai.py @@ -439,6 +439,7 @@ async def get_all_models_responses(request: Request, user: UserModel) -> list: connection_type = api_config.get('connection_type', 'external') prefix_id = api_config.get('prefix_id', None) tags = api_config.get('tags', []) + provider = api_config.get('provider', '') model_list = response if isinstance(response, list) else response.get('data', []) if not isinstance(model_list, list): @@ -459,6 +460,9 @@ async def get_all_models_responses(request: Request, user: UserModel) -> list: if connection_type: model['connection_type'] = connection_type + if provider: + model['provider'] = provider + log.debug(f'get_all_models:responses() {responses}') return responses @@ -488,6 +492,41 @@ async def get_filtered_models(models, user, db=None): return filtered_models +async def get_openai_loaded_models(request: Request, models: dict, api_base_urls: list): + """ + Fetch loaded-model state from providers that expose it and annotate + each model dict with a ``loaded`` boolean. + + Currently supports: + - **llama.cpp** – queries ``GET /slots`` and matches slot model IDs. + """ + api_configs = request.app.state.config.OPENAI_API_CONFIGS + api_keys = request.app.state.config.OPENAI_API_KEYS + + for idx, url in enumerate(api_base_urls): + api_config = api_configs.get( + str(idx), + api_configs.get(url, {}), + ) + provider = api_config.get('provider', '') + + if provider == 'llama.cpp': + try: + root_url = url.rstrip('/').removesuffix('/v1') + key = api_keys[idx] if idx < len(api_keys) else None + slots = await send_get_request(url=f'{root_url}/slots', key=key) + loaded_model_ids = ( + {s.get('model') for s in slots if s.get('model')} + if isinstance(slots, list) + else set() + ) + for model_id, model in models.items(): + if model.get('urlIdx') == idx: + model['loaded'] = model_id in loaded_model_ids + except Exception as e: + log.debug(f'Failed to fetch llama.cpp slots for idx {idx}: {e}') + + @cached( ttl=MODELS_CACHE_TTL, key=lambda _, user: f'openai_all_models_{user.id}' if user else 'openai_all_models', @@ -548,6 +587,7 @@ async def get_all_models(request: Request, user: UserModel) -> dict[str, list]: 'owned_by': 'openai', 'openai': model, 'connection_type': model.get('connection_type', 'external'), + 'provider': model.get('provider', ''), 'urlIdx': idx, } @@ -556,6 +596,9 @@ async def get_all_models(request: Request, user: UserModel) -> dict[str, list]: models = get_merged_models(map(extract_data, responses)) log.debug(f'models: {models}') + # Fetch loaded state for providers that support it (e.g. llama.cpp /slots) + await get_openai_loaded_models(request, models, api_base_urls) + request.app.state.OPENAI_MODELS = models return {'data': list(models.values())} diff --git a/backend/open_webui/utils/models.py b/backend/open_webui/utils/models.py index cc8c5fad3a..e9201bb621 100644 --- a/backend/open_webui/utils/models.py +++ b/backend/open_webui/utils/models.py @@ -47,6 +47,7 @@ async def fetch_ollama_models(request: Request, user: UserModel = None): 'created': int(time.time()), 'owned_by': 'ollama', 'ollama': model, + 'loaded': 'expires_at' in model, 'connection_type': model.get('connection_type', 'local'), 'tags': model.get('tags', []), } @@ -199,6 +200,8 @@ async def get_all_models(request, refresh: bool = False, user: UserModel = None) 'connection_type': connection_type, 'preset': True, **({'pipe': pipe} if pipe is not None else {}), + **({'provider': base_model.get('provider')} if base_model and base_model.get('provider') else {}), + **({'loaded': base_model.get('loaded')} if base_model and base_model.get('loaded') is not None else {}), } info = custom_model.model_dump() diff --git a/src/lib/apis/index.ts b/src/lib/apis/index.ts index 5faf56d4d4..833979ada0 100644 --- a/src/lib/apis/index.ts +++ b/src/lib/apis/index.ts @@ -159,6 +159,39 @@ export const getModels = async ( return models; }; +export const unloadModel = async (token: string, model: string) => { + let error = null; + + const res = await fetch(`${WEBUI_BASE_URL}/api/models/unload`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }) + }, + body: JSON.stringify({ model }) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.error(err); + if ('detail' in err) { + error = err.detail; + } else { + error = err; + } + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + type ChatCompletedForm = { model: string; messages: Record[]; diff --git a/src/lib/components/AddConnectionModal.svelte b/src/lib/components/AddConnectionModal.svelte index ae1d353642..d1f04f3e0a 100644 --- a/src/lib/components/AddConnectionModal.svelte +++ b/src/lib/components/AddConnectionModal.svelte @@ -36,9 +36,10 @@ let auth_type = 'bearer'; let connectionType = 'external'; - let azure = false; + let provider = ''; $: azure = - (url.includes('azure.') || url.includes('cognitive.microsoft.com')) && !direct ? true : false; + provider === 'azure' || + ((url.includes('azure.') || url.includes('cognitive.microsoft.com')) && !direct && provider === ''); let prefixId = ''; let enable = true; @@ -98,7 +99,7 @@ key, config: { auth_type, - azure: azure, + ...(provider ? { provider } : azure ? { azure: true } : {}), api_version: apiVersion, ...(_headers ? { headers: _headers } : {}) } @@ -186,7 +187,8 @@ connection_type: connectionType, auth_type, headers: headers ? JSON.parse(headers) : undefined, - ...(!ollama && azure ? { azure: true, api_version: apiVersion } : {}), + ...(provider ? { provider } : !ollama && azure ? { azure: true } : {}), + ...(azure ? { api_version: apiVersion } : {}), ...(apiType ? { api_type: apiType } : {}) } }; @@ -223,7 +225,7 @@ connectionType = connection.config?.connection_type ?? 'local'; } else { connectionType = connection.config?.connection_type ?? 'external'; - azure = connection.config?.azure ?? false; + provider = connection.config?.provider ?? (connection.config?.azure ? 'azure' : ''); apiVersion = connection.config?.api_version ?? ''; apiType = connection.config?.api_type ?? ''; } @@ -491,22 +493,22 @@ {#if !ollama && !direct}
{$i18n.t('Provider')}
-
{/if} diff --git a/src/lib/components/chat/ModelSelector/ModelItem.svelte b/src/lib/components/chat/ModelSelector/ModelItem.svelte index cd5fe453c3..8f46619868 100644 --- a/src/lib/components/chat/ModelSelector/ModelItem.svelte +++ b/src/lib/components/chat/ModelSelector/ModelItem.svelte @@ -120,25 +120,28 @@ {/if} - {#if item.model.ollama?.expires_at && new Date(item.model.ollama?.expires_at * 1000) > new Date()} -
- + new Date() + ? `${$i18n.t('Unloads {{FROM_NOW}}', { FROM_NOW: dayjs(item.model.ollama?.expires_at * 1000).fromNow() - })}`} - className="self-end" - > -
- - - - -
- -
- {/if} + })}` + : `${$i18n.t('Loaded')}`} + className="self-end" + > +
+ + + + +
+ + {/if} @@ -233,7 +236,7 @@
- {#if $user?.role === 'admin' && item.model.owned_by === 'ollama' && item.model.ollama?.expires_at && new Date(item.model.ollama?.expires_at * 1000) > new Date()} + {#if $user?.role === 'admin' && item.model.loaded}