fix: keep dynamic model switching and agent catalogs reliable (#114760)

* fix: keep dynamic model switching and agent catalogs reliable

* test: use schema-valid dynamic model regression fixtures

* test: align utility completion with configured model fallback
This commit is contained in:
Peter Steinberger
2026-07-27 18:30:40 -04:00
committed by GitHub
parent 0073862e23
commit e5e77b656a
19 changed files with 392 additions and 21 deletions
@@ -709,6 +709,15 @@ describe("qa scenario catalog", () => {
);
});
it("binds model switch follow-up assertions to the configured alternate model", () => {
const scenario = requireFlowScenario(readQaScenarioById("model-switch-follow-up"));
const flow = JSON.stringify(scenario.execution.flow);
expect(flow).toContain("alternate?.model");
expect(flow).toContain("config.followupPrompt");
expect(flow).not.toContain("gpt-5.6-luna-alt");
});
it("keeps provider-sensitive QA flow scenarios on their supported lanes", () => {
const strandedConfig = readQaScenarioExecutionConfig("message-tool-stranded-final-reply") as
| { requiredProviderMode?: string }
@@ -69,8 +69,17 @@ flow:
- lambda:
expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && (() => { const lower = normalizeLowercaseStringOrEmpty(candidate.text); return lower.includes('switch') || lower.includes('handoff'); })()).at(-1)"
- expr: resolveQaLiveTurnTimeoutMs(env, 20000, env.alternateModel)
- assert:
expr: "!env.mock || ((await fetchJson(`${env.mock.baseUrl}/debug/last-request`))?.body?.model === 'gpt-5.6-luna-alt')"
message:
expr: "`expected gpt-5.6-luna-alt, got ${String((await fetchJson(`${env.mock.baseUrl}/debug/last-request`))?.body?.model ?? '')}`"
- if:
expr: "Boolean(env.mock)"
then:
- set: switchDebugRequests
value:
expr: "await fetchJson(`${env.mock.baseUrl}/debug/requests`)"
- set: switchRequest
value:
expr: "switchDebugRequests.find((request) => String(request.allInputText ?? '').includes(config.followupPrompt))"
- assert:
expr: "String(switchRequest?.model ?? '') === String(alternate?.model ?? '')"
message:
expr: "`expected alternate model ${String(alternate?.model ?? '')}, got ${String(switchRequest?.model ?? '')}`"
detailsExpr: outbound.text
@@ -0,0 +1,130 @@
import { describe, expect, it } from "vitest";
import type { OpenClawConfig } from "../config/types.js";
import { resolveConfiguredProviderFallback } from "./configured-provider-fallback.js";
type ModelProviders = NonNullable<NonNullable<OpenClawConfig["models"]>["providers"]>;
type ConfiguredModel = ModelProviders[string]["models"][number];
function configuredProviders(providers: ModelProviders): Pick<OpenClawConfig, "models"> {
return { models: { providers } };
}
function configuredModel(id: string, name: string): ConfiguredModel {
return {
id,
name,
reasoning: false,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 128_000,
maxTokens: 4_096,
};
}
function configuredProvider(
baseUrl: string,
models: ConfiguredModel[] = [],
): ModelProviders[string] {
return { baseUrl, models };
}
const defaultProviderBaseUrl = "https://openai.example.com/v1";
const localProvider = configuredProvider("http://127.0.0.1:9191/v1", [
configuredModel("local-good", "Local Good"),
]);
describe("resolveConfiguredProviderFallback", () => {
it("uses a configured model when the default provider is only an empty overlay", () => {
expect(
resolveConfiguredProviderFallback({
cfg: configuredProviders({
openai: configuredProvider(defaultProviderBaseUrl),
"local-provider": localProvider,
}),
defaultProvider: "openai",
defaultModel: undefined,
}),
).toEqual({ provider: "local-provider", model: "local-good" });
});
it("preserves configured provider order when the default model is absent", () => {
expect(
resolveConfiguredProviderFallback({
cfg: configuredProviders({
openai: configuredProvider(defaultProviderBaseUrl, [
configuredModel("other-openai-model", "Other OpenAI Model"),
]),
"local-provider": localProvider,
}),
defaultProvider: "openai",
defaultModel: "missing-default-model",
}),
).toEqual({ provider: "openai", model: "other-openai-model" });
});
it("recognizes normalized default provider keys", () => {
expect(
resolveConfiguredProviderFallback({
cfg: configuredProviders({
" OpenAI ": configuredProvider(defaultProviderBaseUrl, [
configuredModel("configured-default", "Configured Default"),
]),
"local-provider": localProvider,
}),
defaultProvider: "openai",
defaultModel: "configured-default",
}),
).toBeNull();
});
it("normalizes the selected custom provider without changing its configured model", () => {
expect(
resolveConfiguredProviderFallback({
cfg: configuredProviders({ " Local-Provider ": localProvider }),
defaultProvider: "openai",
defaultModel: undefined,
}),
).toEqual({ provider: "local-provider", model: "local-good" });
});
it("preserves the configured default model when it is available", () => {
expect(
resolveConfiguredProviderFallback({
cfg: configuredProviders({
openai: configuredProvider(defaultProviderBaseUrl, [
configuredModel("configured-default", "Configured Default"),
]),
"local-provider": localProvider,
}),
defaultProvider: "openai",
defaultModel: "configured-default",
}),
).toBeNull();
});
it("preserves configured provider preference order", () => {
expect(
resolveConfiguredProviderFallback({
cfg: configuredProviders({
openai: configuredProvider(defaultProviderBaseUrl),
first: configuredProvider("http://127.0.0.1:9192/v1", [
configuredModel("first-model", "First Model"),
]),
second: localProvider,
}),
defaultProvider: "openai",
defaultModel: undefined,
}),
).toEqual({ provider: "first", model: "first-model" });
});
it("returns no fallback when no provider has a configured model", () => {
expect(
resolveConfiguredProviderFallback({
cfg: configuredProviders({ openai: configuredProvider(defaultProviderBaseUrl) }),
defaultProvider: "openai",
defaultModel: "missing-default-model",
}),
).toBeNull();
});
});
+15 -6
View File
@@ -2,6 +2,10 @@
* Chooses a configured provider/model fallback when defaults are absent from
* the user's model config.
*/
import {
findNormalizedProviderValue,
normalizeProviderId,
} from "@openclaw/model-catalog-core/provider-id";
import type { OpenClawConfig } from "../config/types.js";
type ProviderModelRef = {
@@ -13,20 +17,25 @@ type ProviderModelRef = {
export function resolveConfiguredProviderFallback(params: {
cfg: Pick<OpenClawConfig, "models">;
defaultProvider: string;
defaultModel?: string;
defaultModel: string | undefined;
}): ProviderModelRef | null {
const configuredProviders = params.cfg.models?.providers;
if (!configuredProviders || typeof configuredProviders !== "object") {
return null;
}
const defaultProviderConfig = configuredProviders[params.defaultProvider];
const defaultProviderConfig = findNormalizedProviderValue(
configuredProviders,
params.defaultProvider,
);
const defaultModel = params.defaultModel?.trim();
const defaultProviderHasConfiguredModel =
Array.isArray(defaultProviderConfig?.models) &&
defaultProviderConfig.models.some((model) => Boolean(model?.id));
const defaultProviderHasDefaultModel =
defaultProviderConfig !== undefined &&
defaultModel !== undefined &&
Array.isArray(defaultProviderConfig.models) &&
Array.isArray(defaultProviderConfig?.models) &&
defaultProviderConfig.models.some((model) => model?.id === defaultModel);
if (defaultProviderConfig && (!defaultModel || defaultProviderHasDefaultModel)) {
if (defaultProviderHasConfiguredModel && (!defaultModel || defaultProviderHasDefaultModel)) {
return null;
}
// Fall back to the first provider with at least one configured model, preserving
@@ -46,5 +55,5 @@ export function resolveConfiguredProviderFallback(params: {
if (!Array.isArray(models) || !models[0]?.id) {
return null;
}
return { provider, model: models[0].id };
return { provider: normalizeProviderId(provider), model: models[0].id };
}
@@ -626,6 +626,7 @@ export async function loadCompactHooksHarness(): Promise<{
configFingerprint: undefined,
compatiblePolicyHashes: undefined,
compatibleConfigFingerprints: undefined,
configIdentities: new WeakSet(),
})),
getCurrentPluginMetadataSnapshot: () => emptyPluginMetadataSnapshot,
resolvePluginMetadataControlPlaneFingerprint: vi.fn(() => "test-plugin-fingerprint"),
+1
View File
@@ -983,6 +983,7 @@ export function resolveConfiguredModelRef(
const fallbackProvider = resolveConfiguredProviderFallback({
cfg: params.cfg,
defaultProvider: params.defaultProvider,
defaultModel: params.defaultModel,
});
if (fallbackProvider) {
return fallbackProvider;
+32
View File
@@ -2373,6 +2373,38 @@ describe("model-selection", () => {
expect(result).toEqual({ provider: "n1n", model: "gpt-5.4" });
});
it("uses a configured custom provider when the default is only an empty overlay", () => {
const cfg = {
models: {
providers: {
openai: { baseUrl: "https://openai.example.com/v1", models: [] },
"local-provider": {
baseUrl: "http://127.0.0.1:9191/v1",
models: [
{
id: "local-good",
name: "Local Good",
reasoning: false,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 128_000,
maxTokens: 4_096,
},
],
},
},
},
} as OpenClawConfig;
expect(
resolveConfiguredModelRef({
cfg,
defaultProvider: "openai",
defaultModel: "missing-default-model",
}),
).toEqual({ provider: "local-provider", model: "local-good" });
});
it("should keep default provider when it is in models.providers", () => {
const cfg = createProviderWithModelsConfig("anthropic", [
{
@@ -198,7 +198,7 @@ describe("resolveSimpleCompletionSelectionForAgent", () => {
expect(selection.modelId).toBe("gpt-5.6-sol");
});
it("uses configured provider fallback when default provider is unavailable", () => {
it("uses the configured provider model when the runtime default is unavailable", () => {
const cfg = {
models: {
providers: {
@@ -229,6 +229,6 @@ describe("resolveSimpleCompletionSelectionForAgent", () => {
resolveSimpleCompletionSelectionForAgent({ cfg, agentId: "main" }),
);
expect(selection.provider).toBe("openai");
expect(selection.modelId).toBe("gpt-5.6-sol");
expect(selection.modelId).toBe("gpt-5");
});
});
@@ -411,6 +411,7 @@ export function resolveImplicitDefaultAgentModelRef(cfg: OpenClawConfig): string
const fallbackProvider = resolveConfiguredProviderFallback({
cfg,
defaultProvider: DEFAULT_PROVIDER,
defaultModel: DEFAULT_MODEL,
});
return fallbackProvider
? normalizeProviderModelRef(fallbackProvider.provider, fallbackProvider.model)
+1
View File
@@ -92,6 +92,7 @@ function resolveConfiguredStatusModelRef(params: {
const fallbackProvider = resolveConfiguredProviderFallback({
cfg: params.cfg,
defaultProvider: params.defaultProvider,
defaultModel: params.defaultModel,
});
if (fallbackProvider) {
return fallbackProvider;
@@ -0,0 +1,22 @@
import { describe, expect, it, vi } from "vitest";
import { loadOptionalServerMethodModelCatalog } from "./optional-model-catalog.js";
import type { GatewayRequestContext } from "./types.js";
describe("loadOptionalServerMethodModelCatalog", () => {
it("forwards the requested agent to the catalog owner", async () => {
const entries = [{ id: "work-only", name: "Work Model", provider: "work-provider" }];
const loadGatewayModelCatalog = vi.fn(async () => entries);
const context = {
loadGatewayModelCatalog,
logGateway: { debug: vi.fn() },
} as unknown as GatewayRequestContext;
await expect(
loadOptionalServerMethodModelCatalog(context, "sessions.list", {
loadParams: { agentId: "work" },
}),
).resolves.toEqual(entries);
expect(loadGatewayModelCatalog).toHaveBeenCalledWith({ agentId: "work" });
});
});
@@ -59,9 +59,11 @@ function startOptionalServerMethodModelCatalogValueLoad<T>(params: {
function startOptionalServerMethodModelCatalogLoad(
context: GatewayRequestContext,
loadParams?: Parameters<GatewayRequestContext["loadGatewayModelCatalog"]>[0],
): OptionalServerMethodModelCatalogLoad<ModelCatalogEntry[]> {
return startOptionalServerMethodModelCatalogValueLoad({
load: () => context.loadGatewayModelCatalog(),
load: () =>
loadParams ? context.loadGatewayModelCatalog(loadParams) : context.loadGatewayModelCatalog(),
normalize: normalizeOptionalModelCatalog,
});
}
@@ -117,7 +119,7 @@ export async function loadOptionalServerMethodModelCatalog(
options?: LoadOptionalServerMethodModelCatalogOptions<ModelCatalogEntry[]>,
): Promise<ModelCatalogEntry[] | undefined> {
return await loadOptionalServerMethodModelCatalogValue(context, surface, options, () =>
startOptionalServerMethodModelCatalogLoad(context),
startOptionalServerMethodModelCatalogLoad(context, options?.loadParams),
);
}
@@ -307,7 +307,7 @@ export const sessionCreateHandlers: GatewayRequestHandlers = {
});
const selection = resolveSessionPatchModelSelection({
cfg,
catalog: await context.loadGatewayModelCatalog(),
catalog: await context.loadGatewayModelCatalog({ agentId: target.agentId }),
raw: requestedTitleModel,
defaultProvider: defaultModel.provider,
defaultModel: defaultModel.model,
@@ -376,6 +376,11 @@ export const sessionCreateHandlers: GatewayRequestHandlers = {
ADMIN_SCOPE,
clientScopes,
).allowed;
const modelCatalogAgentId = normalizeAgentId(
sessionAgentId ??
parseAgentSessionKey(sessionKey ?? "")?.agentId ??
resolveDefaultAgentId(cfg),
);
const created = await createGatewaySession({
cfg,
key: sessionKey,
@@ -410,7 +415,8 @@ export const sessionCreateHandlers: GatewayRequestHandlers = {
commandSource: "webchat",
creation: resolveOperatorSessionCreation(client, { allowTrustedHint: true }),
authorizedPluginId: normalizeOptionalString(client?.internal?.pluginRuntimeOwnerId),
loadGatewayModelCatalog: context.loadGatewayModelCatalog,
loadGatewayModelCatalog: () =>
context.loadGatewayModelCatalog({ agentId: modelCatalogAgentId }),
afterCreate: hasInitialTurn
? async ({ key, agentId, entry, storePath }) => {
messageSeq =
@@ -122,7 +122,7 @@ export const sessionMutationHandlers: GatewayRequestHandlers = {
}
let patchModelCatalog: Awaited<ReturnType<typeof context.loadGatewayModelCatalog>> | undefined;
const loadPatchModelCatalog = async () => {
const catalog = await context.loadGatewayModelCatalog();
const catalog = await context.loadGatewayModelCatalog({ agentId: target.agentId });
patchModelCatalog = catalog;
return catalog;
};
+6 -1
View File
@@ -262,7 +262,12 @@ export const sessionReadHandlers: GatewayRequestHandlers = {
: (durableStorePath ?? storePath);
const modelCatalog = await measureDiagnosticsTimelineSpan(
"gateway.sessions.list.model_catalog",
() => loadOptionalServerMethodModelCatalog(context, "sessions.list"),
() =>
loadOptionalServerMethodModelCatalog(
context,
"sessions.list",
p.agentId ? { loadParams: { agentId: p.agentId } } : undefined,
),
{
config: cfg,
phase: "sessions.list",
@@ -0,0 +1,97 @@
import { afterEach, expect, test, vi } from "vitest";
import { loadSessionEntry } from "../config/sessions/session-accessor.js";
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
import { writeSessionStore } from "./test-helpers.js";
import {
directSessionReq,
sessionStoreEntry,
setupGatewaySessionsTestHarness,
} from "./test/server-sessions.test-helpers.js";
const { createSelectedGlobalSessionStore } = setupGatewaySessionsTestHarness();
const mainModel = { id: "main-only", name: "Main Model", provider: "main-provider" };
const workModel = { id: "work-only", name: "Work Model", provider: "work-provider" };
function createAgentModelCatalogLoader() {
return vi.fn(async (params?: { agentId?: string }) =>
params?.agentId === "work" ? [workModel] : [mainModel],
);
}
afterEach(() => {
closeOpenClawStateDatabaseForTest();
});
test.each([
{ label: "explicit agent", agentId: "work" },
{ label: "agent-qualified session", agentId: undefined },
])("sessions.patch loads the $label model catalog", async ({ agentId }) => {
const { workStorePath } = await createSelectedGlobalSessionStore();
const key = "agent:work:dashboard:catalog-owner-patch";
await writeSessionStore({
agentId: "work",
storePath: workStorePath,
entries: { [key]: sessionStoreEntry("work-catalog-patch") },
});
const loadGatewayModelCatalog = createAgentModelCatalogLoader();
const patched = await directSessionReq<{
entry?: { modelOverride?: string; providerOverride?: string };
}>(
"sessions.patch",
{
key,
...(agentId ? { agentId } : {}),
model: "work-provider/work-only",
},
{ context: { loadGatewayModelCatalog } },
);
expect(patched.ok, patched.error?.message).toBe(true);
expect(loadGatewayModelCatalog).toHaveBeenCalledWith({ agentId: "work" });
expect(patched.payload?.entry).toMatchObject({
providerOverride: "work-provider",
modelOverride: "work-only",
});
expect(
loadSessionEntry({ agentId: "work", sessionKey: key, storePath: workStorePath }),
).toMatchObject({
providerOverride: "work-provider",
modelOverride: "work-only",
});
});
test.each([
{ label: "explicit agent", agentId: "work" },
{ label: "agent-qualified session", agentId: undefined },
])("sessions.create loads the $label model catalog", async ({ agentId }) => {
const { workStorePath } = await createSelectedGlobalSessionStore();
const key = `agent:work:dashboard:catalog-owner-create-${agentId ? "explicit" : "key"}`;
const loadGatewayModelCatalog = createAgentModelCatalogLoader();
const created = await directSessionReq<{
entry?: { modelOverride?: string; providerOverride?: string };
}>(
"sessions.create",
{
key,
...(agentId ? { agentId } : {}),
model: "work-provider/work-only",
},
{ context: { loadGatewayModelCatalog } },
);
expect(created.ok, created.error?.message).toBe(true);
expect(loadGatewayModelCatalog).toHaveBeenCalledWith({ agentId: "work" });
expect(created.payload?.entry).toMatchObject({
providerOverride: "work-provider",
modelOverride: "work-only",
});
expect(
loadSessionEntry({ agentId: "work", sessionKey: key, storePath: workStorePath }),
).toMatchObject({
providerOverride: "work-provider",
modelOverride: "work-only",
});
});
@@ -341,6 +341,39 @@ describe("current plugin metadata snapshot", () => {
expect(getCurrentPluginMetadataSnapshot({ config: secondConfig })).toBeUndefined();
});
it("restores exact config identity across a temporary metadata snapshot", () => {
const config = { plugins: { load: { paths: ["~/plugins"] } } };
const snapshot = createSnapshot({ config });
const originalEnv = {
HOME: "/home/original-snapshot",
OPENCLAW_HOME: undefined,
} as NodeJS.ProcessEnv;
const changedEnv = {
HOME: "/home/changed-snapshot",
OPENCLAW_HOME: undefined,
} as NodeJS.ProcessEnv;
setCurrentPluginMetadataSnapshot(snapshot, { config, env: originalEnv });
const captured = captureCurrentPluginMetadataSnapshotState();
setCurrentPluginMetadataSnapshot(createSnapshot());
restoreCurrentPluginMetadataSnapshotState(captured);
expect(getCurrentPluginMetadataSnapshot({ config, env: changedEnv })).toBe(snapshot);
});
it("restores exact config identity after in-place changes", () => {
const config = { plugins: { allow: ["first"] } };
const snapshot = createSnapshot({ config });
setCurrentPluginMetadataSnapshot(snapshot, { config });
const captured = captureCurrentPluginMetadataSnapshotState();
setCurrentPluginMetadataSnapshot(createSnapshot());
restoreCurrentPluginMetadataSnapshotState(captured);
config.plugins.allow = ["changed"];
expect(getCurrentPluginMetadataSnapshot({ config })).toBe(snapshot);
});
it("clears the current snapshot when the persisted installed index changes", () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-plugin-metadata-"));
try {
@@ -17,7 +17,11 @@ import type {
} from "./plugin-metadata-snapshot.types.js";
import { normalizePluginIdScope, serializePluginIdScope } from "./plugin-scope.js";
type CurrentPluginMetadataSnapshotState = ReturnType<typeof getCurrentPluginMetadataSnapshotState>;
type CurrentPluginMetadataSnapshotState = ReturnType<
typeof getCurrentPluginMetadataSnapshotState
> & {
configIdentities: WeakSet<OpenClawConfig>;
};
function resolvePluginMetadataControlPlaneFingerprint(
config?: OpenClawConfig,
@@ -106,13 +110,16 @@ export function setCurrentPluginMetadataSnapshot(
}
export function captureCurrentPluginMetadataSnapshotState(): CurrentPluginMetadataSnapshotState {
return getCurrentPluginMetadataSnapshotState();
return {
...getCurrentPluginMetadataSnapshotState(),
configIdentities: currentPluginMetadataConfigIdentityCache.capture(),
};
}
export function restoreCurrentPluginMetadataSnapshotState(
state: CurrentPluginMetadataSnapshotState,
): void {
currentPluginMetadataConfigIdentityCache.clear();
currentPluginMetadataConfigIdentityCache.restore(state.configIdentities);
const snapshot = state.snapshot as PluginMetadataSnapshot | undefined;
const defaultDiscoveryConfigFingerprint = snapshot
? resolvePluginMetadataControlPlaneFingerprint(
@@ -13,12 +13,18 @@ export const currentPluginMetadataConfigIdentityCache = {
add(config: OpenClawConfig): void {
currentPluginMetadataConfigIdentities.add(config);
},
capture(): WeakSet<OpenClawConfig> {
return currentPluginMetadataConfigIdentities;
},
clear(): void {
currentPluginMetadataConfigIdentities = new WeakSet();
},
has(config: OpenClawConfig): boolean {
return currentPluginMetadataConfigIdentities.has(config);
},
restore(identities: WeakSet<OpenClawConfig>): void {
currentPluginMetadataConfigIdentities = identities;
},
};
/** Stores the process-current plugin metadata snapshot and compatible config fingerprints. */