diff --git a/extensions/openai/provider-policy-api.test.ts b/extensions/openai/provider-policy-api.test.ts index e1f32970e3ed..c98df08011b2 100644 --- a/extensions/openai/provider-policy-api.test.ts +++ b/extensions/openai/provider-policy-api.test.ts @@ -1,6 +1,7 @@ // Openai tests cover provider policy api plugin behavior. import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { + isResponseModelEquivalent, normalizeModelCatalogId, resolveModelRoutes, resolveThinkingProfile, @@ -15,6 +16,22 @@ describe("OpenAI provider policy artifact", () => { vi.unstubAllEnvs(); }); + it.each([ + ["openai", "gpt-5.6", "gpt-5.6-sol", true], + ["openai", "gpt-5.6", "gpt-5.6-terra", false], + ["openai", "gpt-5.6", "gpt-5.6-luna", false], + ["openai", "gpt-5.6-sol", "gpt-5.6", false], + ["openai", "gpt-5.5", "gpt-5.6-sol", false], + ["anthropic", "gpt-5.6", "gpt-5.6-sol", false], + ])( + "declares response-model equivalence for %s/%s -> %s as %s", + (provider, requestedModelId, responseModelId, expected) => { + expect(isResponseModelEquivalent({ provider, requestedModelId, responseModelId })).toBe( + expected, + ); + }, + ); + it("normalizes the legacy Codex model alias at the provider boundary", () => { expect(normalizeModelCatalogId({ provider: " OpenAI ", modelId: "openai/GPT-5.4-CODEX" })).toBe( "gpt-5.4", diff --git a/extensions/openai/provider-policy-api.ts b/extensions/openai/provider-policy-api.ts index 1e394f600674..79c2ef507d67 100644 --- a/extensions/openai/provider-policy-api.ts +++ b/extensions/openai/provider-policy-api.ts @@ -8,6 +8,7 @@ import type { ProviderModelRouteResolution, ProviderModelRouteSource, ProviderNormalizeModelCatalogIdContext, + ProviderResponseModelEquivalenceContext, ProviderResolveModelRoutesContext, } from "openclaw/plugin-sdk/provider-model-types"; import { @@ -21,6 +22,8 @@ import { isOpenAIPlatformOnlyRouteModelId, isOpenAISubscriptionOnlyRouteModelId, normalizeOpenAIModelRouteId, + OPENAI_GPT_56_MODEL_ID, + OPENAI_GPT_56_SOL_MODEL_ID, } from "./model-route-contract.js"; import { resolveUnifiedOpenAIThinkingProfile } from "./thinking-policy.js"; @@ -55,6 +58,14 @@ export function normalizeModelCatalogId(params: ProviderNormalizeModelCatalogIdC : null; } +export function isResponseModelEquivalent(params: ProviderResponseModelEquivalenceContext) { + return ( + params.provider.trim().toLowerCase() === OPENAI_PROVIDER_ID && + params.requestedModelId === OPENAI_GPT_56_MODEL_ID && + params.responseModelId === OPENAI_GPT_56_SOL_MODEL_ID + ); +} + /** Resolves authored OpenAI provider config without activating the runtime plugin. */ export function resolveAuthoredOpenAIProviderConfig(params: { provider: string; diff --git a/src/gateway/server-methods/system-agent-greeting-welcome.test.ts b/src/gateway/server-methods/system-agent-greeting-welcome.test.ts index 85ece0786a10..b37292753278 100644 --- a/src/gateway/server-methods/system-agent-greeting-welcome.test.ts +++ b/src/gateway/server-methods/system-agent-greeting-welcome.test.ts @@ -7,7 +7,9 @@ import { resetCommandQueueStateForTest } from "../../process/command-queue.test- import { systemAgentHandlers, type SystemAgentChatSession } from "./system-agent.js"; import type { GatewayClient, GatewayRequestContext } from "./types.js"; -const setupInferenceMocks = vi.hoisted(() => ({ verifySetupInference: vi.fn() })); +const inferenceFallbackMocks = vi.hoisted(() => ({ + verifySystemAgentInferenceWithFallback: vi.fn(), +})); const transcriptStoreMocks = vi.hoisted(() => ({ appendTranscriptReset: vi.fn(), appendTranscriptTurn: vi.fn(), @@ -21,8 +23,9 @@ const greetingMocks = vi.hoisted(() => ({ })); const onboardingWelcomeMocks = vi.hoisted(() => ({ buildOnboardingWelcome: vi.fn() })); -vi.mock("../../system-agent/setup-inference.js", () => ({ - verifySetupInference: setupInferenceMocks.verifySetupInference, +vi.mock("../../system-agent/inference-fallback.js", () => ({ + verifySystemAgentInferenceWithFallback: + inferenceFallbackMocks.verifySystemAgentInferenceWithFallback, })); vi.mock("../../system-agent/transcript-store.js", () => ({ appendTranscriptReset: transcriptStoreMocks.appendTranscriptReset, @@ -121,7 +124,10 @@ const quickActions = { beforeEach(() => { createdEngines.length = 0; - setupInferenceMocks.verifySetupInference.mockResolvedValue({ ok: true, binding: {} }); + inferenceFallbackMocks.verifySystemAgentInferenceWithFallback.mockResolvedValue({ + ok: true, + binding: {}, + }); greetingMocks.loadSystemAgentGreetingFacts.mockReturnValue({ updateAvailable: null, channelHealth: { available: true, degraded: [] }, diff --git a/src/gateway/server-methods/system-agent-reset-boundary.test.ts b/src/gateway/server-methods/system-agent-reset-boundary.test.ts index 9d8fe4116e60..90d8c7023a26 100644 --- a/src/gateway/server-methods/system-agent-reset-boundary.test.ts +++ b/src/gateway/server-methods/system-agent-reset-boundary.test.ts @@ -15,7 +15,9 @@ import { withTempDir } from "../../test-helpers/temp-dir.js"; import { systemAgentHandlers, type SystemAgentChatSession } from "./system-agent.js"; import type { GatewayClient, GatewayRequestContext } from "./types.js"; -const setupInferenceMocks = vi.hoisted(() => ({ verifySetupInference: vi.fn() })); +const inferenceFallbackMocks = vi.hoisted(() => ({ + verifySystemAgentInferenceWithFallback: vi.fn(), +})); const greetingMocks = vi.hoisted(() => ({ acknowledgeSystemAgentGreetingDelivery: vi.fn(), buildSystemAgentGreetingQuestion: vi.fn(() => undefined), @@ -28,8 +30,9 @@ const greetingMocks = vi.hoisted(() => ({ resolveSystemAgentGreeting: vi.fn(async () => ({ text: "welcome text", source: "template" })), })); -vi.mock("../../system-agent/setup-inference.js", () => ({ - verifySetupInference: setupInferenceMocks.verifySetupInference, +vi.mock("../../system-agent/inference-fallback.js", () => ({ + verifySystemAgentInferenceWithFallback: + inferenceFallbackMocks.verifySystemAgentInferenceWithFallback, })); // The transcript store is deliberately NOT mocked: the boundary under test is // what survives in the durable store. Only the caretaker greeting is stubbed so @@ -55,7 +58,7 @@ const originalStateDir = process.env.OPENCLAW_STATE_DIR; beforeEach(async () => { const fixture = await createSystemAgentVerifiedInferenceTestFixture(verifiedConfig); - setupInferenceMocks.verifySetupInference.mockResolvedValue({ + inferenceFallbackMocks.verifySystemAgentInferenceWithFallback.mockResolvedValue({ ok: true, modelRef: "openai/gpt-5.5", latencyMs: 10, @@ -65,7 +68,7 @@ beforeEach(async () => { afterEach(() => { vi.restoreAllMocks(); - setupInferenceMocks.verifySetupInference.mockReset(); + inferenceFallbackMocks.verifySystemAgentInferenceWithFallback.mockReset(); closeOpenClawStateDatabase(); if (originalStateDir === undefined) { delete process.env.OPENCLAW_STATE_DIR; @@ -137,7 +140,7 @@ describe("openclaw.chat reset boundary", () => { [ "inference verification fails", () => { - setupInferenceMocks.verifySetupInference.mockResolvedValueOnce({ + inferenceFallbackMocks.verifySystemAgentInferenceWithFallback.mockResolvedValueOnce({ ok: false, status: "unavailable", error: "no configured model", @@ -180,7 +183,7 @@ describe("openclaw.chat reset boundary", () => { for (const turn of PRE_RESET_TURNS) { appendTranscriptTurn(turn); } - setupInferenceMocks.verifySetupInference.mockResolvedValueOnce({ + inferenceFallbackMocks.verifySystemAgentInferenceWithFallback.mockResolvedValueOnce({ ok: false, status: "unavailable", error: "no configured model", diff --git a/src/gateway/server-methods/system-agent-session-ownership.test.ts b/src/gateway/server-methods/system-agent-session-ownership.test.ts index 3258721872cc..905c90a7637e 100644 --- a/src/gateway/server-methods/system-agent-session-ownership.test.ts +++ b/src/gateway/server-methods/system-agent-session-ownership.test.ts @@ -7,8 +7,7 @@ import { SystemAgentWizardAnswerError } from "../../system-agent/chat-engine.js" import { systemAgentHandlers, type SystemAgentChatSession } from "./system-agent.js"; import type { GatewayClient, GatewayRequestContext, RespondFn } from "./types.js"; -const setupInferenceMocks = vi.hoisted(() => ({ verifySetupInference: vi.fn() })); -const delegatedInferenceMocks = vi.hoisted(() => ({ +const inferenceFallbackMocks = vi.hoisted(() => ({ verifySystemAgentInferenceWithFallback: vi.fn(), })); const transcriptStoreMocks = vi.hoisted(() => ({ @@ -17,12 +16,9 @@ const transcriptStoreMocks = vi.hoisted(() => ({ readTranscriptTail: vi.fn(() => []), })); -vi.mock("../../system-agent/setup-inference.js", () => ({ - verifySetupInference: setupInferenceMocks.verifySetupInference, -})); vi.mock("../../system-agent/inference-fallback.js", () => ({ verifySystemAgentInferenceWithFallback: - delegatedInferenceMocks.verifySystemAgentInferenceWithFallback, + inferenceFallbackMocks.verifySystemAgentInferenceWithFallback, })); vi.mock("../../system-agent/transcript-store.js", () => transcriptStoreMocks); // Ownership tests exercise fresh-session creation; keep the caretaker greeting @@ -142,8 +138,7 @@ async function callChat( beforeEach(() => { createdEngines.length = 0; - setupInferenceMocks.verifySetupInference.mockResolvedValue({ ok: true, binding: {} }); - delegatedInferenceMocks.verifySystemAgentInferenceWithFallback.mockResolvedValue({ + inferenceFallbackMocks.verifySystemAgentInferenceWithFallback.mockResolvedValue({ ok: true, binding: {}, }); @@ -234,7 +229,7 @@ describe("openclaw.chat session ownership", () => { }); expect(expire).not.toHaveBeenCalled(); expect(engine.dispose).not.toHaveBeenCalled(); - expect(setupInferenceMocks.verifySetupInference).not.toHaveBeenCalled(); + expect(inferenceFallbackMocks.verifySystemAgentInferenceWithFallback).not.toHaveBeenCalled(); }); it("lets the same authenticated principal resume after reconnecting", async () => { @@ -300,6 +295,10 @@ describe("openclaw.chat session ownership", () => { { sessionId: "delegated", delegation }, makeClient({ connId: "conn-owner", deviceId: "device-owner" }), ); + expect(inferenceFallbackMocks.verifySystemAgentInferenceWithFallback).toHaveBeenCalledWith({ + requestingAgentId: "main", + runtime: expect.anything(), + }); const handle = expectDefined(createdEngines[0], "created delegated engine").handle; const resumed = await callChat( @@ -314,6 +313,7 @@ describe("openclaw.chat session ownership", () => { expect(resumed.ok).toBe(true); expect(handle).toHaveBeenCalledWith("continue"); + expect(inferenceFallbackMocks.verifySystemAgentInferenceWithFallback).toHaveBeenCalledOnce(); }); it("rejects delegated reuse of a non-delegated session", async () => { @@ -367,7 +367,7 @@ describe("openclaw.chat session responses", () => { details: { code: "system_agent_session_invalidated" }, }, }); - expect(setupInferenceMocks.verifySetupInference).not.toHaveBeenCalled(); + expect(inferenceFallbackMocks.verifySystemAgentInferenceWithFallback).not.toHaveBeenCalled(); }); it("rejects a structured answer when the active session has no hosted wizard", async () => { diff --git a/src/gateway/server-methods/system-agent.test.ts b/src/gateway/server-methods/system-agent.test.ts index 017bc10db5d6..ac3e639360a4 100644 --- a/src/gateway/server-methods/system-agent.test.ts +++ b/src/gateway/server-methods/system-agent.test.ts @@ -39,10 +39,10 @@ import type { GatewayClient, GatewayRequestContext } from "./types.js"; const setupInferenceMocks = vi.hoisted(() => ({ activateSetupInference: vi.fn(), - detectSetupInference: vi.fn(), resolvePersistentApplyInference: vi.fn(), verifySetupInference: vi.fn(), })); +const inferenceFallbackMocks = vi.hoisted(() => ({ verify: vi.fn() })); const setupInferenceDetectionMocks = vi.hoisted(() => ({ detectSetupInferenceIsolated: vi.fn(), })); @@ -71,10 +71,12 @@ const onboardingWelcomeMocks = vi.hoisted(() => ({ vi.mock("../../system-agent/setup-inference.js", () => ({ activateSetupInference: setupInferenceMocks.activateSetupInference, - detectSetupInference: setupInferenceMocks.detectSetupInference, resolvePersistentApplyInference: setupInferenceMocks.resolvePersistentApplyInference, verifySetupInference: setupInferenceMocks.verifySetupInference, })); +vi.mock("../../system-agent/inference-fallback.js", () => ({ + verifySystemAgentInferenceWithFallback: inferenceFallbackMocks.verify, +})); vi.mock("../../system-agent/setup-inference-detection.js", () => ({ detectSetupInferenceIsolated: setupInferenceDetectionMocks.detectSetupInferenceIsolated, })); @@ -238,6 +240,12 @@ beforeEach(() => { latencyMs: 10, binding: verifiedInference, }); + inferenceFallbackMocks.verify.mockResolvedValue({ + ok: true, + modelRef: "openai/gpt-5.5", + latencyMs: 10, + binding: verifiedInference, + }); setupInferenceMocks.resolvePersistentApplyInference.mockResolvedValue( requireVerifiedInferenceFixture().configuredRoute, ); @@ -467,7 +475,7 @@ describe("openclaw.setup", () => { describe("openclaw.chat", () => { it("refuses to create a session before inference is available", async () => { - setupInferenceMocks.verifySetupInference.mockResolvedValueOnce({ + inferenceFallbackMocks.verify.mockResolvedValueOnce({ ok: false, status: "unavailable", error: "no configured model", @@ -487,13 +495,16 @@ describe("openclaw.chat", () => { }, }); expect(sessions.size).toBe(0); + expect(inferenceFallbackMocks.verify).toHaveBeenCalledWith({ + runtime: defaultRuntime, + }); }); it("coalesces concurrent initialization for the same session", async () => { stubEngineOverview(); const started = createDeferred(); const release = createDeferred(); - setupInferenceMocks.verifySetupInference.mockImplementation(async () => { + inferenceFallbackMocks.verify.mockImplementation(async () => { started.resolve(); await release.promise; return { @@ -513,7 +524,7 @@ describe("openclaw.chat", () => { release.resolve(); const [firstCall, secondCall] = await Promise.all([first, second]); - expect(setupInferenceMocks.verifySetupInference).toHaveBeenCalledOnce(); + expect(inferenceFallbackMocks.verify).toHaveBeenCalledOnce(); expect(sessions.size).toBe(1); expect([firstCall.ok, secondCall.ok]).toEqual([true, true]); }); @@ -928,7 +939,7 @@ describe("openclaw.chat", () => { expect(createSafeGatewayRestartPreflight().counts.queueSize).toBe(0); }); - it("drops a failed session and requires fresh inference on retry", async () => { + it("reuses a live session, then requires fresh fallback verification after failure", async () => { stubEngineOverview(); const engine = makeVerifiedEngine(); vi.spyOn(engine, "handle").mockRejectedValue( @@ -950,12 +961,12 @@ describe("openclaw.chat", () => { }); expect(dispose).toHaveBeenCalledOnce(); expect(sessions.has("s1")).toBe(false); - expect(setupInferenceMocks.verifySetupInference).not.toHaveBeenCalled(); + expect(inferenceFallbackMocks.verify).not.toHaveBeenCalled(); const retried = await callChat(context, { sessionId: "s1" }); expect(retried.ok).toBe(true); - expect(setupInferenceMocks.verifySetupInference).toHaveBeenCalledOnce(); + expect(inferenceFallbackMocks.verify).toHaveBeenCalledOnce(); expect(sessions.has("s1")).toBe(true); }); diff --git a/src/gateway/server-methods/system-agent.ts b/src/gateway/server-methods/system-agent.ts index 4cf380eb72d6..4e6e2e0d333c 100644 --- a/src/gateway/server-methods/system-agent.ts +++ b/src/gateway/server-methods/system-agent.ts @@ -578,18 +578,12 @@ export const systemAgentHandlers: GatewayRequestHandlers = { params.wizardAnswer === undefined && (params.message === undefined || !params.message.trim()); if (!session) { - const inference = params.delegation - ? await import("../../system-agent/inference-fallback.js").then( - ({ verifySystemAgentInferenceWithFallback }) => - verifySystemAgentInferenceWithFallback({ - requestingAgentId: params.delegation?.agentId, - runtime: defaultRuntime, - }), - ) - : await import("../../system-agent/setup-inference.js").then( - ({ verifySetupInference }) => - verifySetupInference({ runtime: defaultRuntime, bindSession: true }), - ); + const { verifySystemAgentInferenceWithFallback } = + await import("../../system-agent/inference-fallback.js"); + const inference = await verifySystemAgentInferenceWithFallback({ + ...(params.delegation ? { requestingAgentId: params.delegation.agentId } : {}), + runtime: defaultRuntime, + }); if (!inference.ok) { respond( false, diff --git a/src/plugin-sdk/provider-model-types.ts b/src/plugin-sdk/provider-model-types.ts index 675ee32fe3f2..2385ddb6f23e 100644 --- a/src/plugin-sdk/provider-model-types.ts +++ b/src/plugin-sdk/provider-model-types.ts @@ -69,3 +69,10 @@ export type ProviderNormalizeModelCatalogIdContext = { provider: string; modelId: string; }; + +/** Compares reported response identity without rewriting authored route identity. */ +export type ProviderResponseModelEquivalenceContext = { + provider: string; + requestedModelId: string; + responseModelId: string; +}; diff --git a/src/plugins/provider-policy-surface.test.ts b/src/plugins/provider-policy-surface.test.ts index aa3e8c91ca99..02f6a510df6f 100644 --- a/src/plugins/provider-policy-surface.test.ts +++ b/src/plugins/provider-policy-surface.test.ts @@ -14,7 +14,11 @@ describe("direct provider policy surface", () => { throw new Error("unexpected manifest registry import"); }); const resolveModelRoutes = vi.fn(); - const loadBundledPluginPublicArtifactModuleSync = vi.fn(() => ({ resolveModelRoutes })); + const isResponseModelEquivalent = vi.fn(); + const loadBundledPluginPublicArtifactModuleSync = vi.fn(() => ({ + resolveModelRoutes, + isResponseModelEquivalent, + })); vi.doMock("./bundled-dir.js", () => ({ resolveBundledPluginsDir: () => "/tmp/bundled-plugins", @@ -31,6 +35,7 @@ describe("direct provider policy surface", () => { const surface = resolveDirectBundledProviderPolicySurface("openai"); expect(surface?.resolveModelRoutes).toBe(resolveModelRoutes); + expect(surface?.isResponseModelEquivalent).toBe(isResponseModelEquivalent); expect(loadBundledPluginPublicArtifactModuleSync).toHaveBeenCalledWith({ dirName: "openai", artifactBasename: "provider-policy-api.js", diff --git a/src/plugins/provider-policy-surface.ts b/src/plugins/provider-policy-surface.ts index 77cb9be9e9da..2a4822e98373 100644 --- a/src/plugins/provider-policy-surface.ts +++ b/src/plugins/provider-policy-surface.ts @@ -4,6 +4,7 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { ProviderModelRouteResolution, ProviderNormalizeModelCatalogIdContext, + ProviderResponseModelEquivalenceContext, ProviderResolveModelRoutesContext, } from "../plugin-sdk/provider-model-types.js"; import { resolveBundledPluginsDir } from "./bundled-dir.js"; @@ -49,6 +50,9 @@ export type ProviderPolicySurface = { normalizeModelCatalogId?: ( ctx: ProviderNormalizeModelCatalogIdContext, ) => string | null | undefined; + isResponseModelEquivalent?: ( + ctx: ProviderResponseModelEquivalenceContext, + ) => boolean | null | undefined; }; /** Provider policy hooks loaded only from bundled plugin public artifacts. */ @@ -71,6 +75,7 @@ const PROVIDER_POLICY_HOOK_KEYS = [ "resolveThinkingProfile", "resolveModelRoutes", "normalizeModelCatalogId", + "isResponseModelEquivalent", ] as const satisfies readonly (keyof ProviderPolicySurface)[]; function extractProviderPolicySurface(mod: Record): ProviderPolicySurface | null { diff --git a/src/system-agent/inference-fallback.test.ts b/src/system-agent/inference-fallback.test.ts index 8e1b66797840..1d32d99255ca 100644 --- a/src/system-agent/inference-fallback.test.ts +++ b/src/system-agent/inference-fallback.test.ts @@ -23,7 +23,7 @@ const config: OpenClawConfig = { agents: { defaults: { model: { primary: "zeta/model" } }, list: [ - { id: "requester", model: "zeta/model" }, + { id: "requester", default: true, model: "zeta/model" }, { id: "beta", model: "beta/model" }, { id: "alpha", model: "alpha/model" }, ], @@ -31,7 +31,7 @@ const config: OpenClawConfig = { }; describe("system-agent inference fallback", () => { - it("tries requester first, then authenticated providers by provider id", async () => { + it("tries the default route first, then authenticated providers by provider id", async () => { const attempts: string[] = []; const verify = vi.fn(async ({ agentId }: { agentId: string }) => { attempts.push(agentId); @@ -41,7 +41,6 @@ describe("system-agent inference fallback", () => { }); const result = await verifySystemAgentInferenceWithFallback({ - requestingAgentId: "requester", runtime, deps: { readConfig: async () => config, @@ -196,6 +195,40 @@ describe("system-agent inference fallback", () => { expect(attempts).toEqual(["requester", "alpha-other"]); }); + it("tries another route of the same provider after a malformed response", async () => { + const attempts: string[] = []; + const cfg: OpenClawConfig = { + agents: { + defaults: { model: { primary: "alpha/model" } }, + list: [ + { id: "requester", model: "alpha/model" }, + { id: "alpha-other", model: "alpha/model" }, + { id: "beta", model: "beta/model" }, + ], + }, + }; + + const result = await verifySystemAgentInferenceWithFallback({ + requestingAgentId: "requester", + runtime, + deps: { + readConfig: async () => cfg, + resolveRoute: async (_cfg, agentId) => + route(agentId, agentId === "beta" ? "beta" : "alpha"), + hasAuth: async () => true, + verify: async ({ agentId }) => { + attempts.push(agentId); + return agentId === "alpha-other" + ? ({ ok: true, modelRef: "alpha/model", latencyMs: 1, binding: {} } as never) + : ({ ok: false, status: "format", error: "bad response" } as const); + }, + }, + }); + + expect(result.ok).toBe(true); + expect(attempts).toEqual(["requester", "alpha-other"]); + }); + it("retires the whole provider after a provider-wide failure", async () => { const attempts: string[] = []; const cfg: OpenClawConfig = { @@ -227,16 +260,15 @@ describe("system-agent inference fallback", () => { }); expect(result.ok).toBe(true); - // alpha-other is skipped: the requester's alpha route failed provider-wide. expect(attempts).toEqual(["requester", "beta"]); }); - it("does not fail over on bad answers", async () => { + it("does not fail over on owner or identity uncertainty", async () => { const verify = vi.fn( - async () => ({ ok: false, status: "format", error: "bad answer" }) as const, + async () => ({ ok: false, status: "unknown", error: "winner identity uncertain" }) as const, ); - await verifySystemAgentInferenceWithFallback({ + const result = await verifySystemAgentInferenceWithFallback({ requestingAgentId: "requester", runtime, deps: { @@ -248,6 +280,11 @@ describe("system-agent inference fallback", () => { }, }); + expect(result).toEqual({ + ok: false, + status: "unknown", + error: "winner identity uncertain", + }); expect(verify).toHaveBeenCalledOnce(); }); }); diff --git a/src/system-agent/inference-fallback.ts b/src/system-agent/inference-fallback.ts index 06c987e7a2e6..3363c8930c85 100644 --- a/src/system-agent/inference-fallback.ts +++ b/src/system-agent/inference-fallback.ts @@ -1,4 +1,4 @@ -// Provider-neutral live inference ladder for delegated OpenClaw sessions. +// Provider-neutral live inference ladder for OpenClaw sessions. import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id"; import { listAgentIds, tryResolveDefaultAgentId } from "../agents/agent-scope.js"; import { hasAvailableAuthForProvider } from "../agents/model-auth.js"; @@ -17,14 +17,14 @@ const RETRYABLE_INFERENCE_STATUSES = new Set([ "rate_limit", "billing", "timeout", + "format", "unavailable", ]); -// auth/billing/rate_limit failures commonly apply to one key/account/project, -// so another credential owner of the same provider may still work. Everything -// else retryable (timeout, unavailable) is provider-wide, so its whole provider -// is skipped for the rest of the ladder. -const CREDENTIAL_SCOPED_FAILURE_STATUSES = new Set(["auth", "billing", "rate_limit"]); +// Only failures that establish provider-wide unavailability retire every route. +// Credential failures may clear with another owner, while format failures can be +// model-specific, so both stay scoped to the attempted route. +const PROVIDER_WIDE_FAILURE_STATUSES = new Set(["timeout", "unavailable"]); type InferenceFallbackDeps = { readConfig?: () => Promise; @@ -131,13 +131,11 @@ export async function verifySystemAgentInferenceWithFallback(params: { return result; } lastFailure = result; - // Bad/empty answers and owner-integrity failures are not availability failover. + // Identity or owner-integrity uncertainty stays fail-closed as unknown. if (!RETRYABLE_INFERENCE_STATUSES.has(result.status)) { return result; } - // A provider-wide failure applies to all of its routes; a credential-scoped - // one may not, so only the former retires the whole provider. - if (!CREDENTIAL_SCOPED_FAILURE_STATUSES.has(result.status)) { + if (PROVIDER_WIDE_FAILURE_STATUSES.has(result.status)) { failedProviders.add(candidate.provider); } } diff --git a/src/system-agent/setup-inference-persist.ts b/src/system-agent/setup-inference-persist.ts index a185a1a37922..f89b13141a90 100644 --- a/src/system-agent/setup-inference-persist.ts +++ b/src/system-agent/setup-inference-persist.ts @@ -591,9 +591,9 @@ export async function runSetupInferenceTest(params: { error: "The model started but did not send a reply. Try again or pick another option.", }; } - const winnerError = extractRunWinnerError(plan, result); + const winnerError = await extractRunWinnerError(plan, result); if (winnerError) { - return { ok: false, status: "format", error: winnerError }; + return { ok: false, status: "unknown", error: winnerError }; } if (requireExecutionOwner && !successfulAuth) { return { diff --git a/src/system-agent/setup-inference-plan-helpers.ts b/src/system-agent/setup-inference-plan-helpers.ts index 92a64379c04d..ec9f37f3f5e1 100644 --- a/src/system-agent/setup-inference-plan-helpers.ts +++ b/src/system-agent/setup-inference-plan-helpers.ts @@ -115,17 +115,30 @@ export function extractRunTerminalError(result: RunResult): string | undefined { ); } -export function extractRunWinnerError( +export async function extractRunWinnerError( plan: SetupInferenceTestPlan, result: RunResult, -): string | undefined { +): Promise { const winnerProvider = result.meta?.executionTrace?.winnerProvider?.trim(); const winnerModel = result.meta?.executionTrace?.winnerModel?.trim(); if (!winnerProvider || !winnerModel) { return "The inference run did not report which provider and model produced its reply."; } - if (winnerProvider === plan.provider && winnerModel === plan.model) { - return undefined; + if (winnerProvider === plan.provider) { + if (winnerModel === plan.model) { + return undefined; + } + const { resolveDirectBundledProviderPolicySurface } = + await import("../plugins/provider-policy-surface.js"); + if ( + resolveDirectBundledProviderPolicySurface(plan.provider)?.isResponseModelEquivalent?.({ + provider: plan.provider, + requestedModelId: plan.model, + responseModelId: winnerModel, + }) === true + ) { + return undefined; + } } return `The inference run answered through ${winnerProvider}/${winnerModel} instead of the requested ${plan.provider}/${plan.model}. Disable model-routing overrides or choose the working route directly, then retry.`; } diff --git a/src/system-agent/setup-inference.test.ts b/src/system-agent/setup-inference.test.ts index 9af203d725a2..0dc87ec15cc3 100644 --- a/src/system-agent/setup-inference.test.ts +++ b/src/system-agent/setup-inference.test.ts @@ -445,6 +445,7 @@ type SuccessfulRunParams = { authProfileId?: string; agentHarnessRuntimeOverride?: string; config?: OpenClawConfig; + reportedModel?: string; }; function successfulAgentHarnessBinding(params?: SuccessfulRunParams): AgentExecutionAuthBinding { @@ -491,13 +492,14 @@ function successfulRun(provider: string, model: string, params?: SuccessfulRunPa return { meta: { finalAssistantVisibleText: "OK", - executionTrace: { winnerProvider: provider, winnerModel: model }, + executionTrace: { winnerProvider: provider, winnerModel: params?.reportedModel ?? model }, }, }; } -function successfulRunner(provider: string, model: string) { - return async (params: SuccessfulRunParams) => successfulRun(provider, model, params); +function successfulRunner(provider: string, model: string, reportedModel?: string) { + return async (params: SuccessfulRunParams) => + successfulRun(provider, model, { ...params, reportedModel }); } function openAiOAuthCredential(token: string, lifetimeMs = 3_600_000) { @@ -2026,7 +2028,7 @@ describe("activateSetupInference", () => { expect(configHarness.current()).toEqual(canonicalizeAgentEntriesForTest(concurrentConfig)); }); - it("preserves authored provider rows and lifts an onboarding-owned lean setting", async () => { + it("accepts OpenAI's gpt-5.6 alias reporting Sol while preserving authored rows", async () => { const sourceConfig = { wizard: { localModelLeanAutoModel: "lmstudio/qwen-local" }, agents: { @@ -2067,17 +2069,21 @@ describe("activateSetupInference", () => { }, ]; const configHarness = createConfigTransformHarness(sourceConfig, runtimeConfig); + const runEmbeddedAgent = vi.fn(successfulRunner("openai", "gpt-5.6", "gpt-5.6-sol")); const result = await activateSetupInference({ kind: "openai-api-key", deps: { readConfigFileSnapshot: mockConfigSnapshot(sourceConfig, { runtimeConfig }), - runEmbeddedAgent: vi.fn(successfulRunner("openai", "gpt-5.6")) as never, + runEmbeddedAgent: runEmbeddedAgent as never, transformConfigWithPendingPluginInstalls: configHarness.transform as never, }, }); expect(result).toMatchObject({ ok: true, modelRef: "openai/gpt-5.6" }); + expect(runEmbeddedAgent).toHaveBeenCalledWith( + expect.objectContaining({ provider: "openai", model: "gpt-5.6" }), + ); expect(configHarness.current().models?.providers?.openai?.models).toEqual( sourceConfig.models.providers.openai.models, ); @@ -2426,14 +2432,20 @@ describe("activateSetupInference", () => { error: "did not report which provider and model", }, { - name: "model-routing override", + name: "provider-routing override", runResult: successfulRun("openai", "gpt-5.5"), error: "instead of the requested anthropic/claude-opus-5", }, - ])("does not persist inference after a $name", async ({ runResult, error }) => { + { + name: "same-provider model-routing override", + kind: "openai-api-key" as const, + runResult: successfulRun("openai", "gpt-5.6-terra"), + error: "instead of the requested openai/gpt-5.6", + }, + ])("does not persist inference after a $name", async ({ runResult, error, kind }) => { const transformConfig = vi.fn(); const result = await activateSetupInference({ - kind: "anthropic-api-key", + kind: kind ?? "anthropic-api-key", deps: { runEmbeddedAgent: vi.fn(async () => runResult) as never, transformConfigWithPendingPluginInstalls: transformConfig as never, @@ -2442,7 +2454,7 @@ describe("activateSetupInference", () => { expect(result).toMatchObject({ ok: false, - status: "format", + status: "unknown", error: expect.stringContaining(error), }); expect(transformConfig).not.toHaveBeenCalled();