From e6d04550cac74f9a2b2e80a751443335bbbf0713 Mon Sep 17 00:00:00 2001 From: Frank Yang Date: Sat, 16 May 2026 20:12:02 +0800 Subject: [PATCH] fix(gateway): route WebChat images through imageModel Route WebChat image attachments through the configured imageModel when the active session model cannot accept images, while keeping one-turn image auth and fallback state ephemeral. Thanks @frankekn. --- CHANGELOG.md | 1 + src/auto-reply/get-reply-options.types.ts | 4 + .../reply/agent-runner-execution.test.ts | 63 +++++++ .../reply/agent-runner-execution.ts | 3 + .../reply/agent-runner-run-params.ts | 32 ++-- .../reply/agent-runner-utils.test.ts | 64 +++++++ ...et-reply-directives.target-session.test.ts | 23 ++- src/auto-reply/reply/get-reply-directives.ts | 3 + .../reply/get-reply-run.media-only.test.ts | 148 +++++++++++++++ src/auto-reply/reply/get-reply-run.ts | 107 ++++++++--- .../reply/get-reply.fast-path.test.ts | 47 ++++- src/auto-reply/reply/get-reply.ts | 54 +++++- .../reply/image-model-override-plan.test.ts | 135 ++++++++++++++ .../reply/image-model-override-plan.ts | 122 +++++++++++++ src/auto-reply/reply/model-selection.test.ts | 39 ++++ src/auto-reply/reply/model-selection.ts | 13 +- src/auto-reply/reply/queue/types.ts | 2 + src/gateway/chat-attachments.ts | 17 ++ src/gateway/server-methods/chat.ts | 47 ++++- .../server.chat.gateway-server-chat-b.test.ts | 172 ++++++++++++++++++ 20 files changed, 1043 insertions(+), 53 deletions(-) create mode 100644 src/auto-reply/reply/image-model-override-plan.test.ts create mode 100644 src/auto-reply/reply/image-model-override-plan.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index d991cdeeae91..1190ebf77a47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ Docs: https://docs.openclaw.ai ### Fixes +- Gateway/WebChat: route image attachments through a configured vision-capable `imageModel` plan before inlining images, and carry that image-model fallback chain through runtime retries. (#82524) Thanks @frankekn. - Codex app-server: limit canonical OpenAI Codex app-server attribution rewrites to local transcript and trajectory records, leaving runtime/tool routing on the selected OpenAI model metadata so OpenAI API-key backup profiles keep their billing path. - Android/chat: make bare and markdown URLs in chat messages tappable by preserving Compose URL annotations in rendered markdown. Fixes #82187. (#82392) Thanks @neeravmakwana. - Codex app-server: release raw assistant completions when `turn/completed` is missing while keeping commentary/status items as progress, preventing completed Codex runs from hanging until timeout. Fixes #82343. (#82403) Thanks @IWhatsskill. diff --git a/src/auto-reply/get-reply-options.types.ts b/src/auto-reply/get-reply-options.types.ts index c1bb54a8604f..54fc5e5053c8 100644 --- a/src/auto-reply/get-reply-options.types.ts +++ b/src/auto-reply/get-reply-options.types.ts @@ -196,4 +196,8 @@ export type GetReplyOptions = { hasRepliedRef?: { value: boolean }; /** Override agent timeout in seconds (0 = no timeout). Threads through to resolveAgentTimeoutMs. */ timeoutOverrideSeconds?: number; + /** Capability-checked one-turn model override for inline image input. */ + modelOverride?: string; + /** Capability-checked runtime fallbacks for the one-turn image model override. */ + modelOverrideFallbacks?: string[]; }; diff --git a/src/auto-reply/reply/agent-runner-execution.test.ts b/src/auto-reply/reply/agent-runner-execution.test.ts index f2f0dc08a7d9..b9a69aea8aa3 100644 --- a/src/auto-reply/reply/agent-runner-execution.test.ts +++ b/src/auto-reply/reply/agent-runner-execution.test.ts @@ -4018,6 +4018,69 @@ describe("runAgentTurnWithFallback", () => { expect(sessionStore.main.authProfileOverride).toBeUndefined(); }); + it("does not persist fallback selection for one-turn image model overrides", async () => { + state.runWithModelFallbackMock.mockImplementation( + async (params: { run: (provider: string, model: string) => Promise }) => ({ + result: await params.run("openai", "gpt-4o-mini"), + provider: "openai", + model: "gpt-4o-mini", + attempts: [], + }), + ); + state.runEmbeddedPiAgentMock.mockResolvedValue({ + payloads: [{ text: "ok" }], + meta: {}, + }); + + const followupRun = createFollowupRun(); + followupRun.run.provider = "openai"; + followupRun.run.model = "gpt-4o"; + followupRun.run.hasOneTurnModelOverride = true; + + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: Date.now(), + totalTokens: 1, + compactionCount: 0, + }; + const sessionStore = { main: sessionEntry }; + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { + Provider: "telegram", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterCompactionFailure: async () => false, + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => sessionEntry, + activeSessionStore: sessionStore, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("success"); + expectMockCallArgFields(state.runEmbeddedPiAgentMock, 0, "embedded run params", { + provider: "openai", + model: "gpt-4o-mini", + }); + expect(sessionEntry.providerOverride).toBeUndefined(); + expect(sessionEntry.modelOverride).toBeUndefined(); + expect(sessionEntry.modelOverrideSource).toBeUndefined(); + }); + it("does not persist fallback selection for legacy user overrides without modelOverrideSource", async () => { // Regression: older persisted sessions can have a user-selected override // (modelOverride set) but no modelOverrideSource field, because the field diff --git a/src/auto-reply/reply/agent-runner-execution.ts b/src/auto-reply/reply/agent-runner-execution.ts index adbaee3f5231..71f5f6c20952 100644 --- a/src/auto-reply/reply/agent-runner-execution.ts +++ b/src/auto-reply/reply/agent-runner-execution.ts @@ -1302,6 +1302,9 @@ export async function runAgentTurnWithFallback(params: { provider: string, model: string, ): Promise<(() => Promise) | undefined> => { + if (params.followupRun.run.hasOneTurnModelOverride === true) { + return undefined; + } if ( !params.sessionKey || !params.activeSessionStore || diff --git a/src/auto-reply/reply/agent-runner-run-params.ts b/src/auto-reply/reply/agent-runner-run-params.ts index 97d619de0f64..e1aa011314b6 100644 --- a/src/auto-reply/reply/agent-runner-run-params.ts +++ b/src/auto-reply/reply/agent-runner-run-params.ts @@ -31,13 +31,15 @@ export function resolveModelFallbackOptions( configOverride: FollowupRun["run"]["config"] = run.config, ) { const config = configOverride; - const fallbacksOverride = resolveEffectiveModelFallbacks({ - cfg: config, - agentId: run.agentId, - hasSessionModelOverride: run.hasSessionModelOverride === true, - modelOverrideSource: run.modelOverrideSource, - hasAutoFallbackProvenance: run.hasAutoFallbackProvenance === true, - }); + const fallbacksOverride = + run.imageModelFallbacksOverride ?? + resolveEffectiveModelFallbacks({ + cfg: config, + agentId: run.agentId, + hasSessionModelOverride: run.hasSessionModelOverride === true, + modelOverrideSource: run.modelOverrideSource, + hasAutoFallbackProvenance: run.hasAutoFallbackProvenance === true, + }); return { cfg: config, provider: run.provider, @@ -57,13 +59,15 @@ export function buildEmbeddedRunBaseParams(params: { isReasoningTagProvider?: ReasoningTagProviderResolver; }) { const config = params.run.config; - const modelFallbacksOverride = resolveEffectiveModelFallbacks({ - cfg: config, - agentId: params.run.agentId, - hasSessionModelOverride: params.run.hasSessionModelOverride === true, - modelOverrideSource: params.run.modelOverrideSource, - hasAutoFallbackProvenance: params.run.hasAutoFallbackProvenance === true, - }); + const modelFallbacksOverride = + params.run.imageModelFallbacksOverride ?? + resolveEffectiveModelFallbacks({ + cfg: config, + agentId: params.run.agentId, + hasSessionModelOverride: params.run.hasSessionModelOverride === true, + modelOverrideSource: params.run.modelOverrideSource, + hasAutoFallbackProvenance: params.run.hasAutoFallbackProvenance === true, + }); return { sessionFile: params.run.sessionFile, workspaceDir: params.run.workspaceDir, diff --git a/src/auto-reply/reply/agent-runner-utils.test.ts b/src/auto-reply/reply/agent-runner-utils.test.ts index f3a132c4cbb4..55a4f943226b 100644 --- a/src/auto-reply/reply/agent-runner-utils.test.ts +++ b/src/auto-reply/reply/agent-runner-utils.test.ts @@ -103,6 +103,28 @@ describe("agent-runner-utils", () => { expect(resolved.fallbacksOverride).toEqual(["fallback-model"]); }); + it("uses image model fallback overrides for model fallback options", () => { + const run = makeRun({ + imageModelFallbacksOverride: ["openai/gpt-4o-mini"], + }); + + const resolved = resolveModelFallbackOptions(run); + + expect(hoisted.resolveEffectiveModelFallbacksMock).not.toHaveBeenCalled(); + expect(resolved.fallbacksOverride).toEqual(["openai/gpt-4o-mini"]); + }); + + it("preserves empty image model fallback overrides for model fallback options", () => { + const run = makeRun({ + imageModelFallbacksOverride: [], + }); + + const resolved = resolveModelFallbackOptions(run); + + expect(hoisted.resolveEffectiveModelFallbacksMock).not.toHaveBeenCalled(); + expect(resolved.fallbacksOverride).toEqual([]); + }); + it("passes through missing agentId for helper-based fallback resolution", () => { hoisted.resolveEffectiveModelFallbacksMock.mockReturnValue(["fallback-model"]); const run = makeRun({ agentId: undefined }); @@ -185,6 +207,48 @@ describe("agent-runner-utils", () => { expect(resolved.modelFallbacksOverride).toEqual(["fallback-model"]); }); + it("uses image model fallback overrides for embedded run params", () => { + const run = makeRun({ + imageModelFallbacksOverride: ["openai/gpt-4o-mini"], + }); + const authProfile = resolveProviderScopedAuthProfile({ + provider: "openai", + primaryProvider: "openai", + }); + + const resolved = buildEmbeddedRunBaseParams({ + run, + provider: "openai", + model: "gpt-4o", + runId: "run-1", + authProfile, + }); + + expect(hoisted.resolveEffectiveModelFallbacksMock).not.toHaveBeenCalled(); + expect(resolved.modelFallbacksOverride).toEqual(["openai/gpt-4o-mini"]); + }); + + it("preserves empty image model fallback overrides for embedded run params", () => { + const run = makeRun({ + imageModelFallbacksOverride: [], + }); + const authProfile = resolveProviderScopedAuthProfile({ + provider: "openai", + primaryProvider: "openai", + }); + + const resolved = buildEmbeddedRunBaseParams({ + run, + provider: "openai", + model: "gpt-4o", + runId: "run-1", + authProfile, + }); + + expect(hoisted.resolveEffectiveModelFallbacksMock).not.toHaveBeenCalled(); + expect(resolved.modelFallbacksOverride).toEqual([]); + }); + it("does not force final-tag enforcement for minimax providers", () => { const run = makeRun(); diff --git a/src/auto-reply/reply/get-reply-directives.target-session.test.ts b/src/auto-reply/reply/get-reply-directives.target-session.test.ts index c2f565f83796..0d7696b79e68 100644 --- a/src/auto-reply/reply/get-reply-directives.target-session.test.ts +++ b/src/auto-reply/reply/get-reply-directives.target-session.test.ts @@ -145,6 +145,9 @@ async function resolveHelloWithModelDefaults(params: { sessionEntry?: SessionEntry; agentCfg?: { reasoningDefault?: "off" | "on" | "stream" }; commandAuthorized?: boolean; + hasOneTurnModelOverride?: boolean; + provider?: string; + model?: string; ctx?: Parameters[0]; }) { const resolveDefaultThinkingLevel = vi.fn(async () => params.defaultThinking); @@ -190,8 +193,9 @@ async function resolveHelloWithModelDefaults(params: { defaultProvider: "openai", defaultModel: "gpt-4o-mini", aliasIndex: { byAlias: new Map(), byKey: new Map() }, - provider: "openai", - model: "gpt-4o-mini", + provider: params.provider ?? "openai", + model: params.model ?? "gpt-4o-mini", + hasOneTurnModelOverride: params.hasOneTurnModelOverride, hasResolvedHeartbeatModelOverride: false, typing: makeTypingController(), opts: undefined, @@ -314,6 +318,21 @@ describe("resolveReplyDirectives", () => { mocks.resolveReplyExecOverrides.mockReturnValue(undefined); }); + it("passes one-turn model override state into model selection", async () => { + await resolveHelloWithModelDefaults({ + defaultThinking: "off", + defaultReasoning: "on", + provider: "openai", + model: "gpt-4o-mini", + hasOneTurnModelOverride: true, + }); + + const modelSelectionInput = mockCallInput(mocks.createModelSelectionState); + expect(modelSelectionInput.provider).toBe("openai"); + expect(modelSelectionInput.model).toBe("gpt-4o-mini"); + expect(modelSelectionInput.hasOneTurnModelOverride).toBe(true); + }); + it("prefers the target session entry from sessionStore for directive state", async () => { const wrapperSessionEntry = makeSessionEntry({ sessionId: "wrapper-session", diff --git a/src/auto-reply/reply/get-reply-directives.ts b/src/auto-reply/reply/get-reply-directives.ts index 64a2caf036c0..4508019a1a96 100644 --- a/src/auto-reply/reply/get-reply-directives.ts +++ b/src/auto-reply/reply/get-reply-directives.ts @@ -171,6 +171,7 @@ export async function resolveReplyDirectives(params: { aliasIndex: ModelAliasIndex; provider: string; model: string; + hasOneTurnModelOverride?: boolean; hasResolvedHeartbeatModelOverride: boolean; typing: TypingController; opts?: GetReplyOptions; @@ -200,6 +201,7 @@ export async function resolveReplyDirectives(params: { primaryModel, provider: initialProvider, model: initialModel, + hasOneTurnModelOverride, hasResolvedHeartbeatModelOverride, typing, opts, @@ -535,6 +537,7 @@ export async function resolveReplyDirectives(params: { provider, model, hasModelDirective: directives.hasModelDirective, + hasOneTurnModelOverride, hasResolvedHeartbeatModelOverride, isHeartbeat: opts?.isHeartbeat === true, }); diff --git a/src/auto-reply/reply/get-reply-run.media-only.test.ts b/src/auto-reply/reply/get-reply-run.media-only.test.ts index 80c2ec6ef4d2..8f14969fe4b3 100644 --- a/src/auto-reply/reply/get-reply-run.media-only.test.ts +++ b/src/auto-reply/reply/get-reply-run.media-only.test.ts @@ -1140,6 +1140,154 @@ describe("runPreparedReply media-only handling", () => { expect(call?.followupRun.run.authProfileId).toBe("profile-after-wait"); expect(vi.mocked(resolveSessionAuthProfileOverride)).toHaveBeenCalledTimes(1); }); + + it("resolves image override auth profile without mutating stored session profile", async () => { + const { resolveSessionAuthProfileOverride } = + await import("../../agents/auth-profiles/session-override.js"); + const sessionEntry: SessionEntry = { + sessionId: "session-image-auth", + sessionFile: "/tmp/session-image-auth.jsonl", + authProfileOverride: "anthropic:work", + authProfileOverrideSource: "user", + updatedAt: 1, + }; + const sessionStore: Record = { + "session-key": sessionEntry, + }; + vi.mocked(resolveSessionAuthProfileOverride).mockImplementationOnce(async (params) => { + expect(params.provider).toBe("openai"); + expect(params.storePath).toBeUndefined(); + expect(params.sessionEntry).not.toBe(sessionEntry); + expect(params.sessionStore).not.toBe(sessionStore); + if (params.sessionEntry) { + params.sessionEntry.authProfileOverride = "openai:vision"; + params.sessionEntry.authProfileOverrideSource = "auto"; + } + return "openai:vision"; + }); + + await runPreparedReply( + baseParams({ + provider: "openai", + model: "gpt-4o", + defaultProvider: "anthropic", + defaultModel: "claude-opus-4-1", + hasAppliedImageModelOverride: true, + isNewSession: false, + sessionId: "session-image-auth", + sessionEntry, + sessionStore, + storePath: "/tmp/sessions.json", + }), + ); + + const call = requireLastRunReplyAgentCall(); + expect(call?.followupRun.run.authProfileId).toBe("openai:vision"); + expect(call?.followupRun.run.authProfileIdSource).toBe("auto"); + expect(sessionEntry.authProfileOverride).toBe("anthropic:work"); + expect(sessionEntry.authProfileOverrideSource).toBe("user"); + expect(sessionStore["session-key"]?.authProfileOverride).toBe("anthropic:work"); + }); + + it("isolates image override auth profile when the override provider matches the default provider", async () => { + const { resolveSessionAuthProfileOverride } = + await import("../../agents/auth-profiles/session-override.js"); + const sessionEntry: SessionEntry = { + sessionId: "session-image-default-provider-auth", + sessionFile: "/tmp/session-image-default-provider-auth.jsonl", + providerOverride: "anthropic", + modelOverride: "claude-opus-4-1", + authProfileOverride: "anthropic:work", + authProfileOverrideSource: "user", + updatedAt: 1, + }; + const sessionStore: Record = { + "session-key": sessionEntry, + }; + vi.mocked(resolveSessionAuthProfileOverride).mockImplementationOnce(async (params) => { + expect(params.provider).toBe("openai"); + expect(params.storePath).toBeUndefined(); + expect(params.sessionEntry).not.toBe(sessionEntry); + expect(params.sessionStore).not.toBe(sessionStore); + if (params.sessionEntry) { + params.sessionEntry.authProfileOverride = "openai:vision"; + params.sessionEntry.authProfileOverrideSource = "auto"; + } + return "openai:vision"; + }); + + await runPreparedReply( + baseParams({ + provider: "openai", + model: "gpt-4o", + defaultProvider: "openai", + defaultModel: "gpt-4o-mini", + hasAppliedImageModelOverride: true, + isNewSession: false, + sessionId: "session-image-default-provider-auth", + sessionEntry, + sessionStore, + storePath: "/tmp/sessions.json", + }), + ); + + const call = requireLastRunReplyAgentCall(); + expect(call?.followupRun.run.authProfileId).toBe("openai:vision"); + expect(call?.followupRun.run.authProfileIdSource).toBe("auto"); + expect(sessionEntry.authProfileOverride).toBe("anthropic:work"); + expect(sessionStore["session-key"]?.authProfileOverride).toBe("anthropic:work"); + }); + + it("isolates image override auth profile from the pre-override runtime provider", async () => { + const { resolveSessionAuthProfileOverride } = + await import("../../agents/auth-profiles/session-override.js"); + const sessionEntry: SessionEntry = { + sessionId: "session-image-runtime-provider-auth", + sessionFile: "/tmp/session-image-runtime-provider-auth.jsonl", + modelProvider: "anthropic", + model: "claude-opus-4-1", + authProfileOverride: "anthropic:work", + authProfileOverrideSource: "user", + updatedAt: 1, + }; + const sessionStore: Record = { + "session-key": sessionEntry, + }; + vi.mocked(resolveSessionAuthProfileOverride).mockImplementationOnce(async (params) => { + expect(params.provider).toBe("openai"); + expect(params.storePath).toBeUndefined(); + expect(params.sessionEntry).not.toBe(sessionEntry); + expect(params.sessionStore).not.toBe(sessionStore); + if (params.sessionEntry) { + params.sessionEntry.authProfileOverride = "openai:vision"; + params.sessionEntry.authProfileOverrideSource = "auto"; + } + return "openai:vision"; + }); + + await runPreparedReply( + baseParams({ + provider: "openai", + model: "gpt-4o", + defaultProvider: "openai", + defaultModel: "gpt-4o-mini", + hasAppliedImageModelOverride: true, + imageModelOverrideBaseProvider: "anthropic", + isNewSession: false, + sessionId: "session-image-runtime-provider-auth", + sessionEntry, + sessionStore, + storePath: "/tmp/sessions.json", + }), + ); + + const call = requireLastRunReplyAgentCall(); + expect(call?.followupRun.run.authProfileId).toBe("openai:vision"); + expect(call?.followupRun.run.authProfileIdSource).toBe("auto"); + expect(sessionEntry.authProfileOverride).toBe("anthropic:work"); + expect(sessionStore["session-key"]?.authProfileOverride).toBe("anthropic:work"); + }); + it("re-resolves same-session ownership after session-id rotation during async prep", async () => { const { resolveSessionAuthProfileOverride } = await import("../../agents/auth-profiles/session-override.js"); diff --git a/src/auto-reply/reply/get-reply-run.ts b/src/auto-reply/reply/get-reply-run.ts index 0624c97080c9..2eaf4c27160c 100644 --- a/src/auto-reply/reply/get-reply-run.ts +++ b/src/auto-reply/reply/get-reply-run.ts @@ -4,6 +4,7 @@ import { resolveSessionAuthProfileOverride } from "../../agents/auth-profiles/se import type { ExecToolDefaults } from "../../agents/bash-tools.js"; import { resolveFastModeState } from "../../agents/fast-mode.js"; import { resolveAgentHarnessPolicy } from "../../agents/harness/selection.js"; +import { normalizeProviderId } from "../../agents/model-selection.js"; import { listOpenAIAuthProfileProvidersForAgentRuntime } from "../../agents/openai-codex-routing.js"; import { resolveEmbeddedFullAccessState } from "../../agents/pi-embedded-runner/sandbox-info.js"; import type { EmbeddedFullAccessBlockedReason } from "../../agents/pi-embedded-runner/types.js"; @@ -75,6 +76,7 @@ import { resolveBareSessionResetPromptState } from "./session-reset-prompt.js"; import { resolveBareResetBootstrapFileAccess } from "./session-reset-prompt.js"; import { drainFormattedSystemEventBlock } from "./session-system-events.js"; import { buildSessionStartupContextPrelude, shouldApplyStartupContext } from "./startup-context.js"; +import { resolveStoredModelOverride } from "./stored-model-override.js"; import { resolveTypingMode } from "./typing-mode.js"; import { resolveRunTypingPolicy } from "./typing-policy.js"; import type { TypingController } from "./typing.js"; @@ -348,6 +350,9 @@ type RunPreparedReplyParams = { storePath?: string; workspaceDir: string; abortedLastRun: boolean; + hasAppliedImageModelOverride?: boolean; + imageModelOverrideBaseProvider?: string; + imageModelFallbacksOverride?: string[]; }; export async function runPreparedReply( @@ -378,6 +383,7 @@ export async function runPreparedReply( perMessageQueueOptions, typing, opts, + defaultProvider, defaultModel, timeoutMs, isNewSession, @@ -388,6 +394,9 @@ export async function runPreparedReply( storePath, workspaceDir, sessionStore, + hasAppliedImageModelOverride, + imageModelOverrideBaseProvider, + imageModelFallbacksOverride, } = params; const runtimePolicySessionKey = resolveRuntimePolicySessionKey({ cfg, @@ -873,21 +882,72 @@ export async function runPreparedReply( harnessRuntime: agentHarnessPolicy.runtime, }) : [provider]; - let authProfileId = useFastReplyRuntime - ? preparedSessionState.sessionEntry?.authProfileOverride - : await traceRunPhase("reply.resolve_auth_profile", () => - resolveSessionAuthProfileOverride({ - cfg, - provider, - acceptedProviderIds: resolveAcceptedAuthProfileProviders(), - agentDir, - sessionEntry: preparedSessionState.sessionEntry, - sessionStore, - sessionKey, - storePath, - isNewSession, - }), - ); + const resolveActiveSessionProviderForAuthProfile = (): string => { + const storedOverride = resolveStoredModelOverride({ + sessionEntry: preparedSessionState.sessionEntry, + sessionStore, + sessionKey, + parentSessionKey: + preparedSessionState.sessionEntry?.parentSessionKey ?? + sessionCtx.ModelParentSessionKey ?? + sessionCtx.ParentSessionKey, + defaultProvider, + }); + return storedOverride?.provider ?? defaultProvider; + }; + const shouldResolveEphemeralAuthProfileForImageOverride = (): boolean => { + if (hasAppliedImageModelOverride !== true) { + return false; + } + const activeSessionProvider = + normalizeOptionalString(imageModelOverrideBaseProvider) ?? + resolveActiveSessionProviderForAuthProfile(); + return normalizeProviderId(provider) !== normalizeProviderId(activeSessionProvider); + }; + const resolveRuntimeAuthProfile = async (): Promise<{ + authProfileId?: string; + authProfileIdSource?: "auto" | "user"; + }> => { + if (useFastReplyRuntime) { + return { + authProfileId: preparedSessionState.sessionEntry?.authProfileOverride, + authProfileIdSource: preparedSessionState.sessionEntry?.authProfileOverrideSource, + }; + } + const shouldUseEphemeralSession = shouldResolveEphemeralAuthProfileForImageOverride(); + const authSessionKey = shouldUseEphemeralSession ? (sessionKey ?? sessionIdFinal) : sessionKey; + const authSessionEntry = + shouldUseEphemeralSession && preparedSessionState.sessionEntry + ? { ...preparedSessionState.sessionEntry } + : preparedSessionState.sessionEntry; + const authSessionStore = + shouldUseEphemeralSession && authSessionEntry + ? { [authSessionKey]: authSessionEntry } + : sessionStore; + const resolvedAuthProfileId = await resolveSessionAuthProfileOverride({ + cfg, + provider, + acceptedProviderIds: resolveAcceptedAuthProfileProviders(), + agentDir, + sessionEntry: authSessionEntry, + sessionStore: authSessionStore, + sessionKey: authSessionKey, + storePath: shouldUseEphemeralSession ? undefined : storePath, + isNewSession, + }); + return { + authProfileId: resolvedAuthProfileId, + authProfileIdSource: + resolvedAuthProfileId && authSessionEntry?.authProfileOverride === resolvedAuthProfileId + ? authSessionEntry.authProfileOverrideSource + : undefined, + }; + }; + let authProfileId: string | undefined; + let authProfileIdSource: "auto" | "user" | undefined; + ({ authProfileId, authProfileIdSource } = await traceRunPhase("reply.resolve_auth_profile", () => + resolveRuntimeAuthProfile(), + )); const { runReplyAgent } = await traceRunPhase("reply.load_agent_runner_runtime", () => loadAgentRunnerRuntime(), ); @@ -936,19 +996,7 @@ export async function runPreparedReply( piRuntime?.waitForEmbeddedPiRunEnd(activeRunSessionId) ?? Promise.resolve(undefined), refreshPreparedState: async () => { preparedSessionState = resolvePreparedSessionState(); - authProfileId = useFastReplyRuntime - ? preparedSessionState.sessionEntry?.authProfileOverride - : await resolveSessionAuthProfileOverride({ - cfg, - provider, - acceptedProviderIds: resolveAcceptedAuthProfileProviders(), - agentDir, - sessionEntry: preparedSessionState.sessionEntry, - sessionStore, - sessionKey, - storePath, - isNewSession, - }); + ({ authProfileId, authProfileIdSource } = await resolveRuntimeAuthProfile()); preparedSessionState = resolvePreparedSessionState(); ({ prefixedCommandBody, queuedBody, transcriptCommandBody, currentTurnContext } = await traceRunPhase("reply.build_prompt_bodies", () => rebuildPromptBodies())); @@ -961,7 +1009,6 @@ export async function runPreparedReply( } ({ activeSessionId, isActive, isStreaming } = queueState.busyState); } - const authProfileIdSource = preparedSessionState.sessionEntry?.authProfileOverrideSource; const runHasSessionModelOverride = Boolean( normalizeOptionalString(preparedSessionState.sessionEntry?.modelOverride) || normalizeOptionalString(preparedSessionState.sessionEntry?.providerOverride), @@ -1024,9 +1071,11 @@ export async function runPreparedReply( skillsSnapshot, provider, model, + hasOneTurnModelOverride: hasAppliedImageModelOverride || undefined, hasSessionModelOverride: runHasSessionModelOverride, modelOverrideSource: runModelOverrideSource, hasAutoFallbackProvenance: runHasAutoFallbackProvenance || undefined, + imageModelFallbacksOverride, authProfileId, authProfileIdSource, thinkLevel: resolvedThinkLevel, diff --git a/src/auto-reply/reply/get-reply.fast-path.test.ts b/src/auto-reply/reply/get-reply.fast-path.test.ts index ac566e497046..56d8be6ec695 100644 --- a/src/auto-reply/reply/get-reply.fast-path.test.ts +++ b/src/auto-reply/reply/get-reply.fast-path.test.ts @@ -61,6 +61,7 @@ registerGetReplyRuntimeOverrides(mocks); let getReplyFromConfig: typeof import("./get-reply.js").getReplyFromConfig; let resolveDefaultModelMock: typeof import("./directive-handling.defaults.js").resolveDefaultModel; +let resolveModelRefFromStringMock: typeof import("../../agents/model-selection.js").resolveModelRefFromString; let loadConfigMock: typeof import("../../config/config.js").getRuntimeConfig; let runPreparedReplyMock: typeof import("./get-reply-run.js").runPreparedReply; @@ -68,6 +69,8 @@ async function loadGetReplyRuntimeForTest() { ({ getReplyFromConfig } = await loadGetReplyModuleForTest({ cacheKey: import.meta.url })); ({ resolveDefaultModel: resolveDefaultModelMock } = await import("./directive-handling.defaults.js")); + ({ resolveModelRefFromString: resolveModelRefFromStringMock } = + await import("../../agents/model-selection.js")); ({ getRuntimeConfig: loadConfigMock } = await import("../../config/config.js")); ({ runPreparedReply: runPreparedReplyMock } = await import("./get-reply-run.js")); } @@ -82,7 +85,13 @@ function requirePreparedReplyParams() { function requireDirectiveParams() { const directiveParams = mocks.resolveReplyDirectives.mock.calls[0]?.[0] as - | { sessionKey?: string; workspaceDir?: string } + | { + sessionKey?: string; + workspaceDir?: string; + provider?: string; + model?: string; + hasOneTurnModelOverride?: boolean; + } | undefined; if (!directiveParams) { throw new Error("expected directive params"); @@ -115,6 +124,8 @@ describe("getReplyFromConfig fast test bootstrap", () => { defaultModel: "gpt-4o-mini", aliasIndex: emptyAliasIndex(), }); + vi.mocked(resolveModelRefFromStringMock).mockReset(); + vi.mocked(resolveModelRefFromStringMock).mockReturnValue(null); vi.mocked(loadConfigMock).mockReset(); vi.mocked(runPreparedReplyMock).mockReset(); vi.mocked(loadConfigMock).mockReturnValue({}); @@ -193,6 +204,40 @@ describe("getReplyFromConfig fast test bootstrap", () => { expect(vi.mocked(runPreparedReplyMock)).toHaveBeenCalledOnce(); }); + it("passes image model overrides as one-turn selections to prepared replies", async () => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-image-model-directives-")); + const cfg = markCompleteReplyConfig({ + agents: { + defaults: { + model: "openai/gpt-4o", + models: { + "openai/gpt-4o": {}, + }, + workspace: home, + }, + }, + session: { store: path.join(home, "sessions.json") }, + } as OpenClawConfig); + vi.mocked(resolveDefaultModelMock).mockReturnValueOnce({ + defaultProvider: "openai", + defaultModel: "gpt-4o", + aliasIndex: emptyAliasIndex(), + }); + vi.mocked(resolveModelRefFromStringMock).mockReturnValueOnce({ + ref: { provider: "openai", model: "gpt-4o-mini" }, + }); + + await expect( + getReplyFromConfig(buildGetReplyCtx(), { modelOverride: "openai/gpt-4o-mini" }, cfg), + ).resolves.toEqual({ text: "ok" }); + + expect(mocks.resolveReplyDirectives).not.toHaveBeenCalled(); + const preparedReplyParams = requirePreparedReplyParams(); + expect(preparedReplyParams.provider).toBe("openai"); + expect(preparedReplyParams.model).toBe("gpt-4o-mini"); + expect(preparedReplyParams.hasAppliedImageModelOverride).toBe(true); + }); + it("clears stale ack-only heartbeat pending delivery before replay", async () => { const home = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-heartbeat-pending-clear-")); const storePath = path.join(home, "sessions.json"); diff --git a/src/auto-reply/reply/get-reply.ts b/src/auto-reply/reply/get-reply.ts index 3c1ca289d070..7651c4d202c9 100644 --- a/src/auto-reply/reply/get-reply.ts +++ b/src/auto-reply/reply/get-reply.ts @@ -6,7 +6,7 @@ import { resolveSessionAgentId, resolveAgentSkillsFilter, } from "../../agents/agent-scope.js"; -import { resolveModelRefFromString } from "../../agents/model-selection.js"; +import { modelKey, resolveModelRefFromString } from "../../agents/model-selection.js"; import { resolveAgentTimeoutMs } from "../../agents/timeout.js"; import { DEFAULT_AGENT_WORKSPACE_DIR, ensureAgentWorkspace } from "../../agents/workspace.js"; import { resolveChannelModelOverride } from "../../channels/model-overrides.js"; @@ -253,7 +253,28 @@ export async function getReplyFromConfig( let provider = defaultProvider; let model = defaultModel; let hasResolvedHeartbeatModelOverride = false; - if (opts?.isHeartbeat) { + let hasAppliedImageModelOverride = false; + let imageModelFallbacksOverride: string[] | undefined; + const modelOverrideRaw = normalizeOptionalString(opts?.modelOverride); + if (modelOverrideRaw) { + const modelOverrideRef = resolveModelRefFromString({ + raw: modelOverrideRaw, + defaultProvider, + aliasIndex, + }); + if (modelOverrideRef) { + provider = modelOverrideRef.ref.provider; + model = modelOverrideRef.ref.model; + hasAppliedImageModelOverride = true; + imageModelFallbacksOverride = opts?.modelOverrideFallbacks?.filter( + (fallback): fallback is string => normalizeOptionalString(fallback) !== undefined, + ); + } else { + defaultRuntime.log?.( + `[image-model-switch] Failed to resolve image model override ${modelOverrideRaw}; using default model ${modelKey(defaultProvider, defaultModel)}`, + ); + } + } else if (opts?.isHeartbeat) { // Prefer the resolved per-agent heartbeat model passed from the heartbeat runner, // fall back to the global defaults heartbeat model for backward compatibility. const heartbeatRaw = @@ -530,6 +551,7 @@ export async function getReplyFromConfig( if ( storedModelOverride?.model && !hasResolvedHeartbeatModelOverride && + !hasAppliedImageModelOverride && !staleHeartbeatAutoFallbackOverride ) { provider = storedModelOverride.provider ?? defaultProvider; @@ -540,11 +562,32 @@ export async function getReplyFromConfig( if ( !hasResolvedHeartbeatModelOverride && !hasEffectiveSessionModelOverride && + !hasAppliedImageModelOverride && resolvedChannelModelOverride ) { provider = resolvedChannelModelOverride.ref.provider; model = resolvedChannelModelOverride.ref.model; } + const imageModelOverrideBaseProvider = hasAppliedImageModelOverride + ? (() => { + if ( + storedModelOverride?.model && + !hasResolvedHeartbeatModelOverride && + !staleHeartbeatAutoFallbackOverride + ) { + return storedModelOverride.provider ?? defaultProvider; + } + if (!hasEffectiveSessionModelOverride && resolvedChannelModelOverride) { + return resolvedChannelModelOverride.ref.provider; + } + const runtimeProvider = normalizeOptionalString(sessionEntry.modelProvider); + const runtimeModel = normalizeOptionalString(sessionEntry.model); + if (runtimeProvider && runtimeModel) { + return runtimeProvider; + } + return defaultProvider; + })() + : undefined; if ( shouldUseReplyFastDirectiveExecution({ @@ -619,6 +662,9 @@ export async function getReplyFromConfig( storePath, workspaceDir, abortedLastRun, + hasAppliedImageModelOverride, + imageModelOverrideBaseProvider, + imageModelFallbacksOverride, }), ); } @@ -649,6 +695,7 @@ export async function getReplyFromConfig( aliasIndex, provider, model, + hasOneTurnModelOverride: hasAppliedImageModelOverride, hasResolvedHeartbeatModelOverride, typing, opts: resolvedOpts, @@ -859,6 +906,9 @@ export async function getReplyFromConfig( storePath, workspaceDir, abortedLastRun, + hasAppliedImageModelOverride, + imageModelOverrideBaseProvider, + imageModelFallbacksOverride, }), ); } diff --git a/src/auto-reply/reply/image-model-override-plan.test.ts b/src/auto-reply/reply/image-model-override-plan.test.ts new file mode 100644 index 000000000000..28b9a1e465e9 --- /dev/null +++ b/src/auto-reply/reply/image-model-override-plan.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it, vi } from "vitest"; +import type { OpenClawConfig } from "../../config/types.openclaw.js"; +import { resolveImageModelOverridePlan } from "./image-model-override-plan.js"; + +function buildConfig(params: { + imageModel?: NonNullable["defaults"]>["imageModel"]; + models?: Record; +}): OpenClawConfig { + return { + agents: { + defaults: { + ...(params.imageModel ? { imageModel: params.imageModel } : {}), + ...(params.models ? { models: params.models } : {}), + }, + }, + } as OpenClawConfig; +} + +describe("resolveImageModelOverridePlan", () => { + it("uses the session model when it already supports images", async () => { + const modelSupportsImages = vi.fn(async () => true); + + const plan = await resolveImageModelOverridePlan({ + cfg: buildConfig({ imageModel: "openai/gpt-4o" }), + defaultProvider: "anthropic", + defaultModel: "claude-opus-4-6", + hasImageAttachments: true, + sessionModelSupportsImages: true, + modelSupportsImages, + }); + + expect(plan).toEqual({ kind: "inline-session" }); + expect(modelSupportsImages).not.toHaveBeenCalled(); + }); + + it("keeps configured image models reachable when a model allowlist is present", async () => { + const plan = await resolveImageModelOverridePlan({ + cfg: buildConfig({ + imageModel: { + primary: "openai/gpt-4o", + fallbacks: ["openai/gpt-4o-mini"], + }, + models: { "anthropic/claude-opus-4-6": {} }, + }), + defaultProvider: "anthropic", + defaultModel: "claude-opus-4-6", + hasImageAttachments: true, + sessionModelSupportsImages: false, + modelSupportsImages: async () => true, + }); + + expect(plan).toEqual({ + kind: "inline-image-model", + modelOverride: "openai/gpt-4o", + modelOverrideFallbacks: ["openai/gpt-4o-mini"], + }); + }); + + it("resolves providerless image models independently of the active session provider", async () => { + const modelSupportsImages = vi.fn(async (ref: { provider: string; model: string }) => { + return ref.provider === "openai" && ref.model === "gpt-4o"; + }); + + const plan = await resolveImageModelOverridePlan({ + cfg: buildConfig({ imageModel: "gpt-4o" }), + defaultProvider: "openai", + defaultModel: "gpt-5.5", + hasImageAttachments: true, + sessionModelSupportsImages: false, + modelSupportsImages, + }); + + expect(plan).toEqual({ + kind: "inline-image-model", + modelOverride: "openai/gpt-4o", + modelOverrideFallbacks: [], + }); + expect(modelSupportsImages).toHaveBeenCalledWith({ + provider: "openai", + model: "gpt-4o", + }); + }); + + it("uses the configured default provider for unmatched providerless image models", async () => { + const modelSupportsImages = vi.fn(async (ref: { provider: string; model: string }) => { + return ref.provider === "ollama" && ref.model === "qwen2.5vl:7b"; + }); + + const plan = await resolveImageModelOverridePlan({ + cfg: buildConfig({ imageModel: "qwen2.5vl:7b" }), + defaultProvider: "ollama", + defaultModel: "llama3.2", + hasImageAttachments: true, + sessionModelSupportsImages: false, + modelSupportsImages, + }); + + expect(plan).toEqual({ + kind: "inline-image-model", + modelOverride: "ollama/qwen2.5vl:7b", + modelOverrideFallbacks: [], + }); + }); + + it("selects the first vision-capable image model and carries later image fallbacks", async () => { + const modelSupportsImages = vi.fn(async (ref: { provider: string; model: string }) => { + return ref.model !== "gpt-4o-blocked"; + }); + + const plan = await resolveImageModelOverridePlan({ + cfg: buildConfig({ + imageModel: { + primary: "openai/gpt-4o-blocked", + fallbacks: ["openai/gpt-4o", "google/gemini-2.5-flash"], + }, + models: { + "openai/gpt-4o-blocked": {}, + "openai/gpt-4o": {}, + "google/gemini-2.5-flash": {}, + }, + }), + defaultProvider: "anthropic", + defaultModel: "claude-opus-4-6", + hasImageAttachments: true, + sessionModelSupportsImages: false, + modelSupportsImages, + }); + + expect(plan).toEqual({ + kind: "inline-image-model", + modelOverride: "openai/gpt-4o", + modelOverrideFallbacks: ["google/gemini-2.5-flash"], + }); + }); +}); diff --git a/src/auto-reply/reply/image-model-override-plan.ts b/src/auto-reply/reply/image-model-override-plan.ts new file mode 100644 index 000000000000..31d9b0b7bffb --- /dev/null +++ b/src/auto-reply/reply/image-model-override-plan.ts @@ -0,0 +1,122 @@ +import { + buildModelAliasIndex, + inferUniqueProviderFromConfiguredModels, + modelKey, + resolveModelRefFromString, + type ModelRef, +} from "../../agents/model-selection.js"; +import { + resolveAgentModelFallbackValues, + resolveAgentModelPrimaryValue, +} from "../../config/model-input.js"; +import type { OpenClawConfig } from "../../config/types.openclaw.js"; + +export type ImageModelOverridePlan = + | { + kind: "inline-session"; + } + | { + kind: "inline-image-model"; + modelOverride: string; + modelOverrideFallbacks: string[]; + } + | { + kind: "media-paths"; + reason: "no-image-attachments" | "no-image-model" | "not-vision-capable"; + }; + +export type ImageModelCapabilityResolver = (ref: ModelRef) => Promise; + +type ImageModelCandidate = { + raw: string; + ref: ModelRef; + key: string; +}; + +function resolveImageModelCandidate(params: { + raw: string; + cfg: OpenClawConfig; + defaultProvider: string; +}): ImageModelCandidate | null { + const trimmed = params.raw.trim(); + if (!trimmed) { + return null; + } + const imageDefaultProvider = trimmed.includes("/") + ? params.defaultProvider + : (inferUniqueProviderFromConfiguredModels({ cfg: params.cfg, model: trimmed }) ?? + params.defaultProvider); + const aliasIndex = buildModelAliasIndex({ + cfg: params.cfg, + defaultProvider: imageDefaultProvider, + }); + const resolved = resolveModelRefFromString({ + cfg: params.cfg, + raw: trimmed, + defaultProvider: imageDefaultProvider, + aliasIndex, + }); + if (!resolved) { + return null; + } + const ref = resolved.ref; + return { + raw: trimmed, + ref, + key: modelKey(ref.provider, ref.model), + }; +} + +export async function resolveImageModelOverridePlan(params: { + cfg: OpenClawConfig; + agentId?: string; + defaultProvider: string; + defaultModel: string; + hasImageAttachments: boolean; + sessionModelSupportsImages: boolean; + modelSupportsImages: ImageModelCapabilityResolver; +}): Promise { + if (!params.hasImageAttachments) { + return { kind: "media-paths", reason: "no-image-attachments" }; + } + if (params.sessionModelSupportsImages) { + return { kind: "inline-session" }; + } + + const imageModelConfig = params.cfg.agents?.defaults?.imageModel; + const primary = resolveAgentModelPrimaryValue(imageModelConfig); + const rawCandidates = [ + ...(primary ? [primary] : []), + ...resolveAgentModelFallbackValues(imageModelConfig), + ]; + if (rawCandidates.length === 0) { + return { kind: "media-paths", reason: "no-image-model" }; + } + + const runnableCandidates: ImageModelCandidate[] = []; + for (const raw of rawCandidates) { + const candidate = resolveImageModelCandidate({ + raw, + cfg: params.cfg, + defaultProvider: params.defaultProvider, + }); + if (!candidate) { + continue; + } + if (!(await params.modelSupportsImages(candidate.ref))) { + continue; + } + runnableCandidates.push(candidate); + } + + const selected = runnableCandidates[0]; + if (!selected) { + return { kind: "media-paths", reason: "not-vision-capable" }; + } + + return { + kind: "inline-image-model", + modelOverride: selected.key, + modelOverrideFallbacks: runnableCandidates.slice(1).map((candidate) => candidate.key), + }; +} diff --git a/src/auto-reply/reply/model-selection.test.ts b/src/auto-reply/reply/model-selection.test.ts index 49d5c9d9dc12..09a0d9d06ed2 100644 --- a/src/auto-reply/reply/model-selection.test.ts +++ b/src/auto-reply/reply/model-selection.test.ts @@ -765,6 +765,45 @@ describe("createModelSelectionState respects session model override", () => { expect(sessionStore[sessionKey]?.providerOverride).toBeUndefined(); }); + it("keeps one-turn model overrides ahead of stored overrides and allowlist fallback", async () => { + const cfg = { + agents: { + defaults: { + model: { primary: "openai/gpt-4o" }, + models: { + "openai/gpt-4o": {}, + }, + }, + }, + } as OpenClawConfig; + const sessionKey = "agent:main:telegram:direct:1"; + const sessionEntry = makeEntry({ + providerOverride: "anthropic", + modelOverride: "claude-opus-4-6", + }); + const sessionStore = { [sessionKey]: sessionEntry }; + + const state = await createModelSelectionState({ + cfg, + agentCfg: cfg.agents?.defaults, + sessionEntry, + sessionStore, + sessionKey, + defaultProvider: "openai", + defaultModel: "gpt-4o", + provider: "openai", + model: "gpt-4o-mini", + hasModelDirective: false, + hasOneTurnModelOverride: true, + }); + + expect(state.provider).toBe("openai"); + expect(state.model).toBe("gpt-4o-mini"); + expect(state.resetModelOverride).toBe(false); + expect(sessionStore[sessionKey]?.providerOverride).toBe("anthropic"); + expect(sessionStore[sessionKey]?.modelOverride).toBe("claude-opus-4-6"); + }); + it("keeps wildcard-provider overrides when configured catalog rows are unavailable", async () => { const cfg = { agents: { diff --git a/src/auto-reply/reply/model-selection.ts b/src/auto-reply/reply/model-selection.ts index 58f614881cb3..2523d27d1dcc 100644 --- a/src/auto-reply/reply/model-selection.ts +++ b/src/auto-reply/reply/model-selection.ts @@ -103,6 +103,7 @@ export async function createModelSelectionState(params: { provider: string; model: string; hasModelDirective: boolean; + hasOneTurnModelOverride?: boolean; /** True when heartbeat.model was explicitly resolved for this run. * In that case, skip session-stored overrides so the heartbeat selection wins. */ hasResolvedHeartbeatModelOverride?: boolean; @@ -135,6 +136,7 @@ export async function createModelSelectionState(params: { let model = params.model; const primaryProvider = params.primaryProvider ?? defaultProvider; const primaryModel = params.primaryModel ?? defaultModel; + const hasOneTurnModelOverride = params.hasOneTurnModelOverride === true; const hasAllowlist = agentCfg?.models && Object.keys(agentCfg.models).length > 0; const visibility = parseConfiguredModelVisibilityEntries({ cfg }); @@ -214,7 +216,13 @@ export async function createModelSelectionState(params: { logStage("configured-catalog-ready", `entries=${configuredModelCatalog.length}`); } - if (sessionEntry && sessionStore && sessionKey && directStoredOverride) { + if ( + sessionEntry && + sessionStore && + sessionKey && + directStoredOverride && + !hasOneTurnModelOverride + ) { const normalizedOverride = normalizeModelRef( directStoredOverride.provider, directStoredOverride.model, @@ -272,6 +280,7 @@ export async function createModelSelectionState(params: { // overrides unless a direct auto fallback override is stale for the current // configured default. const skipStoredOverride = + hasOneTurnModelOverride || params.hasResolvedHeartbeatModelOverride === true || (staleHeartbeatAutoFallbackOverride && storedOverride?.source === "session"); @@ -287,7 +296,7 @@ export async function createModelSelectionState(params: { } } - if (!params.hasModelDirective) { + if (!params.hasModelDirective && !hasOneTurnModelOverride) { const allowedInitialSelection = visibilityPolicy.resolveSelection({ provider, model, diff --git a/src/auto-reply/reply/queue/types.ts b/src/auto-reply/reply/queue/types.ts index 13713a93392c..f3755c1a810c 100644 --- a/src/auto-reply/reply/queue/types.ts +++ b/src/auto-reply/reply/queue/types.ts @@ -85,9 +85,11 @@ export type FollowupRun = { skillsSnapshot?: SkillSnapshot; provider: string; model: string; + hasOneTurnModelOverride?: boolean; hasSessionModelOverride?: boolean; modelOverrideSource?: "auto" | "user"; hasAutoFallbackProvenance?: boolean; + imageModelFallbacksOverride?: string[]; authProfileId?: string; authProfileIdSource?: "auto" | "user"; thinkLevel?: ThinkLevel; diff --git a/src/gateway/chat-attachments.ts b/src/gateway/chat-attachments.ts index 4c3cf0d02cdb..5c222e8d74cc 100644 --- a/src/gateway/chat-attachments.ts +++ b/src/gateway/chat-attachments.ts @@ -428,6 +428,23 @@ export async function parseMessageWithAttachments( }; } +export async function resolveChatAttachmentLooksLikeImage( + attachment: ChatAttachment, + index = 0, +): Promise { + const normalized = normalizeAttachment(attachment, index, { + stripDataUrlPrefix: true, + requireImageMime: false, + }); + if (!isValidBase64(normalized.base64)) { + throw new Error(`attachment ${normalized.label}: invalid base64 content`); + } + const providedMime = normalizeMime(normalized.mime); + const sniffedMime = normalizeMime(await sniffMimeFromBase64(normalized.base64)); + const labelMime = normalizeMime(mimeTypeFromFilePath(normalized.label)); + return isImageMime(resolveAttachmentMime({ sniffedMime, providedMime, labelMime })); +} + /** * @deprecated Use parseMessageWithAttachments instead. * This function converts images to markdown data URLs which Claude API cannot process as images. diff --git a/src/gateway/server-methods/chat.ts b/src/gateway/server-methods/chat.ts index d9ca63b20713..805d5f181606 100644 --- a/src/gateway/server-methods/chat.ts +++ b/src/gateway/server-methods/chat.ts @@ -5,12 +5,14 @@ import type { AgentMessage } from "@earendil-works/pi-agent-core"; import { CURRENT_SESSION_VERSION } from "@earendil-works/pi-coding-agent"; import { resolveSendableOutboundReplyParts } from "openclaw/plugin-sdk/reply-payload"; import { resolveAgentWorkspaceDir, resolveSessionAgentId } from "../../agents/agent-scope.js"; +import { resolveDefaultModelForAgent } from "../../agents/model-selection.js"; import { rewriteTranscriptEntriesInSessionFile } from "../../agents/pi-embedded-runner/transcript-rewrite.js"; import { resolveProviderIdForAuth } from "../../agents/provider-auth-aliases.js"; import { ensureSandboxWorkspaceForSession } from "../../agents/sandbox/context.js"; import { resolveAgentTimeoutMs } from "../../agents/timeout.js"; import { dispatchInboundMessage } from "../../auto-reply/dispatch.js"; import type { ReplyPayload } from "../../auto-reply/reply-payload.js"; +import { resolveImageModelOverridePlan } from "../../auto-reply/reply/image-model-override-plan.js"; import { createReplyDispatcher } from "../../auto-reply/reply/reply-dispatcher.js"; import { stageSandboxMedia } from "../../auto-reply/reply/stage-sandbox-media.js"; import type { MsgContext, TemplateContext } from "../../auto-reply/templating.js"; @@ -66,10 +68,12 @@ import { updateChatRunProvider, } from "../chat-abort.js"; import { + type ChatAttachment, type ChatImageContent, MediaOffloadError, type OffloadedRef, parseMessageWithAttachments, + resolveChatAttachmentLooksLikeImage, resolveChatAttachmentMaxBytes, UnsupportedAttachmentError, } from "../chat-attachments.js"; @@ -179,6 +183,15 @@ function stripVisibleTextFromTtsSupplement(payload: ReplyPayload): ReplyPayload return isTtsSupplementPayload(payload) ? { ...payload, text: undefined } : payload; } +async function hasImageChatAttachments(attachments: ChatAttachment[]): Promise { + for (const [index, attachment] of attachments.entries()) { + if (await resolveChatAttachmentLooksLikeImage(attachment, index)) { + return true; + } + } + return false; +} + async function buildWebchatAssistantMediaMessage( payloads: ReplyPayload[], options?: { @@ -2030,6 +2043,7 @@ export const chatHandlers: GatewayRequestHandlers = { sessionKey, config: cfg, }); + const resolvedConfiguredDefaultModel = resolveDefaultModelForAgent({ cfg, agentId }); const resolvedSessionModel = resolveSessionModelRef(cfg, entry, agentId); const resolvedSessionAuthProvider = resolveProviderIdForAuth(resolvedSessionModel.provider, { config: cfg, @@ -2132,22 +2146,47 @@ export const chatHandlers: GatewayRequestHandlers = { const explicitOriginTargetsPlugin = explicitOriginTargetsPluginBinding( explicitOriginResult.value, ); + let modelOverride: string | undefined; + let modelOverrideFallbacks: string[] | undefined; if (normalizedAttachments.length > 0) { try { await measureDiagnosticsTimelineSpan( "gateway.chat_send.prepare_attachments", async () => { + const hasImageAttachments = await hasImageChatAttachments(normalizedAttachments); const supportsSessionModelImages = await resolveGatewayModelSupportsImages({ loadGatewayModelCatalog: context.loadGatewayModelCatalog, provider: resolvedSessionModel.provider, model: resolvedSessionModel.model, }); + const explicitOriginSupportsInlineImages = + explicitOriginTargetsAcpSession(explicitOriginResult.value) || + explicitOriginTargetsPlugin; + const imageModelPlan = await resolveImageModelOverridePlan({ + cfg, + agentId, + defaultProvider: resolvedConfiguredDefaultModel.provider, + defaultModel: resolvedConfiguredDefaultModel.model, + hasImageAttachments, + sessionModelSupportsImages: + supportsSessionModelImages || explicitOriginSupportsInlineImages, + modelSupportsImages: (ref) => + resolveGatewayModelSupportsImages({ + loadGatewayModelCatalog: context.loadGatewayModelCatalog, + provider: ref.provider, + model: ref.model, + }), + }); + if (imageModelPlan.kind === "inline-image-model") { + modelOverride = imageModelPlan.modelOverride; + modelOverrideFallbacks = imageModelPlan.modelOverrideFallbacks; + } // Bound plugin sessions own the real recipient model, so keep image // attachments even when the parent OpenClaw session model is text-only. const supportsImages = - supportsSessionModelImages || - explicitOriginTargetsAcpSession(explicitOriginResult.value) || - explicitOriginTargetsPlugin; + imageModelPlan.kind === "inline-session" || + imageModelPlan.kind === "inline-image-model" || + explicitOriginSupportsInlineImages; const routeImageOffloadsAsMediaPaths = !supportsImages; const parsed = await parseMessageWithAttachments( inboundMessage, @@ -2546,6 +2585,8 @@ export const chatHandlers: GatewayRequestHandlers = { abortSignal: activeRunAbort.controller.signal, images: parsedImages.length > 0 ? parsedImages : undefined, imageOrder: imageOrder.length > 0 ? imageOrder : undefined, + modelOverride, + modelOverrideFallbacks, thinkingLevelOverride: p.thinking, fastModeOverride: p.fastMode, onAgentRunStart: (runId) => { diff --git a/src/gateway/server.chat.gateway-server-chat-b.test.ts b/src/gateway/server.chat.gateway-server-chat-b.test.ts index 1e41916dde7f..c979fb9f2a92 100644 --- a/src/gateway/server.chat.gateway-server-chat-b.test.ts +++ b/src/gateway/server.chat.gateway-server-chat-b.test.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { afterAll, beforeAll, describe, expect, test, vi } from "vitest"; import type { GetReplyOptions } from "../auto-reply/get-reply-options.types.js"; import { clearConfigCache } from "../config/config.js"; +import type { AgentModelConfig } from "../config/types.agents-shared.js"; import { GATEWAY_CLIENT_MODES, GATEWAY_CLIENT_NAMES } from "../utils/message-channel.js"; import { __setMaxChatHistoryMessagesBytesForTest } from "./server-constants.js"; import type { GatewayRequestContext, RespondFn } from "./server-methods/shared-types.js"; @@ -128,6 +129,25 @@ async function fetchHistoryMessages( return historyRes.payload?.messages ?? []; } +type ConfiguredImageModelCase = { + id: string; + imageModel: AgentModelConfig; + expectedFallbacks: string[]; +}; + +const configuredImageModelCases: ConfiguredImageModelCase[] = [ + { + id: "with-image-fallback", + imageModel: { primary: "openai/gpt-4o", fallbacks: ["openai/gpt-4o-mini"] }, + expectedFallbacks: ["openai/gpt-4o-mini"], + }, + { + id: "without-image-fallback", + imageModel: { primary: "openai/gpt-4o" }, + expectedFallbacks: [], + }, +]; + async function prepareMainHistoryHarness(params: { ws: GatewaySocket; createSessionDir: () => Promise; @@ -344,6 +364,158 @@ describe("gateway server chat", () => { } }); + test.each(configuredImageModelCases)( + "chat.send inlines image attachments through configured imageModel with allowlist present: $id", + async ({ id, imageModel, expectedFallbacks }) => { + const sessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-gw-")); + try { + testState.sessionStorePath = path.join(sessionDir, "sessions.json"); + testState.agentConfig = { + model: { + primary: "anthropic/claude-opus-4-6", + fallbacks: ["anthropic/claude-haiku-4-6"], + }, + imageModel, + models: { + "anthropic/claude-opus-4-6": {}, + }, + }; + await writeSessionStore({ + entries: { + main: { + sessionId: "sess-main", + modelProvider: "anthropic", + model: "claude-opus-4-6", + updatedAt: Date.now(), + }, + }, + }); + + const context = { + loadGatewayModelCatalog: vi.fn( + async () => [ + { + id: "claude-opus-4-6", + name: "Claude Opus 4.6", + provider: "anthropic", + input: ["text"], + }, + { + id: "gpt-4o", + name: "GPT-4o", + provider: "openai", + input: ["text", "image"], + }, + { + id: "gpt-4o-mini", + name: "GPT-4o mini", + provider: "openai", + input: ["text", "image"], + }, + { + id: "claude-haiku-4-6", + name: "Claude Haiku 4.6", + provider: "anthropic", + input: ["text"], + }, + ], + ), + logGateway: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + }, + agentRunSeq: new Map(), + chatAbortControllers: new Map(), + chatAbortedRuns: new Map(), + chatRunBuffers: new Map(), + chatDeltaSentAt: new Map(), + chatDeltaLastBroadcastLen: new Map(), + chatDeltaLastBroadcastText: new Map(), + addChatRun: vi.fn(), + removeChatRun: vi.fn(), + broadcast: vi.fn(), + nodeSendToSession: vi.fn(), + registerToolEventRecipient: vi.fn(), + dedupe: new Map(), + } as unknown as GatewayRequestContext; + const pngB64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/woAAn8B9FD5fHAAAAAASUVORK5CYII="; + let captured: { ctx?: Record; replyOptions?: GetReplyOptions } | undefined; + dispatchInboundMessageMock.mockImplementationOnce(async (...args: unknown[]) => { + const [params] = args as [ + { + ctx: Record; + replyOptions?: GetReplyOptions; + }, + ]; + captured = { + ctx: params.ctx, + replyOptions: params.replyOptions, + }; + }); + + const { chatHandlers } = await import("./server-methods/chat.js"); + const responses: Array<{ ok: boolean; payload?: unknown; error?: unknown }> = []; + await chatHandlers["chat.send"]({ + req: { + type: "req", + id: `configured-image-model-${id}`, + method: "chat.send", + params: { + sessionKey: "main", + message: "see image", + idempotencyKey: `idem-configured-image-model-${id}`, + attachments: [ + { + type: "image", + mimeType: "image/png", + fileName: "dot.png", + content: pngB64, + }, + ], + }, + }, + params: { + sessionKey: "main", + message: "see image", + idempotencyKey: `idem-configured-image-model-${id}`, + attachments: [ + { + type: "image", + mimeType: "image/png", + fileName: "dot.png", + content: pngB64, + }, + ], + }, + client: null, + isWebchatConnect: () => false, + respond: ((ok, payload, error) => { + responses.push({ ok, payload, error }); + }) as RespondFn, + context, + }); + + expect(responses[0]?.ok).toBe(true); + await vi.waitFor(() => expect(captured).toBeDefined(), FAST_WAIT_OPTS); + expect(captured?.replyOptions?.images).toEqual([ + expect.objectContaining({ type: "image", mimeType: "image/png" }), + ]); + expect(captured?.replyOptions?.modelOverride).toBe("openai/gpt-4o"); + expect(captured?.replyOptions?.modelOverrideFallbacks).toEqual(expectedFallbacks); + expect(captured?.ctx?.MediaPaths).toBeUndefined(); + } finally { + dispatchInboundMessageMock.mockReset(); + testState.agentConfig = undefined; + testState.sessionStorePath = undefined; + clearConfigCache(); + await fs.rm(sessionDir, { recursive: true, force: true }); + } + }, + ); + test("chat.send reuses an active internal run for duplicate WebChat text sends", async () => { const sessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-gw-")); const dispatchRelease = createDeferred();