fix(gateway): route WebChat images through imageModel

Route WebChat image attachments through the configured imageModel when the active session model cannot accept images, while keeping one-turn image auth and fallback state ephemeral.

Thanks @frankekn.
This commit is contained in:
Frank Yang
2026-05-16 20:12:02 +08:00
committed by GitHub
parent e0870473b2
commit e6d04550ca
20 changed files with 1043 additions and 53 deletions
+1
View File
@@ -16,6 +16,7 @@ Docs: https://docs.openclaw.ai
### Fixes
- Gateway/WebChat: route image attachments through a configured vision-capable `imageModel` plan before inlining images, and carry that image-model fallback chain through runtime retries. (#82524) Thanks @frankekn.
- Codex app-server: limit canonical OpenAI Codex app-server attribution rewrites to local transcript and trajectory records, leaving runtime/tool routing on the selected OpenAI model metadata so OpenAI API-key backup profiles keep their billing path.
- Android/chat: make bare and markdown URLs in chat messages tappable by preserving Compose URL annotations in rendered markdown. Fixes #82187. (#82392) Thanks @neeravmakwana.
- Codex app-server: release raw assistant completions when `turn/completed` is missing while keeping commentary/status items as progress, preventing completed Codex runs from hanging until timeout. Fixes #82343. (#82403) Thanks @IWhatsskill.
@@ -196,4 +196,8 @@ export type GetReplyOptions = {
hasRepliedRef?: { value: boolean };
/** Override agent timeout in seconds (0 = no timeout). Threads through to resolveAgentTimeoutMs. */
timeoutOverrideSeconds?: number;
/** Capability-checked one-turn model override for inline image input. */
modelOverride?: string;
/** Capability-checked runtime fallbacks for the one-turn image model override. */
modelOverrideFallbacks?: string[];
};
@@ -4018,6 +4018,69 @@ describe("runAgentTurnWithFallback", () => {
expect(sessionStore.main.authProfileOverride).toBeUndefined();
});
it("does not persist fallback selection for one-turn image model overrides", async () => {
state.runWithModelFallbackMock.mockImplementation(
async (params: { run: (provider: string, model: string) => Promise<unknown> }) => ({
result: await params.run("openai", "gpt-4o-mini"),
provider: "openai",
model: "gpt-4o-mini",
attempts: [],
}),
);
state.runEmbeddedPiAgentMock.mockResolvedValue({
payloads: [{ text: "ok" }],
meta: {},
});
const followupRun = createFollowupRun();
followupRun.run.provider = "openai";
followupRun.run.model = "gpt-4o";
followupRun.run.hasOneTurnModelOverride = true;
const sessionEntry: SessionEntry = {
sessionId: "session",
updatedAt: Date.now(),
totalTokens: 1,
compactionCount: 0,
};
const sessionStore = { main: sessionEntry };
const runAgentTurnWithFallback = await getRunAgentTurnWithFallback();
const result = await runAgentTurnWithFallback({
commandBody: "hello",
followupRun,
sessionCtx: {
Provider: "telegram",
MessageSid: "msg",
} as unknown as TemplateContext,
opts: {},
typingSignals: createMockTypingSignaler(),
blockReplyPipeline: null,
blockStreamingEnabled: false,
resolvedBlockStreamingBreak: "message_end",
applyReplyToMode: (payload) => payload,
shouldEmitToolResult: () => true,
shouldEmitToolOutput: () => false,
pendingToolTasks: new Set(),
resetSessionAfterCompactionFailure: async () => false,
resetSessionAfterRoleOrderingConflict: async () => false,
isHeartbeat: false,
sessionKey: "main",
getActiveSessionEntry: () => sessionEntry,
activeSessionStore: sessionStore,
resolvedVerboseLevel: "off",
});
expect(result.kind).toBe("success");
expectMockCallArgFields(state.runEmbeddedPiAgentMock, 0, "embedded run params", {
provider: "openai",
model: "gpt-4o-mini",
});
expect(sessionEntry.providerOverride).toBeUndefined();
expect(sessionEntry.modelOverride).toBeUndefined();
expect(sessionEntry.modelOverrideSource).toBeUndefined();
});
it("does not persist fallback selection for legacy user overrides without modelOverrideSource", async () => {
// Regression: older persisted sessions can have a user-selected override
// (modelOverride set) but no modelOverrideSource field, because the field
@@ -1302,6 +1302,9 @@ export async function runAgentTurnWithFallback(params: {
provider: string,
model: string,
): Promise<(() => Promise<void>) | undefined> => {
if (params.followupRun.run.hasOneTurnModelOverride === true) {
return undefined;
}
if (
!params.sessionKey ||
!params.activeSessionStore ||
+18 -14
View File
@@ -31,13 +31,15 @@ export function resolveModelFallbackOptions(
configOverride: FollowupRun["run"]["config"] = run.config,
) {
const config = configOverride;
const fallbacksOverride = resolveEffectiveModelFallbacks({
cfg: config,
agentId: run.agentId,
hasSessionModelOverride: run.hasSessionModelOverride === true,
modelOverrideSource: run.modelOverrideSource,
hasAutoFallbackProvenance: run.hasAutoFallbackProvenance === true,
});
const fallbacksOverride =
run.imageModelFallbacksOverride ??
resolveEffectiveModelFallbacks({
cfg: config,
agentId: run.agentId,
hasSessionModelOverride: run.hasSessionModelOverride === true,
modelOverrideSource: run.modelOverrideSource,
hasAutoFallbackProvenance: run.hasAutoFallbackProvenance === true,
});
return {
cfg: config,
provider: run.provider,
@@ -57,13 +59,15 @@ export function buildEmbeddedRunBaseParams(params: {
isReasoningTagProvider?: ReasoningTagProviderResolver;
}) {
const config = params.run.config;
const modelFallbacksOverride = resolveEffectiveModelFallbacks({
cfg: config,
agentId: params.run.agentId,
hasSessionModelOverride: params.run.hasSessionModelOverride === true,
modelOverrideSource: params.run.modelOverrideSource,
hasAutoFallbackProvenance: params.run.hasAutoFallbackProvenance === true,
});
const modelFallbacksOverride =
params.run.imageModelFallbacksOverride ??
resolveEffectiveModelFallbacks({
cfg: config,
agentId: params.run.agentId,
hasSessionModelOverride: params.run.hasSessionModelOverride === true,
modelOverrideSource: params.run.modelOverrideSource,
hasAutoFallbackProvenance: params.run.hasAutoFallbackProvenance === true,
});
return {
sessionFile: params.run.sessionFile,
workspaceDir: params.run.workspaceDir,
@@ -103,6 +103,28 @@ describe("agent-runner-utils", () => {
expect(resolved.fallbacksOverride).toEqual(["fallback-model"]);
});
it("uses image model fallback overrides for model fallback options", () => {
const run = makeRun({
imageModelFallbacksOverride: ["openai/gpt-4o-mini"],
});
const resolved = resolveModelFallbackOptions(run);
expect(hoisted.resolveEffectiveModelFallbacksMock).not.toHaveBeenCalled();
expect(resolved.fallbacksOverride).toEqual(["openai/gpt-4o-mini"]);
});
it("preserves empty image model fallback overrides for model fallback options", () => {
const run = makeRun({
imageModelFallbacksOverride: [],
});
const resolved = resolveModelFallbackOptions(run);
expect(hoisted.resolveEffectiveModelFallbacksMock).not.toHaveBeenCalled();
expect(resolved.fallbacksOverride).toEqual([]);
});
it("passes through missing agentId for helper-based fallback resolution", () => {
hoisted.resolveEffectiveModelFallbacksMock.mockReturnValue(["fallback-model"]);
const run = makeRun({ agentId: undefined });
@@ -185,6 +207,48 @@ describe("agent-runner-utils", () => {
expect(resolved.modelFallbacksOverride).toEqual(["fallback-model"]);
});
it("uses image model fallback overrides for embedded run params", () => {
const run = makeRun({
imageModelFallbacksOverride: ["openai/gpt-4o-mini"],
});
const authProfile = resolveProviderScopedAuthProfile({
provider: "openai",
primaryProvider: "openai",
});
const resolved = buildEmbeddedRunBaseParams({
run,
provider: "openai",
model: "gpt-4o",
runId: "run-1",
authProfile,
});
expect(hoisted.resolveEffectiveModelFallbacksMock).not.toHaveBeenCalled();
expect(resolved.modelFallbacksOverride).toEqual(["openai/gpt-4o-mini"]);
});
it("preserves empty image model fallback overrides for embedded run params", () => {
const run = makeRun({
imageModelFallbacksOverride: [],
});
const authProfile = resolveProviderScopedAuthProfile({
provider: "openai",
primaryProvider: "openai",
});
const resolved = buildEmbeddedRunBaseParams({
run,
provider: "openai",
model: "gpt-4o",
runId: "run-1",
authProfile,
});
expect(hoisted.resolveEffectiveModelFallbacksMock).not.toHaveBeenCalled();
expect(resolved.modelFallbacksOverride).toEqual([]);
});
it("does not force final-tag enforcement for minimax providers", () => {
const run = makeRun();
@@ -145,6 +145,9 @@ async function resolveHelloWithModelDefaults(params: {
sessionEntry?: SessionEntry;
agentCfg?: { reasoningDefault?: "off" | "on" | "stream" };
commandAuthorized?: boolean;
hasOneTurnModelOverride?: boolean;
provider?: string;
model?: string;
ctx?: Parameters<typeof buildTestCtx>[0];
}) {
const resolveDefaultThinkingLevel = vi.fn(async () => params.defaultThinking);
@@ -190,8 +193,9 @@ async function resolveHelloWithModelDefaults(params: {
defaultProvider: "openai",
defaultModel: "gpt-4o-mini",
aliasIndex: { byAlias: new Map(), byKey: new Map() },
provider: "openai",
model: "gpt-4o-mini",
provider: params.provider ?? "openai",
model: params.model ?? "gpt-4o-mini",
hasOneTurnModelOverride: params.hasOneTurnModelOverride,
hasResolvedHeartbeatModelOverride: false,
typing: makeTypingController(),
opts: undefined,
@@ -314,6 +318,21 @@ describe("resolveReplyDirectives", () => {
mocks.resolveReplyExecOverrides.mockReturnValue(undefined);
});
it("passes one-turn model override state into model selection", async () => {
await resolveHelloWithModelDefaults({
defaultThinking: "off",
defaultReasoning: "on",
provider: "openai",
model: "gpt-4o-mini",
hasOneTurnModelOverride: true,
});
const modelSelectionInput = mockCallInput(mocks.createModelSelectionState);
expect(modelSelectionInput.provider).toBe("openai");
expect(modelSelectionInput.model).toBe("gpt-4o-mini");
expect(modelSelectionInput.hasOneTurnModelOverride).toBe(true);
});
it("prefers the target session entry from sessionStore for directive state", async () => {
const wrapperSessionEntry = makeSessionEntry({
sessionId: "wrapper-session",
@@ -171,6 +171,7 @@ export async function resolveReplyDirectives(params: {
aliasIndex: ModelAliasIndex;
provider: string;
model: string;
hasOneTurnModelOverride?: boolean;
hasResolvedHeartbeatModelOverride: boolean;
typing: TypingController;
opts?: GetReplyOptions;
@@ -200,6 +201,7 @@ export async function resolveReplyDirectives(params: {
primaryModel,
provider: initialProvider,
model: initialModel,
hasOneTurnModelOverride,
hasResolvedHeartbeatModelOverride,
typing,
opts,
@@ -535,6 +537,7 @@ export async function resolveReplyDirectives(params: {
provider,
model,
hasModelDirective: directives.hasModelDirective,
hasOneTurnModelOverride,
hasResolvedHeartbeatModelOverride,
isHeartbeat: opts?.isHeartbeat === true,
});
@@ -1140,6 +1140,154 @@ describe("runPreparedReply media-only handling", () => {
expect(call?.followupRun.run.authProfileId).toBe("profile-after-wait");
expect(vi.mocked(resolveSessionAuthProfileOverride)).toHaveBeenCalledTimes(1);
});
it("resolves image override auth profile without mutating stored session profile", async () => {
const { resolveSessionAuthProfileOverride } =
await import("../../agents/auth-profiles/session-override.js");
const sessionEntry: SessionEntry = {
sessionId: "session-image-auth",
sessionFile: "/tmp/session-image-auth.jsonl",
authProfileOverride: "anthropic:work",
authProfileOverrideSource: "user",
updatedAt: 1,
};
const sessionStore: Record<string, SessionEntry> = {
"session-key": sessionEntry,
};
vi.mocked(resolveSessionAuthProfileOverride).mockImplementationOnce(async (params) => {
expect(params.provider).toBe("openai");
expect(params.storePath).toBeUndefined();
expect(params.sessionEntry).not.toBe(sessionEntry);
expect(params.sessionStore).not.toBe(sessionStore);
if (params.sessionEntry) {
params.sessionEntry.authProfileOverride = "openai:vision";
params.sessionEntry.authProfileOverrideSource = "auto";
}
return "openai:vision";
});
await runPreparedReply(
baseParams({
provider: "openai",
model: "gpt-4o",
defaultProvider: "anthropic",
defaultModel: "claude-opus-4-1",
hasAppliedImageModelOverride: true,
isNewSession: false,
sessionId: "session-image-auth",
sessionEntry,
sessionStore,
storePath: "/tmp/sessions.json",
}),
);
const call = requireLastRunReplyAgentCall();
expect(call?.followupRun.run.authProfileId).toBe("openai:vision");
expect(call?.followupRun.run.authProfileIdSource).toBe("auto");
expect(sessionEntry.authProfileOverride).toBe("anthropic:work");
expect(sessionEntry.authProfileOverrideSource).toBe("user");
expect(sessionStore["session-key"]?.authProfileOverride).toBe("anthropic:work");
});
it("isolates image override auth profile when the override provider matches the default provider", async () => {
const { resolveSessionAuthProfileOverride } =
await import("../../agents/auth-profiles/session-override.js");
const sessionEntry: SessionEntry = {
sessionId: "session-image-default-provider-auth",
sessionFile: "/tmp/session-image-default-provider-auth.jsonl",
providerOverride: "anthropic",
modelOverride: "claude-opus-4-1",
authProfileOverride: "anthropic:work",
authProfileOverrideSource: "user",
updatedAt: 1,
};
const sessionStore: Record<string, SessionEntry> = {
"session-key": sessionEntry,
};
vi.mocked(resolveSessionAuthProfileOverride).mockImplementationOnce(async (params) => {
expect(params.provider).toBe("openai");
expect(params.storePath).toBeUndefined();
expect(params.sessionEntry).not.toBe(sessionEntry);
expect(params.sessionStore).not.toBe(sessionStore);
if (params.sessionEntry) {
params.sessionEntry.authProfileOverride = "openai:vision";
params.sessionEntry.authProfileOverrideSource = "auto";
}
return "openai:vision";
});
await runPreparedReply(
baseParams({
provider: "openai",
model: "gpt-4o",
defaultProvider: "openai",
defaultModel: "gpt-4o-mini",
hasAppliedImageModelOverride: true,
isNewSession: false,
sessionId: "session-image-default-provider-auth",
sessionEntry,
sessionStore,
storePath: "/tmp/sessions.json",
}),
);
const call = requireLastRunReplyAgentCall();
expect(call?.followupRun.run.authProfileId).toBe("openai:vision");
expect(call?.followupRun.run.authProfileIdSource).toBe("auto");
expect(sessionEntry.authProfileOverride).toBe("anthropic:work");
expect(sessionStore["session-key"]?.authProfileOverride).toBe("anthropic:work");
});
it("isolates image override auth profile from the pre-override runtime provider", async () => {
const { resolveSessionAuthProfileOverride } =
await import("../../agents/auth-profiles/session-override.js");
const sessionEntry: SessionEntry = {
sessionId: "session-image-runtime-provider-auth",
sessionFile: "/tmp/session-image-runtime-provider-auth.jsonl",
modelProvider: "anthropic",
model: "claude-opus-4-1",
authProfileOverride: "anthropic:work",
authProfileOverrideSource: "user",
updatedAt: 1,
};
const sessionStore: Record<string, SessionEntry> = {
"session-key": sessionEntry,
};
vi.mocked(resolveSessionAuthProfileOverride).mockImplementationOnce(async (params) => {
expect(params.provider).toBe("openai");
expect(params.storePath).toBeUndefined();
expect(params.sessionEntry).not.toBe(sessionEntry);
expect(params.sessionStore).not.toBe(sessionStore);
if (params.sessionEntry) {
params.sessionEntry.authProfileOverride = "openai:vision";
params.sessionEntry.authProfileOverrideSource = "auto";
}
return "openai:vision";
});
await runPreparedReply(
baseParams({
provider: "openai",
model: "gpt-4o",
defaultProvider: "openai",
defaultModel: "gpt-4o-mini",
hasAppliedImageModelOverride: true,
imageModelOverrideBaseProvider: "anthropic",
isNewSession: false,
sessionId: "session-image-runtime-provider-auth",
sessionEntry,
sessionStore,
storePath: "/tmp/sessions.json",
}),
);
const call = requireLastRunReplyAgentCall();
expect(call?.followupRun.run.authProfileId).toBe("openai:vision");
expect(call?.followupRun.run.authProfileIdSource).toBe("auto");
expect(sessionEntry.authProfileOverride).toBe("anthropic:work");
expect(sessionStore["session-key"]?.authProfileOverride).toBe("anthropic:work");
});
it("re-resolves same-session ownership after session-id rotation during async prep", async () => {
const { resolveSessionAuthProfileOverride } =
await import("../../agents/auth-profiles/session-override.js");
+78 -29
View File
@@ -4,6 +4,7 @@ import { resolveSessionAuthProfileOverride } from "../../agents/auth-profiles/se
import type { ExecToolDefaults } from "../../agents/bash-tools.js";
import { resolveFastModeState } from "../../agents/fast-mode.js";
import { resolveAgentHarnessPolicy } from "../../agents/harness/selection.js";
import { normalizeProviderId } from "../../agents/model-selection.js";
import { listOpenAIAuthProfileProvidersForAgentRuntime } from "../../agents/openai-codex-routing.js";
import { resolveEmbeddedFullAccessState } from "../../agents/pi-embedded-runner/sandbox-info.js";
import type { EmbeddedFullAccessBlockedReason } from "../../agents/pi-embedded-runner/types.js";
@@ -75,6 +76,7 @@ import { resolveBareSessionResetPromptState } from "./session-reset-prompt.js";
import { resolveBareResetBootstrapFileAccess } from "./session-reset-prompt.js";
import { drainFormattedSystemEventBlock } from "./session-system-events.js";
import { buildSessionStartupContextPrelude, shouldApplyStartupContext } from "./startup-context.js";
import { resolveStoredModelOverride } from "./stored-model-override.js";
import { resolveTypingMode } from "./typing-mode.js";
import { resolveRunTypingPolicy } from "./typing-policy.js";
import type { TypingController } from "./typing.js";
@@ -348,6 +350,9 @@ type RunPreparedReplyParams = {
storePath?: string;
workspaceDir: string;
abortedLastRun: boolean;
hasAppliedImageModelOverride?: boolean;
imageModelOverrideBaseProvider?: string;
imageModelFallbacksOverride?: string[];
};
export async function runPreparedReply(
@@ -378,6 +383,7 @@ export async function runPreparedReply(
perMessageQueueOptions,
typing,
opts,
defaultProvider,
defaultModel,
timeoutMs,
isNewSession,
@@ -388,6 +394,9 @@ export async function runPreparedReply(
storePath,
workspaceDir,
sessionStore,
hasAppliedImageModelOverride,
imageModelOverrideBaseProvider,
imageModelFallbacksOverride,
} = params;
const runtimePolicySessionKey = resolveRuntimePolicySessionKey({
cfg,
@@ -873,21 +882,72 @@ export async function runPreparedReply(
harnessRuntime: agentHarnessPolicy.runtime,
})
: [provider];
let authProfileId = useFastReplyRuntime
? preparedSessionState.sessionEntry?.authProfileOverride
: await traceRunPhase("reply.resolve_auth_profile", () =>
resolveSessionAuthProfileOverride({
cfg,
provider,
acceptedProviderIds: resolveAcceptedAuthProfileProviders(),
agentDir,
sessionEntry: preparedSessionState.sessionEntry,
sessionStore,
sessionKey,
storePath,
isNewSession,
}),
);
const resolveActiveSessionProviderForAuthProfile = (): string => {
const storedOverride = resolveStoredModelOverride({
sessionEntry: preparedSessionState.sessionEntry,
sessionStore,
sessionKey,
parentSessionKey:
preparedSessionState.sessionEntry?.parentSessionKey ??
sessionCtx.ModelParentSessionKey ??
sessionCtx.ParentSessionKey,
defaultProvider,
});
return storedOverride?.provider ?? defaultProvider;
};
const shouldResolveEphemeralAuthProfileForImageOverride = (): boolean => {
if (hasAppliedImageModelOverride !== true) {
return false;
}
const activeSessionProvider =
normalizeOptionalString(imageModelOverrideBaseProvider) ??
resolveActiveSessionProviderForAuthProfile();
return normalizeProviderId(provider) !== normalizeProviderId(activeSessionProvider);
};
const resolveRuntimeAuthProfile = async (): Promise<{
authProfileId?: string;
authProfileIdSource?: "auto" | "user";
}> => {
if (useFastReplyRuntime) {
return {
authProfileId: preparedSessionState.sessionEntry?.authProfileOverride,
authProfileIdSource: preparedSessionState.sessionEntry?.authProfileOverrideSource,
};
}
const shouldUseEphemeralSession = shouldResolveEphemeralAuthProfileForImageOverride();
const authSessionKey = shouldUseEphemeralSession ? (sessionKey ?? sessionIdFinal) : sessionKey;
const authSessionEntry =
shouldUseEphemeralSession && preparedSessionState.sessionEntry
? { ...preparedSessionState.sessionEntry }
: preparedSessionState.sessionEntry;
const authSessionStore =
shouldUseEphemeralSession && authSessionEntry
? { [authSessionKey]: authSessionEntry }
: sessionStore;
const resolvedAuthProfileId = await resolveSessionAuthProfileOverride({
cfg,
provider,
acceptedProviderIds: resolveAcceptedAuthProfileProviders(),
agentDir,
sessionEntry: authSessionEntry,
sessionStore: authSessionStore,
sessionKey: authSessionKey,
storePath: shouldUseEphemeralSession ? undefined : storePath,
isNewSession,
});
return {
authProfileId: resolvedAuthProfileId,
authProfileIdSource:
resolvedAuthProfileId && authSessionEntry?.authProfileOverride === resolvedAuthProfileId
? authSessionEntry.authProfileOverrideSource
: undefined,
};
};
let authProfileId: string | undefined;
let authProfileIdSource: "auto" | "user" | undefined;
({ authProfileId, authProfileIdSource } = await traceRunPhase("reply.resolve_auth_profile", () =>
resolveRuntimeAuthProfile(),
));
const { runReplyAgent } = await traceRunPhase("reply.load_agent_runner_runtime", () =>
loadAgentRunnerRuntime(),
);
@@ -936,19 +996,7 @@ export async function runPreparedReply(
piRuntime?.waitForEmbeddedPiRunEnd(activeRunSessionId) ?? Promise.resolve(undefined),
refreshPreparedState: async () => {
preparedSessionState = resolvePreparedSessionState();
authProfileId = useFastReplyRuntime
? preparedSessionState.sessionEntry?.authProfileOverride
: await resolveSessionAuthProfileOverride({
cfg,
provider,
acceptedProviderIds: resolveAcceptedAuthProfileProviders(),
agentDir,
sessionEntry: preparedSessionState.sessionEntry,
sessionStore,
sessionKey,
storePath,
isNewSession,
});
({ authProfileId, authProfileIdSource } = await resolveRuntimeAuthProfile());
preparedSessionState = resolvePreparedSessionState();
({ prefixedCommandBody, queuedBody, transcriptCommandBody, currentTurnContext } =
await traceRunPhase("reply.build_prompt_bodies", () => rebuildPromptBodies()));
@@ -961,7 +1009,6 @@ export async function runPreparedReply(
}
({ activeSessionId, isActive, isStreaming } = queueState.busyState);
}
const authProfileIdSource = preparedSessionState.sessionEntry?.authProfileOverrideSource;
const runHasSessionModelOverride = Boolean(
normalizeOptionalString(preparedSessionState.sessionEntry?.modelOverride) ||
normalizeOptionalString(preparedSessionState.sessionEntry?.providerOverride),
@@ -1024,9 +1071,11 @@ export async function runPreparedReply(
skillsSnapshot,
provider,
model,
hasOneTurnModelOverride: hasAppliedImageModelOverride || undefined,
hasSessionModelOverride: runHasSessionModelOverride,
modelOverrideSource: runModelOverrideSource,
hasAutoFallbackProvenance: runHasAutoFallbackProvenance || undefined,
imageModelFallbacksOverride,
authProfileId,
authProfileIdSource,
thinkLevel: resolvedThinkLevel,
@@ -61,6 +61,7 @@ registerGetReplyRuntimeOverrides(mocks);
let getReplyFromConfig: typeof import("./get-reply.js").getReplyFromConfig;
let resolveDefaultModelMock: typeof import("./directive-handling.defaults.js").resolveDefaultModel;
let resolveModelRefFromStringMock: typeof import("../../agents/model-selection.js").resolveModelRefFromString;
let loadConfigMock: typeof import("../../config/config.js").getRuntimeConfig;
let runPreparedReplyMock: typeof import("./get-reply-run.js").runPreparedReply;
@@ -68,6 +69,8 @@ async function loadGetReplyRuntimeForTest() {
({ getReplyFromConfig } = await loadGetReplyModuleForTest({ cacheKey: import.meta.url }));
({ resolveDefaultModel: resolveDefaultModelMock } =
await import("./directive-handling.defaults.js"));
({ resolveModelRefFromString: resolveModelRefFromStringMock } =
await import("../../agents/model-selection.js"));
({ getRuntimeConfig: loadConfigMock } = await import("../../config/config.js"));
({ runPreparedReply: runPreparedReplyMock } = await import("./get-reply-run.js"));
}
@@ -82,7 +85,13 @@ function requirePreparedReplyParams() {
function requireDirectiveParams() {
const directiveParams = mocks.resolveReplyDirectives.mock.calls[0]?.[0] as
| { sessionKey?: string; workspaceDir?: string }
| {
sessionKey?: string;
workspaceDir?: string;
provider?: string;
model?: string;
hasOneTurnModelOverride?: boolean;
}
| undefined;
if (!directiveParams) {
throw new Error("expected directive params");
@@ -115,6 +124,8 @@ describe("getReplyFromConfig fast test bootstrap", () => {
defaultModel: "gpt-4o-mini",
aliasIndex: emptyAliasIndex(),
});
vi.mocked(resolveModelRefFromStringMock).mockReset();
vi.mocked(resolveModelRefFromStringMock).mockReturnValue(null);
vi.mocked(loadConfigMock).mockReset();
vi.mocked(runPreparedReplyMock).mockReset();
vi.mocked(loadConfigMock).mockReturnValue({});
@@ -193,6 +204,40 @@ describe("getReplyFromConfig fast test bootstrap", () => {
expect(vi.mocked(runPreparedReplyMock)).toHaveBeenCalledOnce();
});
it("passes image model overrides as one-turn selections to prepared replies", async () => {
const home = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-image-model-directives-"));
const cfg = markCompleteReplyConfig({
agents: {
defaults: {
model: "openai/gpt-4o",
models: {
"openai/gpt-4o": {},
},
workspace: home,
},
},
session: { store: path.join(home, "sessions.json") },
} as OpenClawConfig);
vi.mocked(resolveDefaultModelMock).mockReturnValueOnce({
defaultProvider: "openai",
defaultModel: "gpt-4o",
aliasIndex: emptyAliasIndex(),
});
vi.mocked(resolveModelRefFromStringMock).mockReturnValueOnce({
ref: { provider: "openai", model: "gpt-4o-mini" },
});
await expect(
getReplyFromConfig(buildGetReplyCtx(), { modelOverride: "openai/gpt-4o-mini" }, cfg),
).resolves.toEqual({ text: "ok" });
expect(mocks.resolveReplyDirectives).not.toHaveBeenCalled();
const preparedReplyParams = requirePreparedReplyParams();
expect(preparedReplyParams.provider).toBe("openai");
expect(preparedReplyParams.model).toBe("gpt-4o-mini");
expect(preparedReplyParams.hasAppliedImageModelOverride).toBe(true);
});
it("clears stale ack-only heartbeat pending delivery before replay", async () => {
const home = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-heartbeat-pending-clear-"));
const storePath = path.join(home, "sessions.json");
+52 -2
View File
@@ -6,7 +6,7 @@ import {
resolveSessionAgentId,
resolveAgentSkillsFilter,
} from "../../agents/agent-scope.js";
import { resolveModelRefFromString } from "../../agents/model-selection.js";
import { modelKey, resolveModelRefFromString } from "../../agents/model-selection.js";
import { resolveAgentTimeoutMs } from "../../agents/timeout.js";
import { DEFAULT_AGENT_WORKSPACE_DIR, ensureAgentWorkspace } from "../../agents/workspace.js";
import { resolveChannelModelOverride } from "../../channels/model-overrides.js";
@@ -253,7 +253,28 @@ export async function getReplyFromConfig(
let provider = defaultProvider;
let model = defaultModel;
let hasResolvedHeartbeatModelOverride = false;
if (opts?.isHeartbeat) {
let hasAppliedImageModelOverride = false;
let imageModelFallbacksOverride: string[] | undefined;
const modelOverrideRaw = normalizeOptionalString(opts?.modelOverride);
if (modelOverrideRaw) {
const modelOverrideRef = resolveModelRefFromString({
raw: modelOverrideRaw,
defaultProvider,
aliasIndex,
});
if (modelOverrideRef) {
provider = modelOverrideRef.ref.provider;
model = modelOverrideRef.ref.model;
hasAppliedImageModelOverride = true;
imageModelFallbacksOverride = opts?.modelOverrideFallbacks?.filter(
(fallback): fallback is string => normalizeOptionalString(fallback) !== undefined,
);
} else {
defaultRuntime.log?.(
`[image-model-switch] Failed to resolve image model override ${modelOverrideRaw}; using default model ${modelKey(defaultProvider, defaultModel)}`,
);
}
} else if (opts?.isHeartbeat) {
// Prefer the resolved per-agent heartbeat model passed from the heartbeat runner,
// fall back to the global defaults heartbeat model for backward compatibility.
const heartbeatRaw =
@@ -530,6 +551,7 @@ export async function getReplyFromConfig(
if (
storedModelOverride?.model &&
!hasResolvedHeartbeatModelOverride &&
!hasAppliedImageModelOverride &&
!staleHeartbeatAutoFallbackOverride
) {
provider = storedModelOverride.provider ?? defaultProvider;
@@ -540,11 +562,32 @@ export async function getReplyFromConfig(
if (
!hasResolvedHeartbeatModelOverride &&
!hasEffectiveSessionModelOverride &&
!hasAppliedImageModelOverride &&
resolvedChannelModelOverride
) {
provider = resolvedChannelModelOverride.ref.provider;
model = resolvedChannelModelOverride.ref.model;
}
const imageModelOverrideBaseProvider = hasAppliedImageModelOverride
? (() => {
if (
storedModelOverride?.model &&
!hasResolvedHeartbeatModelOverride &&
!staleHeartbeatAutoFallbackOverride
) {
return storedModelOverride.provider ?? defaultProvider;
}
if (!hasEffectiveSessionModelOverride && resolvedChannelModelOverride) {
return resolvedChannelModelOverride.ref.provider;
}
const runtimeProvider = normalizeOptionalString(sessionEntry.modelProvider);
const runtimeModel = normalizeOptionalString(sessionEntry.model);
if (runtimeProvider && runtimeModel) {
return runtimeProvider;
}
return defaultProvider;
})()
: undefined;
if (
shouldUseReplyFastDirectiveExecution({
@@ -619,6 +662,9 @@ export async function getReplyFromConfig(
storePath,
workspaceDir,
abortedLastRun,
hasAppliedImageModelOverride,
imageModelOverrideBaseProvider,
imageModelFallbacksOverride,
}),
);
}
@@ -649,6 +695,7 @@ export async function getReplyFromConfig(
aliasIndex,
provider,
model,
hasOneTurnModelOverride: hasAppliedImageModelOverride,
hasResolvedHeartbeatModelOverride,
typing,
opts: resolvedOpts,
@@ -859,6 +906,9 @@ export async function getReplyFromConfig(
storePath,
workspaceDir,
abortedLastRun,
hasAppliedImageModelOverride,
imageModelOverrideBaseProvider,
imageModelFallbacksOverride,
}),
);
}
@@ -0,0 +1,135 @@
import { describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import { resolveImageModelOverridePlan } from "./image-model-override-plan.js";
function buildConfig(params: {
imageModel?: NonNullable<NonNullable<OpenClawConfig["agents"]>["defaults"]>["imageModel"];
models?: Record<string, object>;
}): OpenClawConfig {
return {
agents: {
defaults: {
...(params.imageModel ? { imageModel: params.imageModel } : {}),
...(params.models ? { models: params.models } : {}),
},
},
} as OpenClawConfig;
}
describe("resolveImageModelOverridePlan", () => {
it("uses the session model when it already supports images", async () => {
const modelSupportsImages = vi.fn(async () => true);
const plan = await resolveImageModelOverridePlan({
cfg: buildConfig({ imageModel: "openai/gpt-4o" }),
defaultProvider: "anthropic",
defaultModel: "claude-opus-4-6",
hasImageAttachments: true,
sessionModelSupportsImages: true,
modelSupportsImages,
});
expect(plan).toEqual({ kind: "inline-session" });
expect(modelSupportsImages).not.toHaveBeenCalled();
});
it("keeps configured image models reachable when a model allowlist is present", async () => {
const plan = await resolveImageModelOverridePlan({
cfg: buildConfig({
imageModel: {
primary: "openai/gpt-4o",
fallbacks: ["openai/gpt-4o-mini"],
},
models: { "anthropic/claude-opus-4-6": {} },
}),
defaultProvider: "anthropic",
defaultModel: "claude-opus-4-6",
hasImageAttachments: true,
sessionModelSupportsImages: false,
modelSupportsImages: async () => true,
});
expect(plan).toEqual({
kind: "inline-image-model",
modelOverride: "openai/gpt-4o",
modelOverrideFallbacks: ["openai/gpt-4o-mini"],
});
});
it("resolves providerless image models independently of the active session provider", async () => {
const modelSupportsImages = vi.fn(async (ref: { provider: string; model: string }) => {
return ref.provider === "openai" && ref.model === "gpt-4o";
});
const plan = await resolveImageModelOverridePlan({
cfg: buildConfig({ imageModel: "gpt-4o" }),
defaultProvider: "openai",
defaultModel: "gpt-5.5",
hasImageAttachments: true,
sessionModelSupportsImages: false,
modelSupportsImages,
});
expect(plan).toEqual({
kind: "inline-image-model",
modelOverride: "openai/gpt-4o",
modelOverrideFallbacks: [],
});
expect(modelSupportsImages).toHaveBeenCalledWith({
provider: "openai",
model: "gpt-4o",
});
});
it("uses the configured default provider for unmatched providerless image models", async () => {
const modelSupportsImages = vi.fn(async (ref: { provider: string; model: string }) => {
return ref.provider === "ollama" && ref.model === "qwen2.5vl:7b";
});
const plan = await resolveImageModelOverridePlan({
cfg: buildConfig({ imageModel: "qwen2.5vl:7b" }),
defaultProvider: "ollama",
defaultModel: "llama3.2",
hasImageAttachments: true,
sessionModelSupportsImages: false,
modelSupportsImages,
});
expect(plan).toEqual({
kind: "inline-image-model",
modelOverride: "ollama/qwen2.5vl:7b",
modelOverrideFallbacks: [],
});
});
it("selects the first vision-capable image model and carries later image fallbacks", async () => {
const modelSupportsImages = vi.fn(async (ref: { provider: string; model: string }) => {
return ref.model !== "gpt-4o-blocked";
});
const plan = await resolveImageModelOverridePlan({
cfg: buildConfig({
imageModel: {
primary: "openai/gpt-4o-blocked",
fallbacks: ["openai/gpt-4o", "google/gemini-2.5-flash"],
},
models: {
"openai/gpt-4o-blocked": {},
"openai/gpt-4o": {},
"google/gemini-2.5-flash": {},
},
}),
defaultProvider: "anthropic",
defaultModel: "claude-opus-4-6",
hasImageAttachments: true,
sessionModelSupportsImages: false,
modelSupportsImages,
});
expect(plan).toEqual({
kind: "inline-image-model",
modelOverride: "openai/gpt-4o",
modelOverrideFallbacks: ["google/gemini-2.5-flash"],
});
});
});
@@ -0,0 +1,122 @@
import {
buildModelAliasIndex,
inferUniqueProviderFromConfiguredModels,
modelKey,
resolveModelRefFromString,
type ModelRef,
} from "../../agents/model-selection.js";
import {
resolveAgentModelFallbackValues,
resolveAgentModelPrimaryValue,
} from "../../config/model-input.js";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
export type ImageModelOverridePlan =
| {
kind: "inline-session";
}
| {
kind: "inline-image-model";
modelOverride: string;
modelOverrideFallbacks: string[];
}
| {
kind: "media-paths";
reason: "no-image-attachments" | "no-image-model" | "not-vision-capable";
};
export type ImageModelCapabilityResolver = (ref: ModelRef) => Promise<boolean>;
type ImageModelCandidate = {
raw: string;
ref: ModelRef;
key: string;
};
function resolveImageModelCandidate(params: {
raw: string;
cfg: OpenClawConfig;
defaultProvider: string;
}): ImageModelCandidate | null {
const trimmed = params.raw.trim();
if (!trimmed) {
return null;
}
const imageDefaultProvider = trimmed.includes("/")
? params.defaultProvider
: (inferUniqueProviderFromConfiguredModels({ cfg: params.cfg, model: trimmed }) ??
params.defaultProvider);
const aliasIndex = buildModelAliasIndex({
cfg: params.cfg,
defaultProvider: imageDefaultProvider,
});
const resolved = resolveModelRefFromString({
cfg: params.cfg,
raw: trimmed,
defaultProvider: imageDefaultProvider,
aliasIndex,
});
if (!resolved) {
return null;
}
const ref = resolved.ref;
return {
raw: trimmed,
ref,
key: modelKey(ref.provider, ref.model),
};
}
export async function resolveImageModelOverridePlan(params: {
cfg: OpenClawConfig;
agentId?: string;
defaultProvider: string;
defaultModel: string;
hasImageAttachments: boolean;
sessionModelSupportsImages: boolean;
modelSupportsImages: ImageModelCapabilityResolver;
}): Promise<ImageModelOverridePlan> {
if (!params.hasImageAttachments) {
return { kind: "media-paths", reason: "no-image-attachments" };
}
if (params.sessionModelSupportsImages) {
return { kind: "inline-session" };
}
const imageModelConfig = params.cfg.agents?.defaults?.imageModel;
const primary = resolveAgentModelPrimaryValue(imageModelConfig);
const rawCandidates = [
...(primary ? [primary] : []),
...resolveAgentModelFallbackValues(imageModelConfig),
];
if (rawCandidates.length === 0) {
return { kind: "media-paths", reason: "no-image-model" };
}
const runnableCandidates: ImageModelCandidate[] = [];
for (const raw of rawCandidates) {
const candidate = resolveImageModelCandidate({
raw,
cfg: params.cfg,
defaultProvider: params.defaultProvider,
});
if (!candidate) {
continue;
}
if (!(await params.modelSupportsImages(candidate.ref))) {
continue;
}
runnableCandidates.push(candidate);
}
const selected = runnableCandidates[0];
if (!selected) {
return { kind: "media-paths", reason: "not-vision-capable" };
}
return {
kind: "inline-image-model",
modelOverride: selected.key,
modelOverrideFallbacks: runnableCandidates.slice(1).map((candidate) => candidate.key),
};
}
@@ -765,6 +765,45 @@ describe("createModelSelectionState respects session model override", () => {
expect(sessionStore[sessionKey]?.providerOverride).toBeUndefined();
});
it("keeps one-turn model overrides ahead of stored overrides and allowlist fallback", async () => {
const cfg = {
agents: {
defaults: {
model: { primary: "openai/gpt-4o" },
models: {
"openai/gpt-4o": {},
},
},
},
} as OpenClawConfig;
const sessionKey = "agent:main:telegram:direct:1";
const sessionEntry = makeEntry({
providerOverride: "anthropic",
modelOverride: "claude-opus-4-6",
});
const sessionStore = { [sessionKey]: sessionEntry };
const state = await createModelSelectionState({
cfg,
agentCfg: cfg.agents?.defaults,
sessionEntry,
sessionStore,
sessionKey,
defaultProvider: "openai",
defaultModel: "gpt-4o",
provider: "openai",
model: "gpt-4o-mini",
hasModelDirective: false,
hasOneTurnModelOverride: true,
});
expect(state.provider).toBe("openai");
expect(state.model).toBe("gpt-4o-mini");
expect(state.resetModelOverride).toBe(false);
expect(sessionStore[sessionKey]?.providerOverride).toBe("anthropic");
expect(sessionStore[sessionKey]?.modelOverride).toBe("claude-opus-4-6");
});
it("keeps wildcard-provider overrides when configured catalog rows are unavailable", async () => {
const cfg = {
agents: {
+11 -2
View File
@@ -103,6 +103,7 @@ export async function createModelSelectionState(params: {
provider: string;
model: string;
hasModelDirective: boolean;
hasOneTurnModelOverride?: boolean;
/** True when heartbeat.model was explicitly resolved for this run.
* In that case, skip session-stored overrides so the heartbeat selection wins. */
hasResolvedHeartbeatModelOverride?: boolean;
@@ -135,6 +136,7 @@ export async function createModelSelectionState(params: {
let model = params.model;
const primaryProvider = params.primaryProvider ?? defaultProvider;
const primaryModel = params.primaryModel ?? defaultModel;
const hasOneTurnModelOverride = params.hasOneTurnModelOverride === true;
const hasAllowlist = agentCfg?.models && Object.keys(agentCfg.models).length > 0;
const visibility = parseConfiguredModelVisibilityEntries({ cfg });
@@ -214,7 +216,13 @@ export async function createModelSelectionState(params: {
logStage("configured-catalog-ready", `entries=${configuredModelCatalog.length}`);
}
if (sessionEntry && sessionStore && sessionKey && directStoredOverride) {
if (
sessionEntry &&
sessionStore &&
sessionKey &&
directStoredOverride &&
!hasOneTurnModelOverride
) {
const normalizedOverride = normalizeModelRef(
directStoredOverride.provider,
directStoredOverride.model,
@@ -272,6 +280,7 @@ export async function createModelSelectionState(params: {
// overrides unless a direct auto fallback override is stale for the current
// configured default.
const skipStoredOverride =
hasOneTurnModelOverride ||
params.hasResolvedHeartbeatModelOverride === true ||
(staleHeartbeatAutoFallbackOverride && storedOverride?.source === "session");
@@ -287,7 +296,7 @@ export async function createModelSelectionState(params: {
}
}
if (!params.hasModelDirective) {
if (!params.hasModelDirective && !hasOneTurnModelOverride) {
const allowedInitialSelection = visibilityPolicy.resolveSelection({
provider,
model,
+2
View File
@@ -85,9 +85,11 @@ export type FollowupRun = {
skillsSnapshot?: SkillSnapshot;
provider: string;
model: string;
hasOneTurnModelOverride?: boolean;
hasSessionModelOverride?: boolean;
modelOverrideSource?: "auto" | "user";
hasAutoFallbackProvenance?: boolean;
imageModelFallbacksOverride?: string[];
authProfileId?: string;
authProfileIdSource?: "auto" | "user";
thinkLevel?: ThinkLevel;
+17
View File
@@ -428,6 +428,23 @@ export async function parseMessageWithAttachments(
};
}
export async function resolveChatAttachmentLooksLikeImage(
attachment: ChatAttachment,
index = 0,
): Promise<boolean> {
const normalized = normalizeAttachment(attachment, index, {
stripDataUrlPrefix: true,
requireImageMime: false,
});
if (!isValidBase64(normalized.base64)) {
throw new Error(`attachment ${normalized.label}: invalid base64 content`);
}
const providedMime = normalizeMime(normalized.mime);
const sniffedMime = normalizeMime(await sniffMimeFromBase64(normalized.base64));
const labelMime = normalizeMime(mimeTypeFromFilePath(normalized.label));
return isImageMime(resolveAttachmentMime({ sniffedMime, providedMime, labelMime }));
}
/**
* @deprecated Use parseMessageWithAttachments instead.
* This function converts images to markdown data URLs which Claude API cannot process as images.
+44 -3
View File
@@ -5,12 +5,14 @@ import type { AgentMessage } from "@earendil-works/pi-agent-core";
import { CURRENT_SESSION_VERSION } from "@earendil-works/pi-coding-agent";
import { resolveSendableOutboundReplyParts } from "openclaw/plugin-sdk/reply-payload";
import { resolveAgentWorkspaceDir, resolveSessionAgentId } from "../../agents/agent-scope.js";
import { resolveDefaultModelForAgent } from "../../agents/model-selection.js";
import { rewriteTranscriptEntriesInSessionFile } from "../../agents/pi-embedded-runner/transcript-rewrite.js";
import { resolveProviderIdForAuth } from "../../agents/provider-auth-aliases.js";
import { ensureSandboxWorkspaceForSession } from "../../agents/sandbox/context.js";
import { resolveAgentTimeoutMs } from "../../agents/timeout.js";
import { dispatchInboundMessage } from "../../auto-reply/dispatch.js";
import type { ReplyPayload } from "../../auto-reply/reply-payload.js";
import { resolveImageModelOverridePlan } from "../../auto-reply/reply/image-model-override-plan.js";
import { createReplyDispatcher } from "../../auto-reply/reply/reply-dispatcher.js";
import { stageSandboxMedia } from "../../auto-reply/reply/stage-sandbox-media.js";
import type { MsgContext, TemplateContext } from "../../auto-reply/templating.js";
@@ -66,10 +68,12 @@ import {
updateChatRunProvider,
} from "../chat-abort.js";
import {
type ChatAttachment,
type ChatImageContent,
MediaOffloadError,
type OffloadedRef,
parseMessageWithAttachments,
resolveChatAttachmentLooksLikeImage,
resolveChatAttachmentMaxBytes,
UnsupportedAttachmentError,
} from "../chat-attachments.js";
@@ -179,6 +183,15 @@ function stripVisibleTextFromTtsSupplement(payload: ReplyPayload): ReplyPayload
return isTtsSupplementPayload(payload) ? { ...payload, text: undefined } : payload;
}
async function hasImageChatAttachments(attachments: ChatAttachment[]): Promise<boolean> {
for (const [index, attachment] of attachments.entries()) {
if (await resolveChatAttachmentLooksLikeImage(attachment, index)) {
return true;
}
}
return false;
}
async function buildWebchatAssistantMediaMessage(
payloads: ReplyPayload[],
options?: {
@@ -2030,6 +2043,7 @@ export const chatHandlers: GatewayRequestHandlers = {
sessionKey,
config: cfg,
});
const resolvedConfiguredDefaultModel = resolveDefaultModelForAgent({ cfg, agentId });
const resolvedSessionModel = resolveSessionModelRef(cfg, entry, agentId);
const resolvedSessionAuthProvider = resolveProviderIdForAuth(resolvedSessionModel.provider, {
config: cfg,
@@ -2132,22 +2146,47 @@ export const chatHandlers: GatewayRequestHandlers = {
const explicitOriginTargetsPlugin = explicitOriginTargetsPluginBinding(
explicitOriginResult.value,
);
let modelOverride: string | undefined;
let modelOverrideFallbacks: string[] | undefined;
if (normalizedAttachments.length > 0) {
try {
await measureDiagnosticsTimelineSpan(
"gateway.chat_send.prepare_attachments",
async () => {
const hasImageAttachments = await hasImageChatAttachments(normalizedAttachments);
const supportsSessionModelImages = await resolveGatewayModelSupportsImages({
loadGatewayModelCatalog: context.loadGatewayModelCatalog,
provider: resolvedSessionModel.provider,
model: resolvedSessionModel.model,
});
const explicitOriginSupportsInlineImages =
explicitOriginTargetsAcpSession(explicitOriginResult.value) ||
explicitOriginTargetsPlugin;
const imageModelPlan = await resolveImageModelOverridePlan({
cfg,
agentId,
defaultProvider: resolvedConfiguredDefaultModel.provider,
defaultModel: resolvedConfiguredDefaultModel.model,
hasImageAttachments,
sessionModelSupportsImages:
supportsSessionModelImages || explicitOriginSupportsInlineImages,
modelSupportsImages: (ref) =>
resolveGatewayModelSupportsImages({
loadGatewayModelCatalog: context.loadGatewayModelCatalog,
provider: ref.provider,
model: ref.model,
}),
});
if (imageModelPlan.kind === "inline-image-model") {
modelOverride = imageModelPlan.modelOverride;
modelOverrideFallbacks = imageModelPlan.modelOverrideFallbacks;
}
// Bound plugin sessions own the real recipient model, so keep image
// attachments even when the parent OpenClaw session model is text-only.
const supportsImages =
supportsSessionModelImages ||
explicitOriginTargetsAcpSession(explicitOriginResult.value) ||
explicitOriginTargetsPlugin;
imageModelPlan.kind === "inline-session" ||
imageModelPlan.kind === "inline-image-model" ||
explicitOriginSupportsInlineImages;
const routeImageOffloadsAsMediaPaths = !supportsImages;
const parsed = await parseMessageWithAttachments(
inboundMessage,
@@ -2546,6 +2585,8 @@ export const chatHandlers: GatewayRequestHandlers = {
abortSignal: activeRunAbort.controller.signal,
images: parsedImages.length > 0 ? parsedImages : undefined,
imageOrder: imageOrder.length > 0 ? imageOrder : undefined,
modelOverride,
modelOverrideFallbacks,
thinkingLevelOverride: p.thinking,
fastModeOverride: p.fastMode,
onAgentRunStart: (runId) => {
@@ -4,6 +4,7 @@ import path from "node:path";
import { afterAll, beforeAll, describe, expect, test, vi } from "vitest";
import type { GetReplyOptions } from "../auto-reply/get-reply-options.types.js";
import { clearConfigCache } from "../config/config.js";
import type { AgentModelConfig } from "../config/types.agents-shared.js";
import { GATEWAY_CLIENT_MODES, GATEWAY_CLIENT_NAMES } from "../utils/message-channel.js";
import { __setMaxChatHistoryMessagesBytesForTest } from "./server-constants.js";
import type { GatewayRequestContext, RespondFn } from "./server-methods/shared-types.js";
@@ -128,6 +129,25 @@ async function fetchHistoryMessages(
return historyRes.payload?.messages ?? [];
}
type ConfiguredImageModelCase = {
id: string;
imageModel: AgentModelConfig;
expectedFallbacks: string[];
};
const configuredImageModelCases: ConfiguredImageModelCase[] = [
{
id: "with-image-fallback",
imageModel: { primary: "openai/gpt-4o", fallbacks: ["openai/gpt-4o-mini"] },
expectedFallbacks: ["openai/gpt-4o-mini"],
},
{
id: "without-image-fallback",
imageModel: { primary: "openai/gpt-4o" },
expectedFallbacks: [],
},
];
async function prepareMainHistoryHarness(params: {
ws: GatewaySocket;
createSessionDir: () => Promise<string>;
@@ -344,6 +364,158 @@ describe("gateway server chat", () => {
}
});
test.each(configuredImageModelCases)(
"chat.send inlines image attachments through configured imageModel with allowlist present: $id",
async ({ id, imageModel, expectedFallbacks }) => {
const sessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-gw-"));
try {
testState.sessionStorePath = path.join(sessionDir, "sessions.json");
testState.agentConfig = {
model: {
primary: "anthropic/claude-opus-4-6",
fallbacks: ["anthropic/claude-haiku-4-6"],
},
imageModel,
models: {
"anthropic/claude-opus-4-6": {},
},
};
await writeSessionStore({
entries: {
main: {
sessionId: "sess-main",
modelProvider: "anthropic",
model: "claude-opus-4-6",
updatedAt: Date.now(),
},
},
});
const context = {
loadGatewayModelCatalog: vi.fn<GatewayRequestContext["loadGatewayModelCatalog"]>(
async () => [
{
id: "claude-opus-4-6",
name: "Claude Opus 4.6",
provider: "anthropic",
input: ["text"],
},
{
id: "gpt-4o",
name: "GPT-4o",
provider: "openai",
input: ["text", "image"],
},
{
id: "gpt-4o-mini",
name: "GPT-4o mini",
provider: "openai",
input: ["text", "image"],
},
{
id: "claude-haiku-4-6",
name: "Claude Haiku 4.6",
provider: "anthropic",
input: ["text"],
},
],
),
logGateway: {
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
debug: vi.fn(),
},
agentRunSeq: new Map<string, number>(),
chatAbortControllers: new Map(),
chatAbortedRuns: new Map(),
chatRunBuffers: new Map(),
chatDeltaSentAt: new Map(),
chatDeltaLastBroadcastLen: new Map(),
chatDeltaLastBroadcastText: new Map(),
addChatRun: vi.fn(),
removeChatRun: vi.fn(),
broadcast: vi.fn(),
nodeSendToSession: vi.fn(),
registerToolEventRecipient: vi.fn(),
dedupe: new Map(),
} as unknown as GatewayRequestContext;
const pngB64 =
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/woAAn8B9FD5fHAAAAAASUVORK5CYII=";
let captured: { ctx?: Record<string, unknown>; replyOptions?: GetReplyOptions } | undefined;
dispatchInboundMessageMock.mockImplementationOnce(async (...args: unknown[]) => {
const [params] = args as [
{
ctx: Record<string, unknown>;
replyOptions?: GetReplyOptions;
},
];
captured = {
ctx: params.ctx,
replyOptions: params.replyOptions,
};
});
const { chatHandlers } = await import("./server-methods/chat.js");
const responses: Array<{ ok: boolean; payload?: unknown; error?: unknown }> = [];
await chatHandlers["chat.send"]({
req: {
type: "req",
id: `configured-image-model-${id}`,
method: "chat.send",
params: {
sessionKey: "main",
message: "see image",
idempotencyKey: `idem-configured-image-model-${id}`,
attachments: [
{
type: "image",
mimeType: "image/png",
fileName: "dot.png",
content: pngB64,
},
],
},
},
params: {
sessionKey: "main",
message: "see image",
idempotencyKey: `idem-configured-image-model-${id}`,
attachments: [
{
type: "image",
mimeType: "image/png",
fileName: "dot.png",
content: pngB64,
},
],
},
client: null,
isWebchatConnect: () => false,
respond: ((ok, payload, error) => {
responses.push({ ok, payload, error });
}) as RespondFn,
context,
});
expect(responses[0]?.ok).toBe(true);
await vi.waitFor(() => expect(captured).toBeDefined(), FAST_WAIT_OPTS);
expect(captured?.replyOptions?.images).toEqual([
expect.objectContaining({ type: "image", mimeType: "image/png" }),
]);
expect(captured?.replyOptions?.modelOverride).toBe("openai/gpt-4o");
expect(captured?.replyOptions?.modelOverrideFallbacks).toEqual(expectedFallbacks);
expect(captured?.ctx?.MediaPaths).toBeUndefined();
} finally {
dispatchInboundMessageMock.mockReset();
testState.agentConfig = undefined;
testState.sessionStorePath = undefined;
clearConfigCache();
await fs.rm(sessionDir, { recursive: true, force: true });
}
},
);
test("chat.send reuses an active internal run for duplicate WebChat text sends", async () => {
const sessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-gw-"));
const dispatchRelease = createDeferred<void>();