fix: session model selection replaced by response aliases and fallback wins (#122910)

* fix(agents): keep session model selection canonical

Preserve configured session model identity across response aliases and turn-local fallbacks.

* fix(gateway): align session model projection consumers

* test(gateway): mark session event model selection
This commit is contained in:
Peter Steinberger
2026-08-12 19:46:56 -07:00
committed by GitHub
parent ddb0fad7df
commit cad7e7f9e1
16 changed files with 143 additions and 80 deletions
@@ -2445,6 +2445,9 @@ describe("agentCommand LiveSessionModelSwitchError retry", () => {
entry: expect.objectContaining({ thinkingLevel: "max" }),
}),
);
expectRecordFields(mockCallArg(state.updateSessionStoreAfterAgentRunMock), {
preserveRuntimeModel: true,
});
});
it("recomputes a model-derived thinking default for each fallback candidate", async () => {
+2
View File
@@ -165,6 +165,8 @@ export async function finalizeEmbeddedAgentCommand(params: {
touchActivity: !isHeartbeatLifecycleRun && !params.opts.internalEvents?.length,
preserveRuntimeModel:
fallbackExhausted ||
fallbackProvider !== provider ||
fallbackModel !== model ||
isHeartbeatLifecycleRun ||
params.preserveUserFacingSessionModelState,
preserveUserFacingSessionModelState: params.preserveUserFacingSessionModelState,
+4 -5
View File
@@ -66,8 +66,8 @@ export async function updateSessionStoreAfterAgentRun(params: {
/**
* When true, preserve the pre-existing runtime model fields (model,
* modelProvider, contextTokens) on the session entry instead of overwriting
* them with the model used by this run. Used for heartbeat turns so the
* heartbeat model does not "bleed" into the main session's perceived state.
* them with the model used by this run. Used for turn-local fallback and
* heartbeat runs so their model does not bleed into the session selection.
*/
preserveRuntimeModel?: boolean;
preserveUserFacingSessionModelState?: boolean;
@@ -146,9 +146,8 @@ export async function updateSessionStoreAfterAgentRun(params: {
);
}
if (preserveRuntimeModel) {
// Keep the pre-existing runtime model and context window so a background
// heartbeat turn using a different model does not bleed into the main
// session's perceived state.
// Keep the pre-existing runtime model and context window so a turn-local
// model does not bleed into the session's perceived selection.
if (entry.model) {
// Prior runtime model exists: preserve its contextTokens. When missing,
// leave contextTokens unset rather than falling back to the heartbeat
@@ -251,7 +251,7 @@ describe("prepareEmbeddedRunTerminal run stats", () => {
expect(prepared.agentMeta).not.toHaveProperty("costUsd");
});
it("builds exact terminal model and successful-tool evidence", async () => {
it("keeps response identity in the terminal receipt without replacing the run model", async () => {
const prepared = await prepareStats({
responseModel: "cost-model-rerouted",
attempt: {
@@ -276,7 +276,7 @@ describe("prepareEmbeddedRunTerminal run stats", () => {
requested: { provider: "cost-test-provider", model: "cost-model" },
effective: {
provider: "cost-test-provider",
model: "cost-model-rerouted",
model: "cost-model",
responseModel: "cost-model-rerouted",
},
successfulToolNames: ["exec", "read", "Zeta", "alpha", "zeta"],
@@ -285,5 +285,7 @@ describe("prepareEmbeddedRunTerminal run stats", () => {
expect(
(prepared.agentMeta as { terminalReceipt?: Record<string, unknown> }).terminalReceipt,
).not.toHaveProperty("terminalDisposition");
expect(prepared.agentMeta.model).toBe("cost-model");
expect(prepared.reportedModelRef.model).toBe("cost-model");
});
});
@@ -75,13 +75,12 @@ export function prepareEmbeddedRunTerminal(input: {
latestUsage: terminalAssistant?.usage as UsageLike | undefined,
lastRunPromptUsage: input.lastRunPromptUsage,
});
const resolvedModelRef = resolveReportedModelRef({
const reportedModelRef = resolveReportedModelRef({
provider: input.provider,
model: input.model,
assistant: terminalAssistant,
});
const responseModel = terminalAssistant?.responseModel?.trim() || resolvedModelRef.model;
const reportedModelRef = { ...resolvedModelRef, model: responseModel };
const responseModel = terminalAssistant?.responseModel?.trim() || reportedModelRef.model;
const finalAssistantStopReason = (terminalAssistant?.stopReason ?? "").trim().toLowerCase();
const terminalAssistantCanOwnFinalText =
finalAssistantStopReason !== "error" && finalAssistantStopReason !== "aborted";
@@ -253,7 +253,8 @@ export async function accountAgentTurn(context: AgentTurnAccountingContext) {
compactionTokensAfter: runResult.meta?.agentMeta?.compactionTokensAfter,
promptTokens,
isHeartbeat,
preserveRuntimeModel: fallbackExhausted,
preserveRuntimeModel:
fallbackExhausted || fallbackTransition.nextState.selectedModel !== undefined,
preserveUserFacingSessionModelState: preserveUserFacingSessionState,
modelUsed,
providerUsed,
@@ -3439,8 +3439,10 @@ describe("runReplyAgent typing (heartbeat)", () => {
for (const testCase of cases) {
const sessionEntry = makeSessionEntry({
providerOverride: "openai",
modelOverride: "gpt-5.6-luna",
modelOverride: "gpt-5.6-sol",
modelOverrideSource: "user",
modelProvider: "openai",
model: "gpt-5.6-sol",
});
await replaceSessionEntry({ storePath, sessionKey: "main" }, sessionEntry);
const sessionStore = { main: sessionEntry };
@@ -3451,10 +3453,10 @@ describe("runReplyAgent typing (heartbeat)", () => {
vi.spyOn(modelFallbackModule, "runWithModelFallback").mockImplementationOnce(async (args) => {
const { run, onFallbackStep } = args;
expect(args.provider, testCase.name).toBe("openai");
expect(args.model, testCase.name).toBe("gpt-5.6-luna");
expect(args.model, testCase.name).toBe("gpt-5.6-sol");
await onFallbackStep?.({
fallbackStepType: "fallback_step",
fallbackStepFromModel: "openai/gpt-5.6-luna",
fallbackStepFromModel: "openai/gpt-5.6-sol",
fallbackStepToModel: "deepinfra/moonshotai/Kimi-K2.5",
fallbackStepFromFailureReason: "rate_limit",
fallbackStepFinalOutcome: "succeeded",
@@ -3467,7 +3469,7 @@ describe("runReplyAgent typing (heartbeat)", () => {
attempts: [
{
provider: "openai",
model: "gpt-5.6-luna",
model: "gpt-5.6-sol",
error: "Provider openai is in cooldown (all profiles unavailable)",
reason: "rate_limit",
},
@@ -3481,7 +3483,7 @@ describe("runReplyAgent typing (heartbeat)", () => {
sessionStore,
sessionKey: "main",
storePath,
runOverrides: { provider: "openai", model: "gpt-5.6-luna" },
runOverrides: { provider: "openai", model: "gpt-5.6-sol" },
});
const phases: string[] = [];
const off = onAgentEvent((evt) => {
@@ -3499,11 +3501,11 @@ describe("runReplyAgent typing (heartbeat)", () => {
expect(payload.text, testCase.name).toContain("Model Fallback:");
expect(payload.text, testCase.name).toContain("deepinfra/moonshotai/Kimi-K2.5");
expect(stored.providerOverride, testCase.name).toBe("openai");
expect(stored.modelOverride, testCase.name).toBe("gpt-5.6-luna");
expect(stored.modelOverride, testCase.name).toBe("gpt-5.6-sol");
expect(stored.modelOverrideSource, testCase.name).toBe("user");
expect(stored.modelProvider, testCase.name).toBe("deepinfra");
expect(stored.model, testCase.name).toBe("moonshotai/Kimi-K2.5");
expect(stored.fallbackNotice?.selectedModel, testCase.name).toBe("openai/gpt-5.6-luna");
expect(stored.modelProvider, testCase.name).toBe("openai");
expect(stored.model, testCase.name).toBe("gpt-5.6-sol");
expect(stored.fallbackNotice?.selectedModel, testCase.name).toBe("openai/gpt-5.6-sol");
expect(stored.fallbackNotice?.activeModel, testCase.name).toBe(
"deepinfra/moonshotai/Kimi-K2.5",
);
@@ -903,13 +903,15 @@ describe("gateway server chat", () => {
}
});
test("chat.history exposes persisted and synthetic session metadata for startup hydration", async () => {
test("chat.history exposes selected and synthetic session metadata for startup hydration", async () => {
await withGatewayChatHarness(async ({ ws, createSessionDir }) => {
await connectOk(ws);
await createSessionDir();
const updatedAt = Date.now();
await writeStoredMainSession({
updatedAt,
providerOverride: "openai",
modelOverride: "gpt-5",
modelProvider: "openai",
model: "gpt-5",
contextTokens: 128_000,
@@ -261,7 +261,7 @@ async function expectListedSessionActiveRun(
expect(session.activeRunIds).toEqual(expected ? ["run-1"] : undefined);
}
test("sessions.list keeps bulk rows lightweight and uses persisted model fields", async () => {
test("sessions.list keeps bulk rows lightweight and uses selected model fields", async () => {
const { storePath } = await createSessionStoreDir();
testState.agentConfig = {
models: {
@@ -273,6 +273,8 @@ test("sessions.list keeps bulk rows lightweight and uses persisted model fields"
main: sessionStoreEntry("sess-parent"),
"dashboard:child": sessionStoreEntry("sess-child", {
updatedAt: Date.now() - 1_000,
providerOverride: "anthropic",
modelOverride: "test-model-without-catalog-context",
modelProvider: "anthropic",
model: "test-model-without-catalog-context",
modelSelectionLocked: true,
@@ -351,13 +353,15 @@ test.each([
["my-ngc", "deepseek-ai/deepseek-v4-pro"],
["my-ngc:nvidia", "nvidia/nemotron-3-ultra-550b-a55b"],
])(
"sessions.list preserves custom provider %s and nested models over WebSocket",
"sessions.list preserves selected custom provider %s and nested models over WebSocket",
async (provider, model) => {
const { storePath } = await createSessionStoreDir();
await writeSessionStore({
entries: {
main: sessionStoreEntry("sess-parent"),
"dashboard:child": sessionStoreEntry("sess-custom-provider", {
providerOverride: provider,
modelOverride: model,
modelProvider: provider,
model,
parentSessionKey: "agent:main:main",
@@ -857,6 +861,8 @@ test("sessions.changed mutation events include live usage metadata", async () =>
await writeSessionStore({
entries: {
main: sessionStoreEntry("sess-main", {
providerOverride: "openai",
modelOverride: "gpt-5.3-codex-spark",
modelProvider: "openai",
model: "gpt-5.3-codex-spark",
contextTokens: 123_456,
@@ -12,6 +12,7 @@ import { expect, test, vi } from "vitest";
import { formatThinkingLevels } from "../auto-reply/thinking.js";
import { testState, writeSessionStore } from "./test-helpers.js";
import {
directSessionReq,
setupGatewaySessionsHandlerTestHarness,
getGatewayConfigModule,
getSessionsHandlers,
@@ -77,6 +78,7 @@ type ThinkingSession = {
key: string;
modelProvider?: string;
model?: string;
agentRuntime?: { id?: string };
thinkingLevels?: Array<{ label: string }>;
thinkingOptions?: string[];
};
@@ -91,6 +93,8 @@ async function listMainSessionWithThinking(params: {
primaryModel: string;
sessionModelProvider: string;
sessionModel: string;
agentRuntime?: "codex" | "openclaw";
selectedByOverride?: boolean;
readPreparedGatewayModelCatalog?: () => Promise<
Array<{
provider: string;
@@ -104,12 +108,25 @@ async function listMainSessionWithThinking(params: {
await createSessionStoreDir();
testState.agentConfig = {
model: { primary: params.primaryModel },
...(params.agentRuntime
? {
models: {
[params.primaryModel]: { agentRuntime: { id: params.agentRuntime } },
},
}
: {}),
};
await writeSessionStore({
entries: {
main: sessionStoreEntry("sess-main", {
modelProvider: params.sessionModelProvider,
model: params.sessionModel,
...(params.selectedByOverride === false
? {}
: {
providerOverride: params.sessionModelProvider,
modelOverride: params.sessionModel,
}),
}),
},
});
@@ -211,3 +228,49 @@ test("e2e #76482: session matching default model inherits default thinking level
expect(resolved).toContain("off");
expect(resolved).toContain("high");
});
test("session rows keep the selected Codex Sol model when runtime metadata contains a response family", async () => {
const loadSolCatalog = async () => [
{
provider: "openai",
id: "gpt-5.6-sol",
name: "GPT-5.6-Sol",
reasoning: true,
compat: {
supportedReasoningEfforts: ["low", "medium", "high", "xhigh", "max", "ultra"],
},
},
];
const { session } = await listMainSessionWithThinking({
reqId: "req-e2e-codex-sol-family",
primaryModel: "openai/gpt-5.6-sol",
sessionModelProvider: "openai",
sessionModel: "gpt-5.6",
agentRuntime: "codex",
selectedByOverride: false,
readPreparedGatewayModelCatalog: loadSolCatalog,
});
expect(session).toMatchObject({
modelProvider: "openai",
model: "gpt-5.6-sol",
});
expect(session?.agentRuntime?.id).toBe("codex");
expect(session?.thinkingOptions).toContain("max");
const patchResponse = await directSessionReq(
"sessions.patch",
{ key: "main", thinkingLevel: "max" },
{ context: { loadGatewayModelCatalog: loadSolCatalog } },
);
expect(patchResponse.ok, patchResponse.error?.message).toBe(true);
expect(patchResponse.error).toBeUndefined();
expect(patchResponse.payload).toMatchObject({
ok: true,
resolved: {
modelProvider: "openai",
model: "gpt-5.6-sol",
thinkingLevel: "max",
},
});
});
@@ -1753,6 +1753,8 @@ describe("session.message websocket events", () => {
main: {
sessionId: "sess-main",
updatedAt: Date.now(),
providerOverride: "openai",
modelOverride: "gpt-5.4",
modelProvider: "openai",
model: "gpt-5.4",
contextTokens: 123_456,
+2 -5
View File
@@ -87,14 +87,11 @@ export function resolveSessionSelectedModelRef(params: {
agentId: string;
rowContext?: SessionListRowContext;
allowPluginNormalization?: boolean;
}): ReturnType<typeof resolveSessionModelRef> | null {
}): ReturnType<typeof resolveSessionModelRef> {
const override = normalizeStoredOverrideModel({
providerOverride: params.entry?.providerOverride,
modelOverride: params.entry?.modelOverride,
});
if (!override.modelOverride) {
return null;
}
if (!params.rowContext) {
return resolveSessionModelRef(params.cfg, params.entry, params.agentId, {
allowPluginNormalization: params.allowPluginNormalization,
@@ -103,7 +100,7 @@ export function resolveSessionSelectedModelRef(params: {
const key = [
normalizeAgentId(params.agentId),
override.providerOverride ?? "",
override.modelOverride,
override.modelOverride ?? "",
].join("\0");
const cached = params.rowContext.selectedModelByOverrideRef.get(key);
if (cached) {
+7 -26
View File
@@ -234,8 +234,6 @@ export function buildGatewaySessionRow(params: {
subagentRun?.model,
{ allowPluginNormalization: !lightweight },
);
const runtimeModelPresent =
Boolean(entry?.model?.trim()) || Boolean(entry?.modelProvider?.trim());
const freshSessionTotalTokens = asNonNegativeFiniteNumber(resolveFreshSessionTotalTokens(entry));
const needsTranscriptTotalTokens = freshSessionTotalTokens === undefined;
const needsTranscriptContextTokens = resolvePositiveNumber(entry?.contextTokens) === undefined;
@@ -263,23 +261,6 @@ export function buildGatewaySessionRow(params: {
agentId: sessionAgentId,
})
: null;
const preferLiveSubagentModelIdentity =
Boolean(subagentRun?.model?.trim()) && subagentStatus === "running";
const shouldUseTranscriptModelIdentity =
runtimeModelPresent &&
!preferLiveSubagentModelIdentity &&
(needsTranscriptTotalTokens || needsTranscriptContextTokens);
const resolvedModelIdentity = {
provider: resolvedModel.provider,
model: resolvedModel.model ?? DEFAULT_MODEL,
};
const modelIdentity = shouldUseTranscriptModelIdentity
? {
provider: transcriptUsage?.modelProvider ?? resolvedModelIdentity.provider,
model: transcriptUsage?.model ?? resolvedModelIdentity.model,
}
: resolvedModelIdentity;
const { provider: modelProvider, model } = modelIdentity;
const totalTokens =
freshSessionTotalTokens ?? asNonNegativeFiniteNumber(transcriptUsage?.totalTokens);
const totalTokensFresh =
@@ -314,15 +295,15 @@ export function buildGatewaySessionRow(params: {
const latestCompactionCheckpoint = buildCompactionCheckpointPreview(
resolveLatestCompactionCheckpoint(compactionCheckpoints),
);
const selectedOrRuntimeModelProvider = selectedModel?.provider ?? modelProvider;
const selectedOrRuntimeModel = selectedModel?.model ?? model;
const selectedModelProvider = selectedModel.provider;
const selectedModelId = selectedModel.model;
const rowModelIdentity = lightweight
? { provider: selectedOrRuntimeModelProvider, model: selectedOrRuntimeModel }
? { provider: selectedModelProvider, model: selectedModelId }
: resolveSessionDisplayModelIdentityRefCached({
cfg,
agentId: sessionAgentId,
provider: selectedOrRuntimeModelProvider,
model: selectedOrRuntimeModel,
provider: selectedModelProvider,
model: selectedModelId,
rowContext: params.rowContext,
});
const rowModelProvider = rowModelIdentity.provider;
@@ -394,8 +375,8 @@ export function buildGatewaySessionRow(params: {
});
const fastModeState = resolveFastModeState({
cfg,
provider: selectedOrRuntimeModelProvider ?? DEFAULT_PROVIDER,
model: selectedOrRuntimeModel ?? DEFAULT_MODEL,
provider: selectedModelProvider,
model: selectedModelId,
agentId: sessionAgentId,
sessionEntry:
entry?.fastMode !== undefined
+3 -12
View File
@@ -2,7 +2,6 @@ import {
normalizeLowercaseStringOrEmpty,
normalizeOptionalString,
} from "@openclaw/normalization-core/string-coerce";
import { DEFAULT_MODEL } from "../agents/defaults.js";
import type { ModelCatalogEntry } from "../agents/model-catalog.js";
import { resolveSessionModelIdentityRef } from "../agents/session-model-ref.js";
import { getSessionDisplaySubagentRunByChildSessionKey } from "../agents/subagents/registry/subagent-registry-read.js";
@@ -128,17 +127,11 @@ export function resolveSessionListSearchModelFields(params: {
subagentRun?.model,
{ allowPluginNormalization: false },
);
const modelIdentity = {
provider: resolvedModel.provider,
model: resolvedModel.model ?? DEFAULT_MODEL,
};
const selectedOrRuntimeModelProvider = selectedModel?.provider ?? modelIdentity.provider;
const selectedOrRuntimeModel = selectedModel?.model ?? modelIdentity.model;
const displayModelIdentity = resolveSessionDisplayModelIdentityRefCached({
cfg: params.cfg,
agentId,
provider: selectedOrRuntimeModelProvider,
model: selectedOrRuntimeModel,
provider: selectedModel.provider,
model: selectedModel.model,
rowContext: params.rowContext,
});
const fields: Array<string | undefined> = [];
@@ -147,9 +140,7 @@ export function resolveSessionListSearchModelFields(params: {
model: params.entry?.model,
});
addSessionListSearchModelFields(fields, resolvedModel);
if (selectedModel) {
addSessionListSearchModelFields(fields, selectedModel);
}
addSessionListSearchModelFields(fields, selectedModel);
addSessionListSearchModelFields(fields, displayModelIdentity);
return fields;
}
+17 -8
View File
@@ -96,7 +96,7 @@ describe("filterAndSortSessionEntries search", () => {
}
});
test("filters by displayed provider and model identity", () => {
test("filters by selected and stored provider and model identity", () => {
const now = Date.now();
const cfg = createModelDefaultsConfig("anthropic/claude-sonnet-4-6");
const store: Record<string, SessionEntry> = {
@@ -121,12 +121,21 @@ describe("filterAndSortSessionEntries search", () => {
} as SessionEntry,
};
const cases = [
{ search: "anthropic", expectedKey: "agent:main:inherited-default" },
{ search: "claude-sonnet", expectedKey: "agent:main:inherited-default" },
{ search: "anthropic/claude-sonnet", expectedKey: "agent:main:inherited-default" },
{ search: "openai/gpt-5.5", expectedKey: "agent:main:override" },
{ search: "gemini-3.1", expectedKey: "agent:main:runtime" },
{ search: "google/gemini", expectedKey: "agent:main:runtime" },
{
search: "anthropic",
expectedKeys: ["agent:main:inherited-default", "agent:main:runtime"],
},
{
search: "claude-sonnet",
expectedKeys: ["agent:main:inherited-default", "agent:main:runtime"],
},
{
search: "anthropic/claude-sonnet",
expectedKeys: ["agent:main:inherited-default", "agent:main:runtime"],
},
{ search: "openai/gpt-5.5", expectedKeys: ["agent:main:override"] },
{ search: "gemini-3.1", expectedKeys: ["agent:main:runtime"] },
{ search: "google/gemini", expectedKeys: ["agent:main:runtime"] },
] as const;
for (const testCase of cases) {
@@ -137,7 +146,7 @@ describe("filterAndSortSessionEntries search", () => {
opts: { search: testCase.search },
now,
}),
).toEqual([testCase.expectedKey]);
).toEqual(testCase.expectedKeys);
}
});
+10 -6
View File
@@ -916,12 +916,16 @@ describe("gateway session utils", () => {
store: {
upper: {
sessionId: "upper",
providerOverride: "custom",
modelOverride: "CaseModel",
modelProvider: "custom",
model: "CaseModel",
updatedAt: 2,
} satisfies SessionEntry,
lower: {
sessionId: "lower",
providerOverride: "custom",
modelOverride: "casemodel",
modelProvider: "custom",
model: "casemodel",
updatedAt: 1,
@@ -3359,7 +3363,7 @@ describe("listSessionsFromStore selected model display", () => {
});
});
test("infers canonical provider for bare CLI models before default-provider fallback", () => {
test("ignores bare CLI runtime metadata when the selected default differs", () => {
const cfg = createModelDefaultsConfig({
primary: "openai/gpt-5.4",
models: {
@@ -3382,8 +3386,8 @@ describe("listSessionsFromStore selected model display", () => {
opts: {},
});
expect(result.sessions[0]?.modelProvider).toBe("anthropic");
expect(result.sessions[0]?.model).toBe("claude-opus-4-7");
expect(result.sessions[0]?.modelProvider).toBe("openai");
expect(result.sessions[0]?.model).toBe("gpt-5.4");
});
test("uses qualified selected defaults for rows without runtime model metadata", () => {
@@ -3435,7 +3439,7 @@ describe("listSessionsFromStore selected model display", () => {
]);
});
test("uses persisted runtime model metadata before selected defaults", () => {
test("uses selected defaults before persisted runtime model metadata", () => {
const cfg = {
agents: {
defaults: { model: { primary: "openai/gpt-5.4" } },
@@ -3457,8 +3461,8 @@ describe("listSessionsFromStore selected model display", () => {
opts: {},
});
expect(result.sessions[0]?.modelProvider).toBe("openai");
expect(result.sessions[0]?.model).toBe("gpt-5.5");
expect(result.sessions[0]?.modelProvider).toBe("anthropic");
expect(result.sessions[0]?.model).toBe("claude-sonnet-4-6");
});
test("uses complete model overrides without default-model fallback", () => {