From 8a3afa8820b4b439a99ef4168dd8fd42ea9d6bbe Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Sun, 23 Aug 2026 14:17:33 -0700 Subject: [PATCH] fix(qa): model-switch proof accepts served model reroutes (#127281) * fix(qa): accept response model reroutes * fix(qa): accept tool continuity response reroutes --------- Co-authored-by: Vincent Koc --- ...talog-model-switch-follow-up-proof.test.ts | 25 ++++++++++++++++++- ...model-switch-tool-continuity-proof.test.ts | 20 +++++++++++++-- .../models/model-switch-follow-up.yaml | 4 +-- .../models/model-switch-tool-continuity.yaml | 2 +- 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/extensions/qa-lab/src/scenario-catalog-model-switch-follow-up-proof.test.ts b/extensions/qa-lab/src/scenario-catalog-model-switch-follow-up-proof.test.ts index 25c2f791c35c..6e16880a714b 100644 --- a/extensions/qa-lab/src/scenario-catalog-model-switch-follow-up-proof.test.ts +++ b/extensions/qa-lab/src/scenario-catalog-model-switch-follow-up-proof.test.ts @@ -32,7 +32,7 @@ function terminalReceipt(params: { sessionId: "session-model-switch", turnId: `turn-${params.runId}`, requested: { provider: params.provider, model: params.model }, - effective: { provider: params.provider, model: responseModel, responseModel }, + effective: { provider: params.provider, model: params.model, responseModel }, successfulToolNames: [], rerouted: responseModel !== params.model, terminalDisposition: "visible", @@ -48,6 +48,8 @@ async function runFollowUp(params?: { alternateReplyText?: string; alternateOutboundText?: string; alternateDelivery?: { status: string; resultCount: number } | null; + primaryResponseModel?: string; + alternateResponseModel?: string; unrelatedPrimaryOutboundText?: string; unrelatedLaterOutboundText?: string; onRun?: () => void; @@ -102,6 +104,8 @@ async function runFollowUp(params?: { runId: call === 2 ? (params?.alternateReceiptRunId ?? runId) : runId, provider, model, + responseModel: + call === 1 ? params?.primaryResponseModel : params?.alternateResponseModel, }), }, }; @@ -159,6 +163,25 @@ describe("model-switch follow-up terminal evidence", () => { ); }); + it("accepts a response-model reroute recorded by the terminal receipt", async () => { + const { result } = await runFollowUp({ + primaryResponseModel: "primary-model-served", + alternateResponseModel: "alternate-model-served", + }); + + expect(result.status).toBe("pass"); + expect(result.modelSwitchEvidence).toMatchObject({ + primary: { + effective: { model: "primary-model", responseModel: "primary-model-served" }, + rerouted: true, + }, + alternate: { + effective: { model: "alternate-model", responseModel: "alternate-model-served" }, + rerouted: true, + }, + }); + }); + it("rejects normalized-identical refs before starting an agent run", async () => { const onRun = vi.fn(); await expect(runFollowUp({ alternateModel: "OPENAI/primary-model", onRun })).rejects.toThrow( diff --git a/extensions/qa-lab/src/scenario-catalog-model-switch-tool-continuity-proof.test.ts b/extensions/qa-lab/src/scenario-catalog-model-switch-tool-continuity-proof.test.ts index 47c9ce53988b..26a26cfe7433 100644 --- a/extensions/qa-lab/src/scenario-catalog-model-switch-tool-continuity-proof.test.ts +++ b/extensions/qa-lab/src/scenario-catalog-model-switch-tool-continuity-proof.test.ts @@ -32,6 +32,7 @@ async function runToolContinuity( alternateReplyText?: string; alternateOutboundText?: string; alternateDelivery?: { status: string; resultCount: number } | null; + alternateResponseModel?: string; unrelatedPrimaryOutboundText?: string; unrelatedLaterOutboundText?: string; }, @@ -44,6 +45,7 @@ async function runToolContinuity( const runId = `run-${call}`; const provider = prompt.provider ?? "openai"; const model = prompt.model ?? "primary-model"; + const responseModel = call === 2 ? (params?.alternateResponseModel ?? model) : model; const replyText = call === 1 ? "the QA scenario pack verifies source and docs" @@ -87,9 +89,9 @@ async function runToolContinuity( sessionId: "session-tools", turnId: `turn-${call}`, requested: { provider, model }, - effective: { provider, model, responseModel: model }, + effective: { provider, model, responseModel }, successfulToolNames: call === 1 ? (params?.primaryTools ?? ["read"]) : alternateTools, - rerouted: false, + rerouted: responseModel !== model, terminalDisposition: "visible", }, }, @@ -155,6 +157,20 @@ describe("model-switch tool continuity terminal evidence", () => { }); }); + it("accepts a response-model reroute recorded by the alternate terminal receipt", async () => { + const { result } = await runToolContinuity(["read"], { + alternateResponseModel: "alternate-model-served", + }); + + expect(result.status).toBe("pass"); + expect(result.modelSwitchEvidence).toMatchObject({ + alternate: { + effective: { model: "alternate-model", responseModel: "alternate-model-served" }, + rerouted: true, + }, + }); + }); + it("rejects a bare successful Code Mode exec without logical read evidence", async () => { await expect(runToolContinuity(["exec"])).rejects.toThrow( "alternate-model run did not return exact owned successful read evidence", diff --git a/qa/scenarios/models/model-switch-follow-up.yaml b/qa/scenarios/models/model-switch-follow-up.yaml index df61728131a7..04407a928961 100644 --- a/qa/scenarios/models/model-switch-follow-up.yaml +++ b/qa/scenarios/models/model-switch-follow-up.yaml @@ -46,7 +46,7 @@ flow: timeoutMs: expr: liveTurnTimeoutMs(env, 30000) - assert: - expr: "(() => { const expected = normalizeModelRef(env.primaryModel); const receipt = primaryRun?.waited?.terminalReceipt; return receipt?.runId === primaryRun?.started?.runId && Boolean(receipt.sessionId) && Boolean(receipt.turnId) && normalizeLowercaseStringOrEmpty(receipt.requested?.provider) === expected.provider && receipt.requested?.model === expected.model && receipt.effective?.model === receipt.effective?.responseModel && receipt.terminalDisposition === 'visible' && typeof receipt.rerouted === 'boolean'; })()" + expr: "(() => { const expected = normalizeModelRef(env.primaryModel); const receipt = primaryRun?.waited?.terminalReceipt; return receipt?.runId === primaryRun?.started?.runId && Boolean(receipt.sessionId) && Boolean(receipt.turnId) && normalizeLowercaseStringOrEmpty(receipt.requested?.provider) === expected.provider && receipt.requested?.model === expected.model && Boolean(receipt.effective?.provider) && Boolean(receipt.effective?.model) && Boolean(receipt.effective?.responseModel) && receipt.rerouted === (receipt.effective.provider !== receipt.requested.provider || receipt.effective.model !== receipt.requested.model || receipt.effective.responseModel !== receipt.requested.model) && receipt.terminalDisposition === 'visible'; })()" message: default-model run did not return an exact owned terminal receipt - assert: expr: "primaryRun?.waited?.terminalReply?.disposition === 'visible'" @@ -71,7 +71,7 @@ flow: timeoutMs: expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.alternateModel) - assert: - expr: "(() => { const receipt = alternateRun?.waited?.terminalReceipt; return receipt?.runId === alternateRun?.started?.runId && Boolean(receipt.sessionId) && Boolean(receipt.turnId) && normalizeLowercaseStringOrEmpty(receipt.requested?.provider) === expectedAlternate.provider && receipt.requested?.model === expectedAlternate.model && receipt.effective?.model === receipt.effective?.responseModel && receipt.terminalDisposition === 'visible' && typeof receipt.rerouted === 'boolean' && `${receipt.effective?.provider}/${receipt.effective?.responseModel}` !== `${primaryRun.waited.terminalReceipt.effective?.provider}/${primaryRun.waited.terminalReceipt.effective?.responseModel}`; })()" + expr: "(() => { const receipt = alternateRun?.waited?.terminalReceipt; return receipt?.runId === alternateRun?.started?.runId && Boolean(receipt.sessionId) && Boolean(receipt.turnId) && normalizeLowercaseStringOrEmpty(receipt.requested?.provider) === expectedAlternate.provider && receipt.requested?.model === expectedAlternate.model && Boolean(receipt.effective?.provider) && Boolean(receipt.effective?.model) && Boolean(receipt.effective?.responseModel) && receipt.rerouted === (receipt.effective.provider !== receipt.requested.provider || receipt.effective.model !== receipt.requested.model || receipt.effective.responseModel !== receipt.requested.model) && receipt.terminalDisposition === 'visible' && `${receipt.effective.provider}/${receipt.effective.responseModel}` !== `${primaryRun.waited.terminalReceipt.effective.provider}/${primaryRun.waited.terminalReceipt.effective.responseModel}`; })()" message: alternate-model run did not return distinct exact owned model evidence - assert: expr: "(() => { const reply = alternateRun?.waited?.terminalReply; if (reply?.disposition !== 'visible') return false; const lower = normalizeLowercaseStringOrEmpty(reply.text); return lower.includes('switch') || lower.includes('handoff'); })()" diff --git a/qa/scenarios/models/model-switch-tool-continuity.yaml b/qa/scenarios/models/model-switch-tool-continuity.yaml index 78fac2202959..a0c9977dd823 100644 --- a/qa/scenarios/models/model-switch-tool-continuity.yaml +++ b/qa/scenarios/models/model-switch-tool-continuity.yaml @@ -90,7 +90,7 @@ flow: terminalDelivery: ref: alternateRun.waited.terminalDelivery - assert: - expr: "(() => { const receipt = alternateRun?.waited?.terminalReceipt; return receipt?.runId === alternateRun?.started?.runId && Boolean(receipt.sessionId) && Boolean(receipt.turnId) && normalizeLowercaseStringOrEmpty(receipt.requested?.provider) === expectedAlternate.provider && receipt.requested?.model === expectedAlternate.model && receipt.effective?.model === receipt.effective?.responseModel && receipt.successfulToolNames?.includes('read') && receipt.terminalDisposition === 'visible' && typeof receipt.rerouted === 'boolean' && `${receipt.effective?.provider}/${receipt.effective?.responseModel}` !== `${primaryRun.waited.terminalReceipt.effective?.provider}/${primaryRun.waited.terminalReceipt.effective?.responseModel}`; })()" + expr: "(() => { const receipt = alternateRun?.waited?.terminalReceipt; return receipt?.runId === alternateRun?.started?.runId && Boolean(receipt.sessionId) && Boolean(receipt.turnId) && normalizeLowercaseStringOrEmpty(receipt.requested?.provider) === expectedAlternate.provider && receipt.requested?.model === expectedAlternate.model && Boolean(receipt.effective?.provider) && Boolean(receipt.effective?.model) && Boolean(receipt.effective?.responseModel) && receipt.rerouted === (receipt.effective.provider !== receipt.requested.provider || receipt.effective.model !== receipt.requested.model || receipt.effective.responseModel !== receipt.requested.model) && receipt.successfulToolNames?.includes('read') && receipt.terminalDisposition === 'visible' && `${receipt.effective.provider}/${receipt.effective.responseModel}` !== `${primaryRun.waited.terminalReceipt.effective.provider}/${primaryRun.waited.terminalReceipt.effective.responseModel}`; })()" message: alternate-model run did not return exact owned successful read evidence - assert: expr: "alternateRun?.waited?.terminalReply?.disposition === 'visible' && hasModelSwitchContinuitySignal(alternateRun.waited.terminalReply.text)"