fix(qa): model-switch proof accepts served model reroutes (#127281)

* fix(qa): accept response model reroutes

* fix(qa): accept tool continuity response reroutes

---------

Co-authored-by: Vincent Koc <vincentkoc@ieee.org>

extensions/qa-lab/src/scenario-catalog-model-switch-follow-up-proof.test.ts

@@ -32,7 +32,7 @@ function terminalReceipt(params: {
     sessionId: "session-model-switch",
     turnId: `turn-${params.runId}`,
     requested: { provider: params.provider, model: params.model },
-    effective: { provider: params.provider, model: responseModel, responseModel },
+    effective: { provider: params.provider, model: params.model, responseModel },
     successfulToolNames: [],
     rerouted: responseModel !== params.model,
     terminalDisposition: "visible",
@@ -48,6 +48,8 @@ async function runFollowUp(params?: {
   alternateReplyText?: string;
   alternateOutboundText?: string;
   alternateDelivery?: { status: string; resultCount: number } | null;
+  primaryResponseModel?: string;
+  alternateResponseModel?: string;
   unrelatedPrimaryOutboundText?: string;
   unrelatedLaterOutboundText?: string;
   onRun?: () => void;
@@ -102,6 +104,8 @@ async function runFollowUp(params?: {
             runId: call === 2 ? (params?.alternateReceiptRunId ?? runId) : runId,
             provider,
             model,
+            responseModel:
+              call === 1 ? params?.primaryResponseModel : params?.alternateResponseModel,
           }),
         },
       };
@@ -159,6 +163,25 @@ describe("model-switch follow-up terminal evidence", () => {
     );
   });
 
+  it("accepts a response-model reroute recorded by the terminal receipt", async () => {
+    const { result } = await runFollowUp({
+      primaryResponseModel: "primary-model-served",
+      alternateResponseModel: "alternate-model-served",
+    });
+
+    expect(result.status).toBe("pass");
+    expect(result.modelSwitchEvidence).toMatchObject({
+      primary: {
+        effective: { model: "primary-model", responseModel: "primary-model-served" },
+        rerouted: true,
+      },
+      alternate: {
+        effective: { model: "alternate-model", responseModel: "alternate-model-served" },
+        rerouted: true,
+      },
+    });
+  });
+
   it("rejects normalized-identical refs before starting an agent run", async () => {
     const onRun = vi.fn();
     await expect(runFollowUp({ alternateModel: "OPENAI/primary-model", onRun })).rejects.toThrow(

extensions/qa-lab/src/scenario-catalog-model-switch-tool-continuity-proof.test.ts

@@ -32,6 +32,7 @@ async function runToolContinuity(
     alternateReplyText?: string;
     alternateOutboundText?: string;
     alternateDelivery?: { status: string; resultCount: number } | null;
+    alternateResponseModel?: string;
     unrelatedPrimaryOutboundText?: string;
     unrelatedLaterOutboundText?: string;
   },
@@ -44,6 +45,7 @@ async function runToolContinuity(
       const runId = `run-${call}`;
       const provider = prompt.provider ?? "openai";
       const model = prompt.model ?? "primary-model";
+      const responseModel = call === 2 ? (params?.alternateResponseModel ?? model) : model;
       const replyText =
         call === 1
           ? "the QA scenario pack verifies source and docs"
@@ -87,9 +89,9 @@ async function runToolContinuity(
             sessionId: "session-tools",
             turnId: `turn-${call}`,
             requested: { provider, model },
-            effective: { provider, model, responseModel: model },
+            effective: { provider, model, responseModel },
             successfulToolNames: call === 1 ? (params?.primaryTools ?? ["read"]) : alternateTools,
-            rerouted: false,
+            rerouted: responseModel !== model,
             terminalDisposition: "visible",
           },
         },
@@ -155,6 +157,20 @@ describe("model-switch tool continuity terminal evidence", () => {
     });
   });
 
+  it("accepts a response-model reroute recorded by the alternate terminal receipt", async () => {
+    const { result } = await runToolContinuity(["read"], {
+      alternateResponseModel: "alternate-model-served",
+    });
+
+    expect(result.status).toBe("pass");
+    expect(result.modelSwitchEvidence).toMatchObject({
+      alternate: {
+        effective: { model: "alternate-model", responseModel: "alternate-model-served" },
+        rerouted: true,
+      },
+    });
+  });
+
   it("rejects a bare successful Code Mode exec without logical read evidence", async () => {
     await expect(runToolContinuity(["exec"])).rejects.toThrow(
       "alternate-model run did not return exact owned successful read evidence",

qa/scenarios/models/model-switch-follow-up.yaml

@@ -46,7 +46,7 @@ flow:
               timeoutMs:
                 expr: liveTurnTimeoutMs(env, 30000)
         - assert:
-            expr: "(() => { const expected = normalizeModelRef(env.primaryModel); const receipt = primaryRun?.waited?.terminalReceipt; return receipt?.runId === primaryRun?.started?.runId && Boolean(receipt.sessionId) && Boolean(receipt.turnId) && normalizeLowercaseStringOrEmpty(receipt.requested?.provider) === expected.provider && receipt.requested?.model === expected.model && receipt.effective?.model === receipt.effective?.responseModel && receipt.terminalDisposition === 'visible' && typeof receipt.rerouted === 'boolean'; })()"
+            expr: "(() => { const expected = normalizeModelRef(env.primaryModel); const receipt = primaryRun?.waited?.terminalReceipt; return receipt?.runId === primaryRun?.started?.runId && Boolean(receipt.sessionId) && Boolean(receipt.turnId) && normalizeLowercaseStringOrEmpty(receipt.requested?.provider) === expected.provider && receipt.requested?.model === expected.model && Boolean(receipt.effective?.provider) && Boolean(receipt.effective?.model) && Boolean(receipt.effective?.responseModel) && receipt.rerouted === (receipt.effective.provider !== receipt.requested.provider || receipt.effective.model !== receipt.requested.model || receipt.effective.responseModel !== receipt.requested.model) && receipt.terminalDisposition === 'visible'; })()"
             message: default-model run did not return an exact owned terminal receipt
         - assert:
             expr: "primaryRun?.waited?.terminalReply?.disposition === 'visible'"
@@ -71,7 +71,7 @@ flow:
               timeoutMs:
                 expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.alternateModel)
         - assert:
-            expr: "(() => { const receipt = alternateRun?.waited?.terminalReceipt; return receipt?.runId === alternateRun?.started?.runId && Boolean(receipt.sessionId) && Boolean(receipt.turnId) && normalizeLowercaseStringOrEmpty(receipt.requested?.provider) === expectedAlternate.provider && receipt.requested?.model === expectedAlternate.model && receipt.effective?.model === receipt.effective?.responseModel && receipt.terminalDisposition === 'visible' && typeof receipt.rerouted === 'boolean' && `${receipt.effective?.provider}/${receipt.effective?.responseModel}` !== `${primaryRun.waited.terminalReceipt.effective?.provider}/${primaryRun.waited.terminalReceipt.effective?.responseModel}`; })()"
+            expr: "(() => { const receipt = alternateRun?.waited?.terminalReceipt; return receipt?.runId === alternateRun?.started?.runId && Boolean(receipt.sessionId) && Boolean(receipt.turnId) && normalizeLowercaseStringOrEmpty(receipt.requested?.provider) === expectedAlternate.provider && receipt.requested?.model === expectedAlternate.model && Boolean(receipt.effective?.provider) && Boolean(receipt.effective?.model) && Boolean(receipt.effective?.responseModel) && receipt.rerouted === (receipt.effective.provider !== receipt.requested.provider || receipt.effective.model !== receipt.requested.model || receipt.effective.responseModel !== receipt.requested.model) && receipt.terminalDisposition === 'visible' && `${receipt.effective.provider}/${receipt.effective.responseModel}` !== `${primaryRun.waited.terminalReceipt.effective.provider}/${primaryRun.waited.terminalReceipt.effective.responseModel}`; })()"
             message: alternate-model run did not return distinct exact owned model evidence
         - assert:
             expr: "(() => { const reply = alternateRun?.waited?.terminalReply; if (reply?.disposition !== 'visible') return false; const lower = normalizeLowercaseStringOrEmpty(reply.text); return lower.includes('switch') || lower.includes('handoff'); })()"

qa/scenarios/models/model-switch-tool-continuity.yaml

@@ -90,7 +90,7 @@ flow:
             terminalDelivery:
               ref: alternateRun.waited.terminalDelivery
         - assert:
-            expr: "(() => { const receipt = alternateRun?.waited?.terminalReceipt; return receipt?.runId === alternateRun?.started?.runId && Boolean(receipt.sessionId) && Boolean(receipt.turnId) && normalizeLowercaseStringOrEmpty(receipt.requested?.provider) === expectedAlternate.provider && receipt.requested?.model === expectedAlternate.model && receipt.effective?.model === receipt.effective?.responseModel && receipt.successfulToolNames?.includes('read') && receipt.terminalDisposition === 'visible' && typeof receipt.rerouted === 'boolean' && `${receipt.effective?.provider}/${receipt.effective?.responseModel}` !== `${primaryRun.waited.terminalReceipt.effective?.provider}/${primaryRun.waited.terminalReceipt.effective?.responseModel}`; })()"
+            expr: "(() => { const receipt = alternateRun?.waited?.terminalReceipt; return receipt?.runId === alternateRun?.started?.runId && Boolean(receipt.sessionId) && Boolean(receipt.turnId) && normalizeLowercaseStringOrEmpty(receipt.requested?.provider) === expectedAlternate.provider && receipt.requested?.model === expectedAlternate.model && Boolean(receipt.effective?.provider) && Boolean(receipt.effective?.model) && Boolean(receipt.effective?.responseModel) && receipt.rerouted === (receipt.effective.provider !== receipt.requested.provider || receipt.effective.model !== receipt.requested.model || receipt.effective.responseModel !== receipt.requested.model) && receipt.successfulToolNames?.includes('read') && receipt.terminalDisposition === 'visible' && `${receipt.effective.provider}/${receipt.effective.responseModel}` !== `${primaryRun.waited.terminalReceipt.effective.provider}/${primaryRun.waited.terminalReceipt.effective.responseModel}`; })()"
             message: alternate-model run did not return exact owned successful read evidence
         - assert:
             expr: "alternateRun?.waited?.terminalReply?.disposition === 'visible' && hasModelSwitchContinuitySignal(alternateRun.waited.terminalReply.text)"