fix(qa): keep live provider scenarios out of mock lanes (#119498)

extensions/qa-lab/src/scenario-catalog-anthropic-live.test.ts

@@ -0,0 +1,47 @@
+// Qa Lab tests keep Anthropic provider smoke evidence on real model lanes.
+import { describe, expect, it } from "vitest";
+import { readQaScenarioById, readQaScenarioExecutionConfig } from "./scenario-catalog.js";
+import { selectQaFlowSuiteScenarios } from "./suite-planning.js";
+
+const ANTHROPIC_OPUS_SCENARIO_IDS = [
+  "anthropic-opus-api-key-smoke",
+  "anthropic-opus-setup-token-smoke",
+] as const;
+
+describe("QA Anthropic live scenario catalog", () => {
+  it.each(ANTHROPIC_OPUS_SCENARIO_IDS)("pins %s to live Anthropic Opus", (scenarioId) => {
+    const scenario = readQaScenarioById(scenarioId);
+
+    expect(readQaScenarioExecutionConfig(scenarioId)).toMatchObject({
+      requiredProviderMode: "live-frontier",
+      requiredProvider: "anthropic",
+      requiredModel: "claude-opus-5",
+    });
+    expect(scenario.execution.flow?.steps.at(-1)?.actions).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ call: "runAgentPrompt" }),
+        expect.objectContaining({ call: "waitForOutboundMessage" }),
+      ]),
+    );
+  });
+
+  it.each(ANTHROPIC_OPUS_SCENARIO_IDS)(
+    "rejects %s from matching-provider mock lanes",
+    (scenarioId) => {
+      const scenario = readQaScenarioById(scenarioId);
+      const mockLane = {
+        scenarios: [scenario],
+        providerMode: "mock-openai" as const,
+        primaryModel: "anthropic/claude-opus-5",
+      };
+
+      expect(selectQaFlowSuiteScenarios({ ...mockLane, providerMode: "live-frontier" })).toEqual([
+        scenario,
+      ]);
+      expect(selectQaFlowSuiteScenarios(mockLane)).toEqual([]);
+      expect(() => selectQaFlowSuiteScenarios({ ...mockLane, scenarioIds: [scenario.id] })).toThrow(
+        "providerMode=live-frontier",
+      );
+    },
+  );
+});

extensions/qa-lab/src/scenario-catalog.openai-live.test.ts

@@ -1,6 +1,7 @@
 // Qa Lab tests cover live OpenAI scenario catalog metadata.
 import { describe, expect, it } from "vitest";
 import { readQaScenarioById, readQaScenarioExecutionConfig } from "./scenario-catalog.js";
+import { selectQaFlowSuiteScenarios } from "./suite-planning.js";
 
 describe("qa scenario catalog", () => {
   it("includes the GPT-5.6 Luna thinking visibility switch scenario", () => {
@@ -36,6 +37,7 @@ describe("qa scenario catalog", () => {
     const scenario = readQaScenarioById("openai-native-web-search-live");
     const config = readQaScenarioExecutionConfig("openai-native-web-search-live") as
       | {
+          requiredProviderMode?: string;
           requiredProvider?: string;
           requiredModel?: string;
           expectedMarker?: string;
@@ -51,6 +53,7 @@ describe("qa scenario catalog", () => {
         },
       },
     });
+    expect(config?.requiredProviderMode).toBe("live-frontier");
     expect(config?.requiredProvider).toBe("openai");
     expect(config?.requiredModel).toBe("gpt-5.6-luna");
     expect(config?.expectedMarker).toBe("WEB-SEARCH-OK");
@@ -58,6 +61,29 @@ describe("qa scenario catalog", () => {
       "confirms live OpenAI GPT-5.6 Luna web search auto mode",
       "searches official OpenAI News through the live model",
     ]);
+    expect(scenario.execution.flow?.steps.at(-1)?.actions).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ call: "runAgentPrompt" }),
+        expect.objectContaining({ call: "waitForOutboundMessage" }),
+      ]),
+    );
+  });
+
+  it("rejects the native web-search probe from matching-provider mock lanes", () => {
+    const scenario = readQaScenarioById("openai-native-web-search-live");
+    const mockLane = {
+      scenarios: [scenario],
+      providerMode: "mock-openai" as const,
+      primaryModel: "openai/gpt-5.6-luna",
+    };
+
+    expect(selectQaFlowSuiteScenarios({ ...mockLane, providerMode: "live-frontier" })).toEqual([
+      scenario,
+    ]);
+    expect(selectQaFlowSuiteScenarios(mockLane)).toEqual([]);
+    expect(() => selectQaFlowSuiteScenarios({ ...mockLane, scenarioIds: [scenario.id] })).toThrow(
+      "providerMode=live-frontier",
+    );
   });
 
   it("includes the live inbound voice talkback scenario", () => {

qa/scenarios/models/anthropic-opus-api-key-smoke.yaml

@@ -25,6 +25,7 @@ scenario:
     kind: flow
     summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --model anthropic/claude-opus-5 --alt-model anthropic/claude-opus-5 --scenario anthropic-opus-api-key-smoke`.
     config:
+      requiredProviderMode: live-frontier
       requiredProvider: anthropic
       requiredModel: claude-opus-5
       chatPrompt: "Anthropic Opus API key smoke. Reply exactly: ANTHROPIC-OPUS-API-KEY-OK"
@@ -38,50 +39,49 @@ flow:
           value:
             expr: splitModelRef(env.primaryModel)
         - assert:
-            expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider"
+            expr: "env.providerMode === config.requiredProviderMode"
+            message:
+              expr: "`expected provider mode ${config.requiredProviderMode}, got ${env.providerMode}`"
+        - assert:
+            expr: "selected?.provider === config.requiredProvider"
             message:
               expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`"
         - assert:
-            expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel"
+            expr: "selected?.model === config.requiredModel"
             message:
               expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`"
         - assert:
-            expr: "env.providerMode !== 'live-frontier' || Boolean(env.gateway.runtimeEnv.ANTHROPIC_API_KEY?.trim())"
+            expr: "Boolean(env.gateway.runtimeEnv.ANTHROPIC_API_KEY?.trim())"
             message: expected ANTHROPIC_API_KEY to be available for API-key QA mode
-      detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} auth=env-api-key` : `mock-compatible provider=${selected?.provider}`"
+      detailsExpr: "`provider=${selected?.provider} model=${selected?.model} auth=env-api-key`"
     - name: talks through regular Anthropic Opus
       actions:
-        - if:
-            expr: "env.providerMode !== 'live-frontier'"
-            then:
-              - assert: "true"
-            else:
-              - call: reset
-              - set: selected
-                value:
-                  expr: splitModelRef(env.primaryModel)
-              - call: runAgentPrompt
-                args:
-                  - ref: env
-                  - sessionKey: agent:qa:anthropic-opus-api-key
-                    message:
-                      expr: config.chatPrompt
-                    provider:
-                      expr: selected?.provider
-                    model:
-                      expr: selected?.model
-                    timeoutMs:
-                      expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
-              - call: waitForOutboundMessage
-                saveAs: chatOutbound
-                args:
-                  - ref: state
-                  - lambda:
-                      params: [candidate]
-                      expr: "candidate.conversation.id === 'qa-operator'"
-                  - expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel)
-              - assert:
-                  expr: "chatOutbound.text.includes(config.chatExpected)"
-                  message:
-                    expr: "`chat marker missing: ${chatOutbound.text}`"
-      detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live Anthropic smoke' : chatOutbound.text"
+        - call: reset
+        - set: selected
+          value:
+            expr: splitModelRef(env.primaryModel)
+        - call: runAgentPrompt
+          args:
+            - ref: env
+            - sessionKey: agent:qa:anthropic-opus-api-key
+              message:
+                expr: config.chatPrompt
+              provider:
+                expr: selected?.provider
+              model:
+                expr: selected?.model
+              timeoutMs:
+                expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
+        - call: waitForOutboundMessage
+          saveAs: chatOutbound
+          args:
+            - ref: state
+            - lambda:
+                params: [candidate]
+                expr: "candidate.conversation.id === 'qa-operator'"
+            - expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel)
+        - assert:
+            expr: "chatOutbound.text.includes(config.chatExpected)"
+            message:
+              expr: "`chat marker missing: ${chatOutbound.text}`"
+      detailsExpr: chatOutbound.text

qa/scenarios/models/anthropic-opus-setup-token-smoke.yaml

@@ -25,6 +25,7 @@ scenario:
     kind: flow
     summary: Run with `OPENCLAW_LIVE_SETUP_TOKEN_VALUE=<setup-token> pnpm openclaw qa suite --provider-mode live-frontier --model anthropic/claude-opus-5 --alt-model anthropic/claude-opus-5 --scenario anthropic-opus-setup-token-smoke`.
     config:
+      requiredProviderMode: live-frontier
       requiredProvider: anthropic
       requiredModel: claude-opus-5
       profileId: "anthropic:qa-setup-token"
@@ -39,54 +40,53 @@ flow:
           value:
             expr: splitModelRef(env.primaryModel)
         - assert:
-            expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider"
+            expr: "env.providerMode === config.requiredProviderMode"
+            message:
+              expr: "`expected provider mode ${config.requiredProviderMode}, got ${env.providerMode}`"
+        - assert:
+            expr: "selected?.provider === config.requiredProvider"
             message:
               expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`"
         - assert:
-            expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel"
+            expr: "selected?.model === config.requiredModel"
             message:
               expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`"
         - assert:
-            expr: "env.providerMode !== 'live-frontier' || env.gateway.cfg.auth?.profiles?.[config.profileId]?.mode === 'token'"
+            expr: "env.gateway.cfg.auth?.profiles?.[config.profileId]?.mode === 'token'"
             message:
               expr: "`expected token profile ${config.profileId} in QA config`"
         - assert:
-            expr: "env.providerMode !== 'live-frontier' || !env.gateway.runtimeEnv.OPENCLAW_LIVE_SETUP_TOKEN_VALUE"
+            expr: "!env.gateway.runtimeEnv.OPENCLAW_LIVE_SETUP_TOKEN_VALUE"
             message: setup-token value should not be passed to the gateway child env
-      detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} auth=setup-token profile=${config.profileId}` : `mock-compatible provider=${selected?.provider}`"
+      detailsExpr: "`provider=${selected?.provider} model=${selected?.model} auth=setup-token profile=${config.profileId}`"
     - name: talks through regular Anthropic Opus
       actions:
-        - if:
-            expr: "env.providerMode !== 'live-frontier'"
-            then:
-              - assert: "true"
-            else:
-              - call: reset
-              - set: selected
-                value:
-                  expr: splitModelRef(env.primaryModel)
-              - call: runAgentPrompt
-                args:
-                  - ref: env
-                  - sessionKey: agent:qa:anthropic-opus-setup-token
-                    message:
-                      expr: config.chatPrompt
-                    provider:
-                      expr: selected?.provider
-                    model:
-                      expr: selected?.model
-                    timeoutMs:
-                      expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
-              - call: waitForOutboundMessage
-                saveAs: chatOutbound
-                args:
-                  - ref: state
-                  - lambda:
-                      params: [candidate]
-                      expr: "candidate.conversation.id === 'qa-operator'"
-                  - expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel)
-              - assert:
-                  expr: "chatOutbound.text.includes(config.chatExpected)"
-                  message:
-                    expr: "`chat marker missing: ${chatOutbound.text}`"
-      detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live Anthropic smoke' : chatOutbound.text"
+        - call: reset
+        - set: selected
+          value:
+            expr: splitModelRef(env.primaryModel)
+        - call: runAgentPrompt
+          args:
+            - ref: env
+            - sessionKey: agent:qa:anthropic-opus-setup-token
+              message:
+                expr: config.chatPrompt
+              provider:
+                expr: selected?.provider
+              model:
+                expr: selected?.model
+              timeoutMs:
+                expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
+        - call: waitForOutboundMessage
+          saveAs: chatOutbound
+          args:
+            - ref: state
+            - lambda:
+                params: [candidate]
+                expr: "candidate.conversation.id === 'qa-operator'"
+            - expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel)
+        - assert:
+            expr: "chatOutbound.text.includes(config.chatExpected)"
+            message:
+              expr: "`chat marker missing: ${chatOutbound.text}`"
+      detailsExpr: chatOutbound.text

qa/scenarios/models/openai-native-web-search-live.yaml

@@ -33,6 +33,7 @@ scenario:
     kind: flow
     summary: Run with `OPENCLAW_LIVE_OPENAI_KEY="${OPENAI_API_KEY}" pnpm openclaw qa suite --provider-mode live-frontier --model openai/gpt-5.6-luna --alt-model openai/gpt-5.6-luna --fast --thinking medium --scenario openai-native-web-search-live`.
     config:
+      requiredProviderMode: live-frontier
       requiredProvider: openai
       requiredModel: gpt-5.6-luna
       expectedMarker: WEB-SEARCH-OK
@@ -61,11 +62,15 @@ flow:
           value:
             expr: splitModelRef(env.primaryModel)
         - assert:
-            expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider"
+            expr: "env.providerMode === config.requiredProviderMode"
+            message:
+              expr: "`expected provider mode ${config.requiredProviderMode}, got ${env.providerMode}`"
+        - assert:
+            expr: "selected?.provider === config.requiredProvider"
             message:
               expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`"
         - assert:
-            expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel"
+            expr: "selected?.model === config.requiredModel"
             message:
               expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`"
         - call: readConfigSnapshot
@@ -84,60 +89,55 @@ flow:
             message:
               expr: "`expected web search provider auto/openai/unset for native OpenAI search, got ${JSON.stringify(searchConfig)}`"
         - assert:
-            expr: "env.providerMode !== 'live-frontier' || Boolean(env.gateway.runtimeEnv.OPENAI_API_KEY?.trim() || env.gateway.runtimeEnv.OPENCLAW_LIVE_OPENAI_KEY?.trim())"
+            expr: "Boolean(env.gateway.runtimeEnv.OPENAI_API_KEY?.trim() || env.gateway.runtimeEnv.OPENCLAW_LIVE_OPENAI_KEY?.trim())"
             message: expected OPENAI_API_KEY or OPENCLAW_LIVE_OPENAI_KEY for live OpenAI QA
-      detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} webSearch=${JSON.stringify(searchConfig)}` : `mock-compatible provider=${selected?.provider}`"
+      detailsExpr: "`provider=${selected?.provider} model=${selected?.model} webSearch=${JSON.stringify(searchConfig)}`"
     - name: searches official OpenAI News through the live model
       actions:
-        - if:
-            expr: "env.providerMode !== 'live-frontier'"
-            then:
-              - assert: "true"
-            else:
-              - call: reset
-              - set: selected
-                value:
-                  expr: splitModelRef(env.primaryModel)
-              - call: runAgentPrompt
-                args:
-                  - ref: env
-                  - sessionKey: agent:qa:openai-native-web-search
-                    message:
-                      expr: config.searchPrompt
-                    provider:
-                      expr: selected?.provider
-                    model:
-                      expr: selected?.model
-                    timeoutMs:
-                      expr: resolveQaLiveTurnTimeoutMs(env, 180000, env.primaryModel)
-              - call: waitForOutboundMessage
-                saveAs: searchOutbound
-                args:
-                  - ref: state
-                  - lambda:
-                      params: [candidate]
-                      expr: "candidate.conversation.id === 'qa-operator'"
-                  - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
-              - set: searchText
-                value:
-                  expr: searchOutbound.text
-              - set: searchTextLower
-                value:
-                  expr: normalizeLowercaseStringOrEmpty(searchText)
-              - assert:
-                  expr: "searchText.includes(config.expectedMarker)"
-                  message:
-                    expr: "`missing ${config.expectedMarker}: ${searchText}`"
-              - assert:
-                  expr: "!searchText.includes(config.failureMarker) && !/(web search is unavailable|unable to search|cannot search|can't search)/i.test(searchText)"
-                  message:
-                    expr: "`search looked unavailable: ${searchText}`"
-              - assert:
-                  expr: "/URL:\\s*https?:\\/\\/[^\\s]*openai\\.com\\/news/i.test(searchText)"
-                  message:
-                    expr: "`missing official OpenAI News URL: ${searchText}`"
-              - assert:
-                  expr: "/HEADLINE:\\s*\\S.{8,}/i.test(searchText)"
-                  message:
-                    expr: "`missing searched headline: ${searchText}`"
-      detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live OpenAI web search probe' : searchText"
+        - call: reset
+        - set: selected
+          value:
+            expr: splitModelRef(env.primaryModel)
+        - call: runAgentPrompt
+          args:
+            - ref: env
+            - sessionKey: agent:qa:openai-native-web-search
+              message:
+                expr: config.searchPrompt
+              provider:
+                expr: selected?.provider
+              model:
+                expr: selected?.model
+              timeoutMs:
+                expr: resolveQaLiveTurnTimeoutMs(env, 180000, env.primaryModel)
+        - call: waitForOutboundMessage
+          saveAs: searchOutbound
+          args:
+            - ref: state
+            - lambda:
+                params: [candidate]
+                expr: "candidate.conversation.id === 'qa-operator'"
+            - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
+        - set: searchText
+          value:
+            expr: searchOutbound.text
+        - set: searchTextLower
+          value:
+            expr: normalizeLowercaseStringOrEmpty(searchText)
+        - assert:
+            expr: "searchText.includes(config.expectedMarker)"
+            message:
+              expr: "`missing ${config.expectedMarker}: ${searchText}`"
+        - assert:
+            expr: "!searchText.includes(config.failureMarker) && !/(web search is unavailable|unable to search|cannot search|can't search)/i.test(searchText)"
+            message:
+              expr: "`search looked unavailable: ${searchText}`"
+        - assert:
+            expr: "/URL:\\s*https?:\\/\\/[^\\s]*openai\\.com\\/news/i.test(searchText)"
+            message:
+              expr: "`missing official OpenAI News URL: ${searchText}`"
+        - assert:
+            expr: "/HEADLINE:\\s*\\S.{8,}/i.test(searchText)"
+            message:
+              expr: "`missing searched headline: ${searchText}`"
+      detailsExpr: searchText