From b3aab44f4bcba452eae895d943d88a8b66f534cc Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 4 Aug 2026 22:52:57 -0700 Subject: [PATCH] fix(qa): keep live provider scenarios out of mock lanes (#119498) --- .../scenario-catalog-anthropic-live.test.ts | 47 ++++++++ .../src/scenario-catalog.openai-live.test.ts | 26 ++++ .../models/anthropic-opus-api-key-smoke.yaml | 76 ++++++------ .../anthropic-opus-setup-token-smoke.yaml | 78 ++++++------ .../models/openai-native-web-search-live.yaml | 112 +++++++++--------- 5 files changed, 206 insertions(+), 133 deletions(-) create mode 100644 extensions/qa-lab/src/scenario-catalog-anthropic-live.test.ts diff --git a/extensions/qa-lab/src/scenario-catalog-anthropic-live.test.ts b/extensions/qa-lab/src/scenario-catalog-anthropic-live.test.ts new file mode 100644 index 000000000000..3f3fa6ec8c6d --- /dev/null +++ b/extensions/qa-lab/src/scenario-catalog-anthropic-live.test.ts @@ -0,0 +1,47 @@ +// Qa Lab tests keep Anthropic provider smoke evidence on real model lanes. +import { describe, expect, it } from "vitest"; +import { readQaScenarioById, readQaScenarioExecutionConfig } from "./scenario-catalog.js"; +import { selectQaFlowSuiteScenarios } from "./suite-planning.js"; + +const ANTHROPIC_OPUS_SCENARIO_IDS = [ + "anthropic-opus-api-key-smoke", + "anthropic-opus-setup-token-smoke", +] as const; + +describe("QA Anthropic live scenario catalog", () => { + it.each(ANTHROPIC_OPUS_SCENARIO_IDS)("pins %s to live Anthropic Opus", (scenarioId) => { + const scenario = readQaScenarioById(scenarioId); + + expect(readQaScenarioExecutionConfig(scenarioId)).toMatchObject({ + requiredProviderMode: "live-frontier", + requiredProvider: "anthropic", + requiredModel: "claude-opus-5", + }); + expect(scenario.execution.flow?.steps.at(-1)?.actions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ call: "runAgentPrompt" }), + expect.objectContaining({ call: "waitForOutboundMessage" }), + ]), + ); + }); + + it.each(ANTHROPIC_OPUS_SCENARIO_IDS)( + "rejects %s from matching-provider mock lanes", + (scenarioId) => { + const scenario = readQaScenarioById(scenarioId); + const mockLane = { + scenarios: [scenario], + providerMode: "mock-openai" as const, + primaryModel: "anthropic/claude-opus-5", + }; + + expect(selectQaFlowSuiteScenarios({ ...mockLane, providerMode: "live-frontier" })).toEqual([ + scenario, + ]); + expect(selectQaFlowSuiteScenarios(mockLane)).toEqual([]); + expect(() => selectQaFlowSuiteScenarios({ ...mockLane, scenarioIds: [scenario.id] })).toThrow( + "providerMode=live-frontier", + ); + }, + ); +}); diff --git a/extensions/qa-lab/src/scenario-catalog.openai-live.test.ts b/extensions/qa-lab/src/scenario-catalog.openai-live.test.ts index 9c133a9732c7..63067cb26f23 100644 --- a/extensions/qa-lab/src/scenario-catalog.openai-live.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.openai-live.test.ts @@ -1,6 +1,7 @@ // Qa Lab tests cover live OpenAI scenario catalog metadata. import { describe, expect, it } from "vitest"; import { readQaScenarioById, readQaScenarioExecutionConfig } from "./scenario-catalog.js"; +import { selectQaFlowSuiteScenarios } from "./suite-planning.js"; describe("qa scenario catalog", () => { it("includes the GPT-5.6 Luna thinking visibility switch scenario", () => { @@ -36,6 +37,7 @@ describe("qa scenario catalog", () => { const scenario = readQaScenarioById("openai-native-web-search-live"); const config = readQaScenarioExecutionConfig("openai-native-web-search-live") as | { + requiredProviderMode?: string; requiredProvider?: string; requiredModel?: string; expectedMarker?: string; @@ -51,6 +53,7 @@ describe("qa scenario catalog", () => { }, }, }); + expect(config?.requiredProviderMode).toBe("live-frontier"); expect(config?.requiredProvider).toBe("openai"); expect(config?.requiredModel).toBe("gpt-5.6-luna"); expect(config?.expectedMarker).toBe("WEB-SEARCH-OK"); @@ -58,6 +61,29 @@ describe("qa scenario catalog", () => { "confirms live OpenAI GPT-5.6 Luna web search auto mode", "searches official OpenAI News through the live model", ]); + expect(scenario.execution.flow?.steps.at(-1)?.actions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ call: "runAgentPrompt" }), + expect.objectContaining({ call: "waitForOutboundMessage" }), + ]), + ); + }); + + it("rejects the native web-search probe from matching-provider mock lanes", () => { + const scenario = readQaScenarioById("openai-native-web-search-live"); + const mockLane = { + scenarios: [scenario], + providerMode: "mock-openai" as const, + primaryModel: "openai/gpt-5.6-luna", + }; + + expect(selectQaFlowSuiteScenarios({ ...mockLane, providerMode: "live-frontier" })).toEqual([ + scenario, + ]); + expect(selectQaFlowSuiteScenarios(mockLane)).toEqual([]); + expect(() => selectQaFlowSuiteScenarios({ ...mockLane, scenarioIds: [scenario.id] })).toThrow( + "providerMode=live-frontier", + ); }); it("includes the live inbound voice talkback scenario", () => { diff --git a/qa/scenarios/models/anthropic-opus-api-key-smoke.yaml b/qa/scenarios/models/anthropic-opus-api-key-smoke.yaml index efa89029b97e..d04412af8815 100644 --- a/qa/scenarios/models/anthropic-opus-api-key-smoke.yaml +++ b/qa/scenarios/models/anthropic-opus-api-key-smoke.yaml @@ -25,6 +25,7 @@ scenario: kind: flow summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --model anthropic/claude-opus-5 --alt-model anthropic/claude-opus-5 --scenario anthropic-opus-api-key-smoke`. config: + requiredProviderMode: live-frontier requiredProvider: anthropic requiredModel: claude-opus-5 chatPrompt: "Anthropic Opus API key smoke. Reply exactly: ANTHROPIC-OPUS-API-KEY-OK" @@ -38,50 +39,49 @@ flow: value: expr: splitModelRef(env.primaryModel) - assert: - expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider" + expr: "env.providerMode === config.requiredProviderMode" + message: + expr: "`expected provider mode ${config.requiredProviderMode}, got ${env.providerMode}`" + - assert: + expr: "selected?.provider === config.requiredProvider" message: expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`" - assert: - expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel" + expr: "selected?.model === config.requiredModel" message: expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`" - assert: - expr: "env.providerMode !== 'live-frontier' || Boolean(env.gateway.runtimeEnv.ANTHROPIC_API_KEY?.trim())" + expr: "Boolean(env.gateway.runtimeEnv.ANTHROPIC_API_KEY?.trim())" message: expected ANTHROPIC_API_KEY to be available for API-key QA mode - detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} auth=env-api-key` : `mock-compatible provider=${selected?.provider}`" + detailsExpr: "`provider=${selected?.provider} model=${selected?.model} auth=env-api-key`" - name: talks through regular Anthropic Opus actions: - - if: - expr: "env.providerMode !== 'live-frontier'" - then: - - assert: "true" - else: - - call: reset - - set: selected - value: - expr: splitModelRef(env.primaryModel) - - call: runAgentPrompt - args: - - ref: env - - sessionKey: agent:qa:anthropic-opus-api-key - message: - expr: config.chatPrompt - provider: - expr: selected?.provider - model: - expr: selected?.model - timeoutMs: - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel) - - call: waitForOutboundMessage - saveAs: chatOutbound - args: - - ref: state - - lambda: - params: [candidate] - expr: "candidate.conversation.id === 'qa-operator'" - - expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel) - - assert: - expr: "chatOutbound.text.includes(config.chatExpected)" - message: - expr: "`chat marker missing: ${chatOutbound.text}`" - detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live Anthropic smoke' : chatOutbound.text" + - call: reset + - set: selected + value: + expr: splitModelRef(env.primaryModel) + - call: runAgentPrompt + args: + - ref: env + - sessionKey: agent:qa:anthropic-opus-api-key + message: + expr: config.chatPrompt + provider: + expr: selected?.provider + model: + expr: selected?.model + timeoutMs: + expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel) + - call: waitForOutboundMessage + saveAs: chatOutbound + args: + - ref: state + - lambda: + params: [candidate] + expr: "candidate.conversation.id === 'qa-operator'" + - expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel) + - assert: + expr: "chatOutbound.text.includes(config.chatExpected)" + message: + expr: "`chat marker missing: ${chatOutbound.text}`" + detailsExpr: chatOutbound.text diff --git a/qa/scenarios/models/anthropic-opus-setup-token-smoke.yaml b/qa/scenarios/models/anthropic-opus-setup-token-smoke.yaml index 5c5479696983..01ce41496cc4 100644 --- a/qa/scenarios/models/anthropic-opus-setup-token-smoke.yaml +++ b/qa/scenarios/models/anthropic-opus-setup-token-smoke.yaml @@ -25,6 +25,7 @@ scenario: kind: flow summary: Run with `OPENCLAW_LIVE_SETUP_TOKEN_VALUE= pnpm openclaw qa suite --provider-mode live-frontier --model anthropic/claude-opus-5 --alt-model anthropic/claude-opus-5 --scenario anthropic-opus-setup-token-smoke`. config: + requiredProviderMode: live-frontier requiredProvider: anthropic requiredModel: claude-opus-5 profileId: "anthropic:qa-setup-token" @@ -39,54 +40,53 @@ flow: value: expr: splitModelRef(env.primaryModel) - assert: - expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider" + expr: "env.providerMode === config.requiredProviderMode" + message: + expr: "`expected provider mode ${config.requiredProviderMode}, got ${env.providerMode}`" + - assert: + expr: "selected?.provider === config.requiredProvider" message: expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`" - assert: - expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel" + expr: "selected?.model === config.requiredModel" message: expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`" - assert: - expr: "env.providerMode !== 'live-frontier' || env.gateway.cfg.auth?.profiles?.[config.profileId]?.mode === 'token'" + expr: "env.gateway.cfg.auth?.profiles?.[config.profileId]?.mode === 'token'" message: expr: "`expected token profile ${config.profileId} in QA config`" - assert: - expr: "env.providerMode !== 'live-frontier' || !env.gateway.runtimeEnv.OPENCLAW_LIVE_SETUP_TOKEN_VALUE" + expr: "!env.gateway.runtimeEnv.OPENCLAW_LIVE_SETUP_TOKEN_VALUE" message: setup-token value should not be passed to the gateway child env - detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} auth=setup-token profile=${config.profileId}` : `mock-compatible provider=${selected?.provider}`" + detailsExpr: "`provider=${selected?.provider} model=${selected?.model} auth=setup-token profile=${config.profileId}`" - name: talks through regular Anthropic Opus actions: - - if: - expr: "env.providerMode !== 'live-frontier'" - then: - - assert: "true" - else: - - call: reset - - set: selected - value: - expr: splitModelRef(env.primaryModel) - - call: runAgentPrompt - args: - - ref: env - - sessionKey: agent:qa:anthropic-opus-setup-token - message: - expr: config.chatPrompt - provider: - expr: selected?.provider - model: - expr: selected?.model - timeoutMs: - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel) - - call: waitForOutboundMessage - saveAs: chatOutbound - args: - - ref: state - - lambda: - params: [candidate] - expr: "candidate.conversation.id === 'qa-operator'" - - expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel) - - assert: - expr: "chatOutbound.text.includes(config.chatExpected)" - message: - expr: "`chat marker missing: ${chatOutbound.text}`" - detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live Anthropic smoke' : chatOutbound.text" + - call: reset + - set: selected + value: + expr: splitModelRef(env.primaryModel) + - call: runAgentPrompt + args: + - ref: env + - sessionKey: agent:qa:anthropic-opus-setup-token + message: + expr: config.chatPrompt + provider: + expr: selected?.provider + model: + expr: selected?.model + timeoutMs: + expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel) + - call: waitForOutboundMessage + saveAs: chatOutbound + args: + - ref: state + - lambda: + params: [candidate] + expr: "candidate.conversation.id === 'qa-operator'" + - expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel) + - assert: + expr: "chatOutbound.text.includes(config.chatExpected)" + message: + expr: "`chat marker missing: ${chatOutbound.text}`" + detailsExpr: chatOutbound.text diff --git a/qa/scenarios/models/openai-native-web-search-live.yaml b/qa/scenarios/models/openai-native-web-search-live.yaml index 0209eb0855cd..49788c080bb8 100644 --- a/qa/scenarios/models/openai-native-web-search-live.yaml +++ b/qa/scenarios/models/openai-native-web-search-live.yaml @@ -33,6 +33,7 @@ scenario: kind: flow summary: Run with `OPENCLAW_LIVE_OPENAI_KEY="${OPENAI_API_KEY}" pnpm openclaw qa suite --provider-mode live-frontier --model openai/gpt-5.6-luna --alt-model openai/gpt-5.6-luna --fast --thinking medium --scenario openai-native-web-search-live`. config: + requiredProviderMode: live-frontier requiredProvider: openai requiredModel: gpt-5.6-luna expectedMarker: WEB-SEARCH-OK @@ -61,11 +62,15 @@ flow: value: expr: splitModelRef(env.primaryModel) - assert: - expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider" + expr: "env.providerMode === config.requiredProviderMode" + message: + expr: "`expected provider mode ${config.requiredProviderMode}, got ${env.providerMode}`" + - assert: + expr: "selected?.provider === config.requiredProvider" message: expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`" - assert: - expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel" + expr: "selected?.model === config.requiredModel" message: expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`" - call: readConfigSnapshot @@ -84,60 +89,55 @@ flow: message: expr: "`expected web search provider auto/openai/unset for native OpenAI search, got ${JSON.stringify(searchConfig)}`" - assert: - expr: "env.providerMode !== 'live-frontier' || Boolean(env.gateway.runtimeEnv.OPENAI_API_KEY?.trim() || env.gateway.runtimeEnv.OPENCLAW_LIVE_OPENAI_KEY?.trim())" + expr: "Boolean(env.gateway.runtimeEnv.OPENAI_API_KEY?.trim() || env.gateway.runtimeEnv.OPENCLAW_LIVE_OPENAI_KEY?.trim())" message: expected OPENAI_API_KEY or OPENCLAW_LIVE_OPENAI_KEY for live OpenAI QA - detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} webSearch=${JSON.stringify(searchConfig)}` : `mock-compatible provider=${selected?.provider}`" + detailsExpr: "`provider=${selected?.provider} model=${selected?.model} webSearch=${JSON.stringify(searchConfig)}`" - name: searches official OpenAI News through the live model actions: - - if: - expr: "env.providerMode !== 'live-frontier'" - then: - - assert: "true" - else: - - call: reset - - set: selected - value: - expr: splitModelRef(env.primaryModel) - - call: runAgentPrompt - args: - - ref: env - - sessionKey: agent:qa:openai-native-web-search - message: - expr: config.searchPrompt - provider: - expr: selected?.provider - model: - expr: selected?.model - timeoutMs: - expr: resolveQaLiveTurnTimeoutMs(env, 180000, env.primaryModel) - - call: waitForOutboundMessage - saveAs: searchOutbound - args: - - ref: state - - lambda: - params: [candidate] - expr: "candidate.conversation.id === 'qa-operator'" - - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel) - - set: searchText - value: - expr: searchOutbound.text - - set: searchTextLower - value: - expr: normalizeLowercaseStringOrEmpty(searchText) - - assert: - expr: "searchText.includes(config.expectedMarker)" - message: - expr: "`missing ${config.expectedMarker}: ${searchText}`" - - assert: - expr: "!searchText.includes(config.failureMarker) && !/(web search is unavailable|unable to search|cannot search|can't search)/i.test(searchText)" - message: - expr: "`search looked unavailable: ${searchText}`" - - assert: - expr: "/URL:\\s*https?:\\/\\/[^\\s]*openai\\.com\\/news/i.test(searchText)" - message: - expr: "`missing official OpenAI News URL: ${searchText}`" - - assert: - expr: "/HEADLINE:\\s*\\S.{8,}/i.test(searchText)" - message: - expr: "`missing searched headline: ${searchText}`" - detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live OpenAI web search probe' : searchText" + - call: reset + - set: selected + value: + expr: splitModelRef(env.primaryModel) + - call: runAgentPrompt + args: + - ref: env + - sessionKey: agent:qa:openai-native-web-search + message: + expr: config.searchPrompt + provider: + expr: selected?.provider + model: + expr: selected?.model + timeoutMs: + expr: resolveQaLiveTurnTimeoutMs(env, 180000, env.primaryModel) + - call: waitForOutboundMessage + saveAs: searchOutbound + args: + - ref: state + - lambda: + params: [candidate] + expr: "candidate.conversation.id === 'qa-operator'" + - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel) + - set: searchText + value: + expr: searchOutbound.text + - set: searchTextLower + value: + expr: normalizeLowercaseStringOrEmpty(searchText) + - assert: + expr: "searchText.includes(config.expectedMarker)" + message: + expr: "`missing ${config.expectedMarker}: ${searchText}`" + - assert: + expr: "!searchText.includes(config.failureMarker) && !/(web search is unavailable|unable to search|cannot search|can't search)/i.test(searchText)" + message: + expr: "`search looked unavailable: ${searchText}`" + - assert: + expr: "/URL:\\s*https?:\\/\\/[^\\s]*openai\\.com\\/news/i.test(searchText)" + message: + expr: "`missing official OpenAI News URL: ${searchText}`" + - assert: + expr: "/HEADLINE:\\s*\\S.{8,}/i.test(searchText)" + message: + expr: "`missing searched headline: ${searchText}`" + detailsExpr: searchText