diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index d17e013ce787..f033f86907f3 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -1530,7 +1530,6 @@ describe("qa cli runtime", () => { scenarioIds: [ "runtime-long-context-cache-stability", "runtime-soak-100-turn", - "runtime-tool-image-generate", "runtime-tool-memory-add", "runtime-tool-memory-recall", "runtime-tool-message-tool", @@ -1540,6 +1539,76 @@ describe("qa cli runtime", () => { "runtime-tool-tts", ], }); + expectWriteContains( + stderrWrite, + "excluded lane-incompatible scenario(s): runtime-tool-image-generate", + ); + }); + + it("keeps runtime-pair tier selection on flow scenarios and reports exclusions", async () => { + await runQaSuiteCommand({ + repoRoot: "/tmp/openclaw-repo", + runtimePair: "openclaw,codex", + runtimeParityTier: ["standard", "live-only"], + }); + + const scenarioIds = mockFirstObjectArg(runQaSuite).scenarioIds as string[]; + expect(scenarioIds).toContain("runtime-first-hour-20-turn"); + expect(scenarioIds).toContain("streaming-final-integrity"); + expect(scenarioIds).not.toContain("gateway-restart-inflight-run"); + expect(scenarioIds).not.toContain("hosted-image-generation-providers-live"); + expect(scenarioIds).not.toContain("hosted-video-generation-providers-live"); + expectFields(mockFirstObjectArg(runQaSuite), { + runtimePair: ["openclaw", "codex"], + }); + expectWriteContains( + stderrWrite, + "excluded incompatible non-flow scenario(s): hosted-image-generation-providers-live (script), hosted-video-generation-providers-live (script)", + ); + expectWriteContains( + stderrWrite, + "excluded lane-incompatible scenario(s): gateway-restart-inflight-run", + ); + }); + + it("rejects explicit runtime-pair scenarios with no compatible flow execution", async () => { + await expect( + runQaSuiteCommand({ + repoRoot: "/tmp/openclaw-repo", + runtimePair: "openclaw,codex", + scenarioIds: ["hosted-image-generation-providers-live"], + }), + ).rejects.toThrow( + "--runtime-pair requires execution.kind: flow scenarios; unsupported scenario(s): hosted-image-generation-providers-live (script)", + ); + + expect(runQaSuite).not.toHaveBeenCalled(); + }); + + it("rejects runtime-pair tiers with no compatible flow scenarios", async () => { + const catalog = readQaScenarioPack(); + const hostedImageScenario = catalog.scenarios.find( + (scenario) => scenario.id === "hosted-image-generation-providers-live", + ); + if (!hostedImageScenario) { + throw new Error("missing hosted image scenario fixture"); + } + readQaScenarioPack.mockReturnValueOnce({ + ...catalog, + scenarios: [hostedImageScenario], + }); + + await expect( + runQaSuiteCommand({ + repoRoot: "/tmp/openclaw-repo", + runtimePair: "openclaw,codex", + runtimeParityTier: ["live-only"], + }), + ).rejects.toThrow( + "--runtime-parity-tier matched no execution.kind: flow scenarios for live-only; incompatible scenario(s): hosted-image-generation-providers-live (script).", + ); + + expect(runQaSuite).not.toHaveBeenCalled(); }); it("rejects unknown runtime parity tier filters", async () => { diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts index 2e1b719edebf..d68130e92799 100644 --- a/extensions/qa-lab/src/cli.runtime.ts +++ b/extensions/qa-lab/src/cli.runtime.ts @@ -312,34 +312,85 @@ function parseQaRuntimeParityTierFilters(input: string[] | undefined): QaRuntime } function resolveQaRuntimeParityTierScenarioIds(params: { + channelDriver?: QaScorecardChannelDriver | null; + claudeCliAuthMode?: QaCliBackendAuthMode; + primaryModel: string; + providerMode: QaProviderMode; scenarioIds: string[]; runtimeParityTiers: readonly QaRuntimeParityTier[]; -}): string[] { + runtimePair: boolean; +}): { + scenarioIds: string[]; + excludedLaneScenarios: string[]; + excludedNonFlowScenarios: string[]; +} { if (params.runtimeParityTiers.length === 0) { - return params.scenarioIds; + return { + scenarioIds: params.scenarioIds, + excludedLaneScenarios: [], + excludedNonFlowScenarios: [], + }; } const tierSet = new Set(params.runtimeParityTiers); - const matchingScenarioIds = readQaScenarioPack() - .scenarios.filter( - (scenario) => scenario.runtimeParityTier && tierSet.has(scenario.runtimeParityTier), - ) - .map((scenario) => scenario.id); - if (matchingScenarioIds.length === 0) { + const matchingScenarios = readQaScenarioPack().scenarios.filter( + (scenario) => scenario.runtimeParityTier && tierSet.has(scenario.runtimeParityTier), + ); + if (matchingScenarios.length === 0) { throw new Error( `--runtime-parity-tier matched no scenarios for ${params.runtimeParityTiers.join(", ")}.`, ); } - return uniqueStrings([...params.scenarioIds, ...matchingScenarioIds]); + const compatibleScenarios = params.runtimePair + ? matchingScenarios.filter((scenario) => scenario.execution.kind === "flow") + : matchingScenarios; + const laneCompatibleScenarios = compatibleScenarios.filter((scenario) => + scenarioMatchesQaProviderLane({ + scenario, + providerMode: params.providerMode, + primaryModel: params.primaryModel, + channelDriver: params.channelDriver, + claudeCliAuthMode: params.claudeCliAuthMode, + }), + ); + const excludedLaneScenarios = compatibleScenarios + .filter((scenario) => !laneCompatibleScenarios.includes(scenario)) + .map((scenario) => scenario.id); + const excludedNonFlowScenarios = params.runtimePair + ? matchingScenarios + .filter((scenario) => scenario.execution.kind !== "flow") + .map((scenario) => `${scenario.id} (${scenario.execution.kind})`) + : []; + if (compatibleScenarios.length === 0) { + throw new Error( + `--runtime-parity-tier matched no execution.kind: flow scenarios for ${params.runtimeParityTiers.join(", ")}; incompatible scenario(s): ${excludedNonFlowScenarios.join(", ")}.`, + ); + } + if (params.scenarioIds.length === 0 && laneCompatibleScenarios.length === 0) { + throw new Error( + `--runtime-parity-tier matched no scenarios for provider mode ${params.providerMode}; incompatible scenario(s): ${excludedLaneScenarios.join(", ")}.`, + ); + } + return { + scenarioIds: uniqueStrings([ + ...params.scenarioIds, + ...laneCompatibleScenarios.map((scenario) => scenario.id), + ]), + excludedLaneScenarios, + excludedNonFlowScenarios, + }; } -function rejectNonFlowScenarioIdsForMultipass(scenarioIds: readonly string[]) { - if (scenarioIds.length === 0) { +function rejectNonFlowScenarioIds(params: { + option: "--runner multipass" | "--runtime-pair"; + scenarioIds: readonly string[]; +}) { + if (params.scenarioIds.length === 0) { return; } const scenarioById = new Map( readQaScenarioPack().scenarios.map((scenario) => [scenario.id, scenario]), ); - const nonFlowScenarios = scenarioIds.flatMap((scenarioId) => { + const nonFlowScenarios = params.scenarioIds.flatMap((scenarioId) => { const scenario = scenarioById.get(scenarioId); return scenario && scenario.execution.kind !== "flow" ? [`${scenario.id} (${scenario.execution.kind})`] @@ -347,7 +398,7 @@ function rejectNonFlowScenarioIdsForMultipass(scenarioIds: readonly string[]) { }); if (nonFlowScenarios.length > 0) { throw new Error( - `--runner multipass requires execution.kind: flow scenarios; unsupported scenario(s): ${nonFlowScenarios.join(", ")}`, + `${params.option} requires execution.kind: flow scenarios; unsupported scenario(s): ${nonFlowScenarios.join(", ")}`, ); } } @@ -881,6 +932,12 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) { const repoRoot = path.resolve(opts.repoRoot ?? process.cwd()); const transportId = normalizeQaTransportId(opts.transportId); const runner = (opts.runner ?? "host").trim().toLowerCase(); + const runtimePair = parseQaRuntimePair(opts.runtimePair); + const providerMode = normalizeQaProviderMode(opts.providerMode); + const claudeCliAuthMode = parseQaCliBackendAuthMode(opts.cliAuthMode); + const primaryModel = normalizeQaOptionalModelRef(opts.primaryModel); + const alternateModel = normalizeQaOptionalModelRef(opts.alternateModel); + const channelDriver = normalizeQaSuiteChannelDriver(opts.channelDriver); const explicitScenarioIds = resolveQaScenarioPackScenarioIds({ pack: opts.pack, scenarioIds: resolveQaParityPackScenarioIds({ @@ -889,17 +946,30 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) { }), }); const runtimeParityTiers = parseQaRuntimeParityTierFilters(opts.runtimeParityTier); - const scenarioIds = resolveQaRuntimeParityTierScenarioIds({ + const runtimeParityTierSelection = resolveQaRuntimeParityTierScenarioIds({ + channelDriver, + claudeCliAuthMode, + primaryModel: primaryModel ?? defaultQaModelForMode(providerMode), + providerMode, scenarioIds: explicitScenarioIds, runtimeParityTiers, + runtimePair: runtimePair !== undefined, }); + const scenarioIds = runtimeParityTierSelection.scenarioIds; + if (runtimePair) { + rejectNonFlowScenarioIds({ option: "--runtime-pair", scenarioIds }); + } + if (runtimeParityTierSelection.excludedNonFlowScenarios.length > 0) { + process.stderr.write( + `QA runtime-pair tier selection excluded incompatible non-flow scenario(s): ${runtimeParityTierSelection.excludedNonFlowScenarios.join(", ")}\n`, + ); + } + if (runtimeParityTierSelection.excludedLaneScenarios.length > 0) { + process.stderr.write( + `QA runtime-pair tier selection excluded lane-incompatible scenario(s): ${runtimeParityTierSelection.excludedLaneScenarios.join(", ")}\n`, + ); + } const allowFailures = opts.allowFailures === true; - const providerMode = normalizeQaProviderMode(opts.providerMode); - const runtimePair = parseQaRuntimePair(opts.runtimePair); - const claudeCliAuthMode = parseQaCliBackendAuthMode(opts.cliAuthMode); - const primaryModel = normalizeQaOptionalModelRef(opts.primaryModel); - const alternateModel = normalizeQaOptionalModelRef(opts.alternateModel); - const channelDriver = normalizeQaSuiteChannelDriver(opts.channelDriver); if (opts.channel?.trim() && channelDriver !== "crabline" && channelDriver !== "live") { throw new Error("--channel override requires --channel-driver crabline or live."); } @@ -959,7 +1029,7 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) { throw new Error("--runtime-pair is not supported with a live QA adapter."); } if (runner === "multipass") { - rejectNonFlowScenarioIdsForMultipass(scenarioIds); + rejectNonFlowScenarioIds({ option: "--runner multipass", scenarioIds }); const thinkingDefault = parseQaThinkingLevel("--thinking", opts.thinking); const result = await runQaMultipass({ repoRoot, diff --git a/extensions/qa-lab/src/gateway-child.ts b/extensions/qa-lab/src/gateway-child.ts index dee1a9656845..45032057552f 100644 --- a/extensions/qa-lab/src/gateway-child.ts +++ b/extensions/qa-lab/src/gateway-child.ts @@ -58,6 +58,7 @@ import { stageQaMockAuthProfiles } from "./providers/shared/mock-auth.js"; import { seedQaAgentWorkspace } from "./qa-agent-workspace.js"; import { buildQaGatewayConfig, type QaThinkingLevel } from "./qa-gateway-config.js"; import type { QaTransportAdapter } from "./qa-transport.js"; +import type { RuntimeId } from "./runtime-parity.js"; import { resolveQaWindowsSystem32ExePath } from "./windows-system-tools.js"; export type { QaCliBackendAuthMode } from "./providers/env.js"; @@ -780,6 +781,7 @@ export async function startQaGatewayChild(params: { alternateModel?: string; fastMode?: boolean; thinkingDefault?: QaThinkingLevel; + forcedRuntime?: RuntimeId; claudeCliAuthMode?: QaCliBackendAuthMode; controlUiEnabled?: boolean; enabledPluginIds?: string[]; diff --git a/extensions/qa-lab/src/qa-gateway-config.ts b/extensions/qa-lab/src/qa-gateway-config.ts index dff985ddc86b..19392af022b0 100644 --- a/extensions/qa-lab/src/qa-gateway-config.ts +++ b/extensions/qa-lab/src/qa-gateway-config.ts @@ -12,6 +12,7 @@ import { getQaProvider } from "./providers/index.js"; import { DEFAULT_QA_PROVIDER_MODE } from "./providers/index.js"; import type { QaThinkingLevel } from "./qa-thinking.js"; import type { QaTransportGatewayConfig } from "./qa-transport.js"; +import type { RuntimeId } from "./runtime-parity.js"; export { normalizeQaThinkingLevel, type QaThinkingLevel } from "./qa-thinking.js"; @@ -68,6 +69,7 @@ export function buildQaGatewayConfig(params: { liveProviderConfigs?: Record; fastMode?: boolean; thinkingDefault?: QaThinkingLevel; + forcedRuntime?: RuntimeId; }): OpenClawConfig { const providerBaseUrl = params.providerBaseUrl ?? "http://127.0.0.1:44080/v1"; const providerMode = normalizeQaProviderMode(params.providerMode ?? DEFAULT_QA_PROVIDER_MODE); @@ -135,12 +137,20 @@ export function buildQaGatewayConfig(params: { ...transportPluginIds, ]), ]; - const resolveModelParams = (modelRef: string) => - provider.resolveModelParams({ - modelRef, - fastMode: params.fastMode, - thinkingDefault: params.thinkingDefault, - }); + const resolveModelEntry = (modelRef: string) => { + // Codex owns its app-server transport. OpenClaw provider params would make + // the forced parity cell an authored route that Codex correctly rejects. + if (params.forcedRuntime === "codex") { + return {}; + } + return { + params: provider.resolveModelParams({ + modelRef, + fastMode: params.fastMode, + thinkingDefault: params.thinkingDefault, + }), + }; + }; const allowedOrigins = mergeQaControlUiAllowedOrigins(params.controlUiAllowedOrigins); const providerGatewayModels = provider.buildGatewayModels({ providerBaseUrl, @@ -210,12 +220,8 @@ export function buildQaGatewayConfig(params: { }, }, models: { - [primaryModel]: { - params: resolveModelParams(primaryModel), - }, - [alternateModel]: { - params: resolveModelParams(alternateModel), - }, + [primaryModel]: resolveModelEntry(primaryModel), + [alternateModel]: resolveModelEntry(alternateModel), }, subagents: { allowAgents: ["*"], @@ -227,6 +233,9 @@ export function buildQaGatewayConfig(params: { id: "qa", default: true, model: buildQaModelSelection(primaryModel, alternateModel), + ...(params.forcedRuntime === "codex" && params.fastMode !== undefined + ? { fastModeDefault: params.fastMode } + : {}), identity: { name: "C-3PO QA", theme: "Flustered Protocol Droid", diff --git a/extensions/qa-lab/src/suite.ts b/extensions/qa-lab/src/suite.ts index 955731f25932..21e1c333069f 100644 --- a/extensions/qa-lab/src/suite.ts +++ b/extensions/qa-lab/src/suite.ts @@ -1649,6 +1649,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise