diff --git a/extensions/qa-lab/src/scenario-catalog-compaction.test.ts b/extensions/qa-lab/src/scenario-catalog-compaction.test.ts index e6a8d5cde1c5..3ba872955188 100644 --- a/extensions/qa-lab/src/scenario-catalog-compaction.test.ts +++ b/extensions/qa-lab/src/scenario-catalog-compaction.test.ts @@ -27,7 +27,7 @@ describe("qa compaction scenario catalog", () => { expect(scenario.gatewayConfigPatch).toMatchObject({ agents: { defaults: { compaction: { mode: "default" } } }, }); - expect(flow).toContain("OPENCLAW_QA_FORCE_RUNTIME === 'openclaw'"); + expect(flow).toContain("env.runtimeId === 'openclaw'"); expect(flow).toContain("initialRequests[0].errorCode === 'context_length_exceeded'"); expect(flow).toContain("initialRequests.length === 2"); expect(flow).toContain("compactionSummaryRequests.length === 2"); @@ -85,7 +85,7 @@ describe("qa compaction scenario catalog", () => { | undefined; expect(firstAction).toBeDefined(); const conditional = firstAction?.if as Record | undefined; - expect(conditional?.expr).toBe("env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === 'codex'"); + expect(conditional?.expr).toBe("env.runtimeId === 'codex'"); expect(conditional?.["then"]).toMatchObject([ { call: "qaImport", @@ -102,9 +102,7 @@ describe("qa compaction scenario catalog", () => { | Record | undefined; expect(runtimeGuard?.assert).toMatchObject({ - expr: expect.stringContaining( - "(env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME ?? 'openclaw') === 'openclaw'", - ), + expr: expect.stringContaining("env.runtimeId === 'openclaw'"), }); const knownGapIndex = flow.indexOf(knownGap); @@ -115,7 +113,7 @@ describe("qa compaction scenario catalog", () => { expect(flow).toContain("new qaErrors.QaSuiteScenarioSkipError"); expect(flow).toContain("seedQaSessionTranscript"); expect(flow).toContain("sessions.compaction.branch"); - expect(flow).toContain("OPENCLAW_QA_FORCE_RUNTIME"); + expect(flow).toContain("env.runtimeId"); expect(flow).toContain('"transcriptToolName":"write"'); expect(flow).toContain('"requireSuccessfulTranscriptToolResult":true'); expect(flow).toContain("outbound.text === config.finalMarker"); diff --git a/extensions/qa-lab/src/scenario-catalog.runtime-pair.test.ts b/extensions/qa-lab/src/scenario-catalog.runtime-pair.test.ts index e7a2fbb195e3..92335088f271 100644 --- a/extensions/qa-lab/src/scenario-catalog.runtime-pair.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.runtime-pair.test.ts @@ -79,7 +79,7 @@ describe("QA runtime-pair scenario catalog", () => { const longContextScenario = readQaScenarioById("long-context-progress-watchdog"); expect(longContextScenario.execution).toMatchObject({ kind: "flow", runtime: "codex" }); const longContextFlow = JSON.stringify(longContextScenario.execution.flow); - expect(longContextFlow).toContain("OPENCLAW_QA_FORCE_RUNTIME"); + expect(longContextFlow).toContain("env.runtimeId"); expect(longContextFlow).not.toContain("patchConfig"); }); }); diff --git a/extensions/qa-lab/src/suite-run-standard.parity-retry.test.ts b/extensions/qa-lab/src/suite-run-standard.parity-retry.test.ts index c96408dee5a3..60e488af39d8 100644 --- a/extensions/qa-lab/src/suite-run-standard.parity-retry.test.ts +++ b/extensions/qa-lab/src/suite-run-standard.parity-retry.test.ts @@ -131,6 +131,27 @@ beforeEach(() => { }); describe("QA runtime parity scenario retry isolation", () => { + it.each([ + { forcedRuntime: undefined, expectedRuntime: "openclaw" }, + { forcedRuntime: "codex" as const, expectedRuntime: "codex" }, + ])( + "records $expectedRuntime as the selected runtime fact", + async ({ forcedRuntime, expectedRuntime }) => { + const runScenario = vi.fn().mockImplementation(async (env) => { + expect(env.runtimeId).toBe(expectedRuntime); + return makeRetryTestResult("pass"); + }); + + await runQaFlowSuiteStandard( + { lab: makeRetryTestLab(), ...(forcedRuntime ? { forcedRuntime } : {}) }, + makeRetryTestContext(), + runScenario, + ); + + expect(runScenario).toHaveBeenCalledOnce(); + }, + ); + it("skips connected-transport readiness for intentionally unhealthy startup", async () => { const context = makeRetryTestContext(); context.gatewayRuntimeOptions = { allowUnhealthyStartup: true }; diff --git a/extensions/qa-lab/src/suite-run-standard.ts b/extensions/qa-lab/src/suite-run-standard.ts index e1afad5595e5..3480b0a1e794 100644 --- a/extensions/qa-lab/src/suite-run-standard.ts +++ b/extensions/qa-lab/src/suite-run-standard.ts @@ -152,6 +152,7 @@ export async function runQaFlowSuiteStandard( lab, mock: activeMock, gateway: activeGateway, + runtimeId: params?.forcedRuntime ?? "openclaw", outputDir, // YAML scenarios should see the full staged gateway config, not just // the transport fragment. Routing/session/plugin assertions depend on it. diff --git a/extensions/qa-lab/src/suite-types.ts b/extensions/qa-lab/src/suite-types.ts index c0627dea653e..58cdeeb0cc9a 100644 --- a/extensions/qa-lab/src/suite-types.ts +++ b/extensions/qa-lab/src/suite-types.ts @@ -26,6 +26,7 @@ export type QaSuiteScenarioResult = { export type QaSuiteEnvironment = { lab: QaLabServerHandle; + runtimeId: RuntimeId; webSessionIds: Set; } & QaSuiteRuntimeEnv; diff --git a/extensions/qa-lab/src/suite.ts b/extensions/qa-lab/src/suite.ts index 575c67662bef..af5f4fb2d619 100644 --- a/extensions/qa-lab/src/suite.ts +++ b/extensions/qa-lab/src/suite.ts @@ -74,6 +74,7 @@ export type QaSuiteScenarioResult = { type QaSuiteEnvironment = { lab: QaLabServerHandle; + runtimeId: RuntimeId; webSessionIds: Set; } & QaSuiteRuntimeEnv; diff --git a/qa/scenarios/models/codex-harness-no-meta-leak.yaml b/qa/scenarios/models/codex-harness-no-meta-leak.yaml index 4622251f32cf..e4624e088ae0 100644 --- a/qa/scenarios/models/codex-harness-no-meta-leak.yaml +++ b/qa/scenarios/models/codex-harness-no-meta-leak.yaml @@ -65,10 +65,10 @@ flow: message: expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`" - assert: - expr: "env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === config.harnessRuntime" + expr: "env.runtimeId === config.harnessRuntime" message: - expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`" - detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`" + expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.runtimeId}`" + detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${env.runtimeId}`" - name: keeps codex coordination chatter out of the visible reply actions: - call: reset diff --git a/qa/scenarios/runtime/compaction-empty-response-recovery.yaml b/qa/scenarios/runtime/compaction-empty-response-recovery.yaml index 4e05a3e35a56..e19189ea085e 100644 --- a/qa/scenarios/runtime/compaction-empty-response-recovery.yaml +++ b/qa/scenarios/runtime/compaction-empty-response-recovery.yaml @@ -39,7 +39,7 @@ flow: - name: retries an empty compaction summary at the compaction owner actions: - assert: - expr: "env.providerMode === 'mock-openai' && env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === 'openclaw' && Boolean(env.mock)" + expr: "env.providerMode === 'mock-openai' && env.runtimeId === 'openclaw' && Boolean(env.mock)" message: compaction output recovery requires the mock-openai OpenClaw runtime - call: waitForGatewayHealthy args: diff --git a/qa/scenarios/runtime/compaction-reasoning-only-recovery.yaml b/qa/scenarios/runtime/compaction-reasoning-only-recovery.yaml index 16d38baf8b1f..dd2dc9b059e2 100644 --- a/qa/scenarios/runtime/compaction-reasoning-only-recovery.yaml +++ b/qa/scenarios/runtime/compaction-reasoning-only-recovery.yaml @@ -39,7 +39,7 @@ flow: - name: retries a reasoning-only compaction summary at the compaction owner actions: - assert: - expr: "env.providerMode === 'mock-openai' && env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === 'openclaw' && Boolean(env.mock)" + expr: "env.providerMode === 'mock-openai' && env.runtimeId === 'openclaw' && Boolean(env.mock)" message: compaction output recovery requires the mock-openai OpenClaw runtime - call: waitForGatewayHealthy args: diff --git a/qa/scenarios/runtime/compaction-retry-mutating-tool.yaml b/qa/scenarios/runtime/compaction-retry-mutating-tool.yaml index c2e91a2e000e..d1de13188cc8 100644 --- a/qa/scenarios/runtime/compaction-retry-mutating-tool.yaml +++ b/qa/scenarios/runtime/compaction-retry-mutating-tool.yaml @@ -48,7 +48,7 @@ flow: - name: compacts bulky history and mutates exactly once after overflow actions: - if: - expr: "env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === 'codex'" + expr: "env.runtimeId === 'codex'" then: - call: qaImport args: @@ -57,7 +57,7 @@ flow: - throw: expr: "new qaErrors.QaSuiteScenarioSkipError('known-harness-gap compaction-retry-mutating-tool: provider-error recovery does not invoke Codex native compaction; native token-threshold compaction needs a separate scenario.')" - assert: - expr: "(env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME ?? 'openclaw') === 'openclaw' && env.providerMode === 'mock-openai' && Boolean(env.mock)" + expr: "env.runtimeId === 'openclaw' && env.providerMode === 'mock-openai' && Boolean(env.mock)" message: compaction overflow injection requires the mock-openai OpenClaw runtime - call: waitForGatewayHealthy args: diff --git a/qa/scenarios/runtime/long-context-progress-watchdog.yaml b/qa/scenarios/runtime/long-context-progress-watchdog.yaml index 7d2d60357a0e..b494c7ecdfbe 100644 --- a/qa/scenarios/runtime/long-context-progress-watchdog.yaml +++ b/qa/scenarios/runtime/long-context-progress-watchdog.yaml @@ -44,9 +44,9 @@ flow: - ref: env - 60000 - assert: - expr: "env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === config.harnessRuntime" + expr: "env.runtimeId === config.harnessRuntime" message: - expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`" + expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.runtimeId}`" - call: reset - set: logCursor value: diff --git a/qa/scenarios/workspace/medium-game-plan-codex-harness.yaml b/qa/scenarios/workspace/medium-game-plan-codex-harness.yaml index 448591863bd1..60c642241753 100644 --- a/qa/scenarios/workspace/medium-game-plan-codex-harness.yaml +++ b/qa/scenarios/workspace/medium-game-plan-codex-harness.yaml @@ -68,10 +68,10 @@ flow: message: expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`" - assert: - expr: "env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === config.harnessRuntime" + expr: "env.runtimeId === config.harnessRuntime" message: - expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`" - detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`" + expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.runtimeId}`" + detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${env.runtimeId}`" - name: builds the medium game artifact actions: - call: reset diff --git a/qa/scenarios/workspace/medium-game-plan-openclaw-harness.yaml b/qa/scenarios/workspace/medium-game-plan-openclaw-harness.yaml index 609cf28e656c..c916b423e5b6 100644 --- a/qa/scenarios/workspace/medium-game-plan-openclaw-harness.yaml +++ b/qa/scenarios/workspace/medium-game-plan-openclaw-harness.yaml @@ -66,10 +66,10 @@ flow: message: expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`" - assert: - expr: "env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === config.harnessRuntime" + expr: "env.runtimeId === config.harnessRuntime" message: - expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`" - detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`" + expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.runtimeId}`" + detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${env.runtimeId}`" - name: builds the medium game artifact actions: - call: reset