ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(qa): carry selected runtime into flow scenarios (#119324)

Browse Source 
* fix(qa): carry selected runtime into flow scenarios

* test(qa): align compaction runtime fact assertion
This commit is contained in:
Peter Steinberger
2026-08-04 13:51:35 -07:00
committed by GitHub
parent 532580fa2b
commit f15120c6d1
13 changed files with 44 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/qa-lab/src/scenario-catalog-compaction.test.ts
+4 -6
View File
@@ -27,7 +27,7 @@ describe("qa compaction scenario catalog", () => {
expect(scenario.gatewayConfigPatch).toMatchObject({
agents: { defaults: { compaction: { mode: "default" } } },
});
expect(flow).toContain("OPENCLAW_QA_FORCE_RUNTIME === 'openclaw'");
expect(flow).toContain("env.runtimeId === 'openclaw'");
expect(flow).toContain("initialRequests[0].errorCode === 'context_length_exceeded'");
expect(flow).toContain("initialRequests.length === 2");
expect(flow).toContain("compactionSummaryRequests.length === 2");
@@ -85,7 +85,7 @@ describe("qa compaction scenario catalog", () => {
| undefined;
expect(firstAction).toBeDefined();
const conditional = firstAction?.if as Record<string, unknown> | undefined;
expect(conditional?.expr).toBe("env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === 'codex'");
expect(conditional?.expr).toBe("env.runtimeId === 'codex'");
expect(conditional?.["then"]).toMatchObject([
{
call: "qaImport",
@@ -102,9 +102,7 @@ describe("qa compaction scenario catalog", () => {
| Record<string, unknown>
| undefined;
expect(runtimeGuard?.assert).toMatchObject({
expr: expect.stringContaining(
"(env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME ?? 'openclaw') === 'openclaw'",
),
expr: expect.stringContaining("env.runtimeId === 'openclaw'"),
});
const knownGapIndex = flow.indexOf(knownGap);
@@ -115,7 +113,7 @@ describe("qa compaction scenario catalog", () => {
expect(flow).toContain("new qaErrors.QaSuiteScenarioSkipError");
expect(flow).toContain("seedQaSessionTranscript");
expect(flow).toContain("sessions.compaction.branch");
expect(flow).toContain("OPENCLAW_QA_FORCE_RUNTIME");
expect(flow).toContain("env.runtimeId");
expect(flow).toContain('"transcriptToolName":"write"');
expect(flow).toContain('"requireSuccessfulTranscriptToolResult":true');
expect(flow).toContain("outbound.text === config.finalMarker");
extensions/qa-lab/src/scenario-catalog.runtime-pair.test.ts
+1 -1
View File
@@ -79,7 +79,7 @@ describe("QA runtime-pair scenario catalog", () => {
const longContextScenario = readQaScenarioById("long-context-progress-watchdog");
expect(longContextScenario.execution).toMatchObject({ kind: "flow", runtime: "codex" });
const longContextFlow = JSON.stringify(longContextScenario.execution.flow);
expect(longContextFlow).toContain("OPENCLAW_QA_FORCE_RUNTIME");
expect(longContextFlow).toContain("env.runtimeId");
expect(longContextFlow).not.toContain("patchConfig");
});
});
extensions/qa-lab/src/suite-run-standard.parity-retry.test.ts
+21
View File
@@ -131,6 +131,27 @@ beforeEach(() => {
});
describe("QA runtime parity scenario retry isolation", () => {
it.each([
{ forcedRuntime: undefined, expectedRuntime: "openclaw" },
{ forcedRuntime: "codex" as const, expectedRuntime: "codex" },
])(
"records $expectedRuntime as the selected runtime fact",
async ({ forcedRuntime, expectedRuntime }) => {
const runScenario = vi.fn<QaSuiteScenarioRunner>().mockImplementation(async (env) => {
expect(env.runtimeId).toBe(expectedRuntime);
return makeRetryTestResult("pass");
});
await runQaFlowSuiteStandard(
{ lab: makeRetryTestLab(), ...(forcedRuntime ? { forcedRuntime } : {}) },
makeRetryTestContext(),
runScenario,
);
expect(runScenario).toHaveBeenCalledOnce();
},
);
it("skips connected-transport readiness for intentionally unhealthy startup", async () => {
const context = makeRetryTestContext();
context.gatewayRuntimeOptions = { allowUnhealthyStartup: true };
extensions/qa-lab/src/suite-run-standard.ts
+1
View File
@@ -152,6 +152,7 @@ export async function runQaFlowSuiteStandard(
lab,
mock: activeMock,
gateway: activeGateway,
runtimeId: params?.forcedRuntime ?? "openclaw",
outputDir,
// YAML scenarios should see the full staged gateway config, not just
// the transport fragment. Routing/session/plugin assertions depend on it.
extensions/qa-lab/src/suite-types.ts
+1
View File
@@ -26,6 +26,7 @@ export type QaSuiteScenarioResult = {
export type QaSuiteEnvironment = {
lab: QaLabServerHandle;
runtimeId: RuntimeId;
webSessionIds: Set<string>;
} & QaSuiteRuntimeEnv;
extensions/qa-lab/src/suite.ts
+1
View File
@@ -74,6 +74,7 @@ export type QaSuiteScenarioResult = {
type QaSuiteEnvironment = {
lab: QaLabServerHandle;
runtimeId: RuntimeId;
webSessionIds: Set<string>;
} & QaSuiteRuntimeEnv;
qa/scenarios/models/codex-harness-no-meta-leak.yaml
+3 -3
View File
@@ -65,10 +65,10 @@ flow:
message:
expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`"
- assert:
expr: "env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === config.harnessRuntime"
expr: "env.runtimeId === config.harnessRuntime"
message:
expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`"
detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`"
expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.runtimeId}`"
detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${env.runtimeId}`"
- name: keeps codex coordination chatter out of the visible reply
actions:
- call: reset
qa/scenarios/runtime/compaction-empty-response-recovery.yaml
+1 -1
View File
@@ -39,7 +39,7 @@ flow:
- name: retries an empty compaction summary at the compaction owner
actions:
- assert:
expr: "env.providerMode === 'mock-openai' && env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === 'openclaw' && Boolean(env.mock)"
expr: "env.providerMode === 'mock-openai' && env.runtimeId === 'openclaw' && Boolean(env.mock)"
message: compaction output recovery requires the mock-openai OpenClaw runtime
- call: waitForGatewayHealthy
args:
qa/scenarios/runtime/compaction-reasoning-only-recovery.yaml
+1 -1
View File
@@ -39,7 +39,7 @@ flow:
- name: retries a reasoning-only compaction summary at the compaction owner
actions:
- assert:
expr: "env.providerMode === 'mock-openai' && env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === 'openclaw' && Boolean(env.mock)"
expr: "env.providerMode === 'mock-openai' && env.runtimeId === 'openclaw' && Boolean(env.mock)"
message: compaction output recovery requires the mock-openai OpenClaw runtime
- call: waitForGatewayHealthy
args:
qa/scenarios/runtime/compaction-retry-mutating-tool.yaml
+2 -2
View File
@@ -48,7 +48,7 @@ flow:
- name: compacts bulky history and mutates exactly once after overflow
actions:
- if:
expr: "env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === 'codex'"
expr: "env.runtimeId === 'codex'"
then:
- call: qaImport
args:
@@ -57,7 +57,7 @@ flow:
- throw:
expr: "new qaErrors.QaSuiteScenarioSkipError('known-harness-gap compaction-retry-mutating-tool: provider-error recovery does not invoke Codex native compaction; native token-threshold compaction needs a separate scenario.')"
- assert:
expr: "(env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME ?? 'openclaw') === 'openclaw' && env.providerMode === 'mock-openai' && Boolean(env.mock)"
expr: "env.runtimeId === 'openclaw' && env.providerMode === 'mock-openai' && Boolean(env.mock)"
message: compaction overflow injection requires the mock-openai OpenClaw runtime
- call: waitForGatewayHealthy
args:
qa/scenarios/runtime/long-context-progress-watchdog.yaml
+2 -2
View File
@@ -44,9 +44,9 @@ flow:
- ref: env
- 60000
- assert:
expr: "env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === config.harnessRuntime"
expr: "env.runtimeId === config.harnessRuntime"
message:
expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`"
expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.runtimeId}`"
- call: reset
- set: logCursor
value:
qa/scenarios/workspace/medium-game-plan-codex-harness.yaml
+3 -3
View File
@@ -68,10 +68,10 @@ flow:
message:
expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`"
- assert:
expr: "env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === config.harnessRuntime"
expr: "env.runtimeId === config.harnessRuntime"
message:
expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`"
detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`"
expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.runtimeId}`"
detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${env.runtimeId}`"
- name: builds the medium game artifact
actions:
- call: reset
qa/scenarios/workspace/medium-game-plan-openclaw-harness.yaml
+3 -3
View File
@@ -66,10 +66,10 @@ flow:
message:
expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`"
- assert:
expr: "env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === config.harnessRuntime"
expr: "env.runtimeId === config.harnessRuntime"
message:
expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`"
detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`"
expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.runtimeId}`"
detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${env.runtimeId}`"
- name: builds the medium game artifact
actions:
- call: reset