ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-25 11:55:47 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(qa): isolate runtime parity readiness

Browse Source
This commit is contained in:
Dallin Romney
2026-08-17 08:23:55 -07:00
parent 89155e5331
commit 069cf88845
2 changed files with 72 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/qa-lab/src/suite-runtime-parity-runner.cleanup.test.ts
+39
View File
@@ -63,6 +63,27 @@ vi.mock("openclaw/plugin-sdk/agent-harness", () => ({
vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
fetchWithSsrFGuard: mocks.fetchWithSsrFGuard,
}));
vi.mock("./crabline-transport.js", () => ({
createQaCrablineTransportAdapter: vi.fn(async () => ({
id: "telegram",
label: "Crabline Telegram",
accountId: "sut",
requiredPluginIds: [],
supportedActions: [],
sendInbound: vi.fn(async () => {}),
createGatewayConfig: () => ({}),
waitReady: vi.fn(async () => {}),
buildAgentDelivery: ({ target }: { target: string }) => ({
channel: "telegram",
to: target,
replyChannel: "telegram",
replyTo: target,
}),
handleAction: vi.fn(async () => {}),
createReportNotes: () => [],
cleanup: vi.fn(async () => {}),
})),
}));
vi.mock("./gateway-child.js", () => ({
startQaGatewayChild: mocks.startQaGatewayChild,
}));
@@ -254,7 +275,14 @@ describe("runtime parity suite transport cleanup", () => {
});
it("prints one generic completion after real nested standard cells and parent cleanup", async () => {
mocks.writeQaSuiteArtifacts.mockClear();
const scenario = makeQaSuiteTestScenario("runtime-cleanup");
const selection = {
capabilityMatrixPath: "crabline-fake-provider-capabilities.json",
channel: "telegram",
channelDriver: "crabline",
smokeArtifactPath: "crabline-fake-provider-smoke.json",
} as const;
const parentLab = createCleanupTestLab();
const openClawLab = createCleanupTestLab();
const codexLab = createCleanupTestLab();
@@ -300,6 +328,8 @@ describe("runtime parity suite transport cleanup", () => {
startedAt: new Date("2026-08-04T00:00:00.000Z"),
providerMode: "mock-openai",
transportId: "qa-channel",
channelId: "telegram",
channelDriverSelection: selection,
primaryModel: "mock-openai/test-model",
alternateModel: "mock-openai/test-model-alt",
fastMode: true,
@@ -317,6 +347,15 @@ describe("runtime parity suite transport cleanup", () => {
.filter((line) => line.startsWith("[qa-suite] run complete"));
expect(completionLines).toEqual(["[qa-suite] run complete"]);
expect(runScenario).toHaveBeenCalledTimes(2);
expect(mocks.writeQaSuiteArtifacts).toHaveBeenCalledTimes(3);
for (const [cellArtifacts] of mocks.writeQaSuiteArtifacts.mock.calls.slice(0, -1)) {
expect(cellArtifacts.channelDriverSelection).toBeUndefined();
}
expect(mocks.writeQaSuiteArtifacts.mock.calls.at(-1)?.[0]).toMatchObject({
channel: "telegram",
channelDriver: "crabline",
channelDriverSelection: selection,
});
expect(openClawLab.stop).toHaveBeenCalledOnce();
expect(codexLab.stop).toHaveBeenCalledOnce();
expect(parentLab.stop).toHaveBeenCalledOnce();
extensions/qa-lab/src/suite-runtime-parity-runner.ts
+33 -30
View File
@@ -35,6 +35,7 @@ import type {
} from "./suite-types.js";
import {
createQaSuiteTransportAdapter,
markQaSuiteNestedRun,
requireQaSuiteStartLab,
runQaSuiteCleanupSteps,
throwQaSuiteCleanupErrors,
@@ -153,39 +154,41 @@ export async function runQaRuntimeParitySuite(params: {
runtime,
);
const cellStartedAt = Date.now();
const cellResult = await params.runQaFlowSuite({
adapterFactories: params.adapterFactories,
channelId: params.channelId,
adapterOptions: params.adapterOptions,
repoRoot: params.repoRoot,
outputDir: cellOutputDir,
providerMode: params.providerMode,
transportId: params.transportId,
channelDriver: params.channelDriver ?? undefined,
channelDriverSelection: params.channelDriverSelection,
primaryModel: remapModelRefForForcedRuntime({
modelRef: params.primaryModel,
const cellResult = await params.runQaFlowSuite(
markQaSuiteNestedRun({
adapterFactories: params.adapterFactories,
channelId: params.channelId,
adapterOptions: params.adapterOptions,
repoRoot: params.repoRoot,
outputDir: cellOutputDir,
providerMode: params.providerMode,
transportId: params.transportId,
channelDriver: params.channelDriver ?? undefined,
channelDriverSelection: params.channelDriverSelection,
primaryModel: remapModelRefForForcedRuntime({
modelRef: params.primaryModel,
providerMode: params.providerMode,
forcedRuntime: runtime,
}),
alternateModel: remapModelRefForForcedRuntime({
modelRef: params.alternateModel,
providerMode: params.providerMode,
forcedRuntime: runtime,
}),
fastMode: params.fastMode,
thinkingDefault: params.thinkingDefault,
claudeCliAuthMode: params.claudeCliAuthMode,
scenarioIds: [scenario.id],
concurrency: 1,
enabledPluginIds: params.enabledPluginIds,
startLab,
controlUiEnabled: params.controlUiEnabled ?? scenarioRequiresControlUi(scenario),
mutateConfig: params.mutateConfig,
forcedRuntime: runtime,
captureRuntimeParityCell: true,
writeEvidenceFile: params.writeEvidenceFile,
}),
alternateModel: remapModelRefForForcedRuntime({
modelRef: params.alternateModel,
providerMode: params.providerMode,
forcedRuntime: runtime,
}),
fastMode: params.fastMode,
thinkingDefault: params.thinkingDefault,
claudeCliAuthMode: params.claudeCliAuthMode,
scenarioIds: [scenario.id],
concurrency: 1,
enabledPluginIds: params.enabledPluginIds,
startLab,
controlUiEnabled: params.controlUiEnabled ?? scenarioRequiresControlUi(scenario),
mutateConfig: params.mutateConfig,
forcedRuntime: runtime,
captureRuntimeParityCell: true,
writeEvidenceFile: params.writeEvidenceFile,
});
);
for (const startedScenarioId of cellResult.startedScenarioIds) {
startedScenarioIds.add(startedScenarioId);
}