fix(qa): isolate Crabline from package live lanes (#108472)

* fix(qa): isolate Crabline from package live lanes

* fix(qa): narrow optional smoke runtime

* ci(release): avoid unrelated Crabline install
This commit is contained in:
Peter Steinberger
2026-07-15 15:09:07 -07:00
committed by GitHub
parent 379aca9f35
commit 3ee408b7b5
7 changed files with 66 additions and 51 deletions
+1 -1
View File
@@ -5,6 +5,7 @@
"description": "OpenClaw QA lab plugin with private debugger UI and scenario runner",
"type": "module",
"dependencies": {
"@openclaw/crabline": "0.1.11",
"@openclaw/matrix": "workspace:*",
"@copilotkit/aimock": "1.35.0",
"@modelcontextprotocol/sdk": "1.29.0",
@@ -17,7 +18,6 @@
"zod": "4.4.3"
},
"devDependencies": {
"@openclaw/crabline": "0.1.11",
"@openclaw/discord": "workspace:*",
"@openclaw/plugin-sdk": "workspace:*",
"@openclaw/slack": "workspace:*",
@@ -3,11 +3,17 @@ import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const { runQaFlowSuite, runQaTestFileScenarios } = vi.hoisted(() => ({
const { crablineRuntimeLoads, runQaFlowSuite, runQaTestFileScenarios } = vi.hoisted(() => ({
crablineRuntimeLoads: vi.fn(),
runQaFlowSuite: vi.fn(),
runQaTestFileScenarios: vi.fn(),
}));
vi.mock("@openclaw/crabline", async (importOriginal) => {
crablineRuntimeLoads();
return await importOriginal<typeof import("@openclaw/crabline")>();
});
vi.mock("./suite.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./suite.js")>()),
runQaFlowSuite,
@@ -108,6 +114,20 @@ describe("qa suite runtime launcher", () => {
);
});
it("keeps Crabline out of unrelated live transport startup", async () => {
expect(crablineRuntimeLoads).not.toHaveBeenCalled();
await runQaSuite({
repoRoot: process.cwd(),
providerMode: "mock-openai",
channelDriver: "live",
channelId: "telegram",
scenarioIds: ["channel-chat-baseline"],
});
expect(crablineRuntimeLoads).not.toHaveBeenCalled();
});
it("routes selected flow scenarios to the flow suite engine", async () => {
const result = await runQaSuite({
repoRoot: process.cwd(),
+15 -13
View File
@@ -1,10 +1,6 @@
// Qa Lab plugin module implements suite launch behavior.
import fs from "node:fs/promises";
import path from "node:path";
import {
OPENCLAW_CRABLINE_DEFAULT_CHANNEL,
resolveOpenClawCrablineChannelDriverSelection,
} from "@openclaw/crabline";
import { renderQaMarkdownReport, type QaReportScenario } from "openclaw/plugin-sdk/qa-runtime";
import { isRepoRootRelativeRef, toRepoRelativePath } from "./cli-paths.js";
import {
@@ -132,10 +128,10 @@ function resolveRequestedScenarios(params: {
});
}
function resolveQaFlowChannelGroups(
async function resolveQaFlowChannelGroups(
runParams: QaSuiteRunParams | undefined,
scenarios: readonly QaSeedScenarioWithSource[],
): QaFlowChannelGroup[] {
): Promise<QaFlowChannelGroup[]> {
if (runParams?.channelDriver !== "crabline") {
return [
{
@@ -145,6 +141,10 @@ function resolveQaFlowChannelGroups(
},
];
}
// Package-only live lanes mount the QA harness without its dev tree. Load
// Crabline only for Crabline-owned runs so unrelated transports stay isolated.
const { OPENCLAW_CRABLINE_DEFAULT_CHANNEL, resolveOpenClawCrablineChannelDriverSelection } =
await import("@openclaw/crabline");
const channels = resolveQaSuiteScenarioChannels({
defaultChannel: OPENCLAW_CRABLINE_DEFAULT_CHANNEL,
explicitChannel: runParams.channelDriverSelection?.channel,
@@ -175,7 +175,9 @@ function resolveQaFlowChannelGroups(
}));
}
function resolveSuiteExecutionPlan(params: QaSuiteRunParams | undefined): QaSuiteExecutionPlan {
async function resolveSuiteExecutionPlan(
params: QaSuiteRunParams | undefined,
): Promise<QaSuiteExecutionPlan> {
const scenarioIds = params?.scenarioIds ?? [];
if (scenarioIds.length === 0) {
return { kind: "flow" };
@@ -195,8 +197,9 @@ function resolveSuiteExecutionPlan(params: QaSuiteRunParams | undefined): QaSuit
testFileScenariosByKind.set(scenario.execution.kind, scenarios);
}
const requiresFlowPartitions =
resolveQaFlowChannelGroups(params, flowScenarios).filter((group) => group.scenarios.length > 0)
.length > 1 ||
(await resolveQaFlowChannelGroups(params, flowScenarios)).filter(
(group) => group.scenarios.length > 0,
).length > 1 ||
(flowScenarios.length > 1 && flowScenarios.some(scenarioRequiresIsolatedQaSuiteWorker));
if (testFileScenariosByKind.size === 0 && !requiresFlowPartitions) {
return { kind: "flow" };
@@ -536,9 +539,8 @@ async function runUnifiedQaSuite(params: {
const testFilePartitionTasks: QaUnifiedPartitionTask[] = [];
const scriptPartitionTasks: QaUnifiedPartitionTask[] = [];
if (params.plan.flowScenarios.length > 0) {
const channelGroups = resolveQaFlowChannelGroups(
params.runParams,
params.plan.flowScenarios,
const channelGroups = (
await resolveQaFlowChannelGroups(params.runParams, params.plan.flowScenarios)
).filter((group) => group.scenarios.length > 0);
const mixedChannelRun = channelGroups.length > 1;
const runFlowSuite = await loadQaFlowSuiteRuntime();
@@ -778,7 +780,7 @@ async function runUnifiedQaSuite(params: {
export async function runQaSuite(...args: [QaSuiteRunParams?]): Promise<QaSuiteRuntimeResult> {
const runParams = args[0];
const plan = resolveSuiteExecutionPlan(runParams);
const plan = await resolveSuiteExecutionPlan(runParams);
if (plan.kind === "unified") {
const result = await runUnifiedQaSuite({
runParams,
+26 -16
View File
@@ -2,11 +2,7 @@
import fs from "node:fs/promises";
import path from "node:path";
import { setTimeout as sleep } from "node:timers/promises";
import {
createOpenClawCrablineChannelReportNotes,
runOpenClawCrablineChannelDriverSmoke,
type OpenClawCrablineChannelDriverSelection,
} from "@openclaw/crabline";
import type { OpenClawCrablineChannelDriverSelection } from "@openclaw/crabline";
import { disposeRegisteredAgentHarnesses } from "openclaw/plugin-sdk/agent-harness";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
@@ -100,8 +96,9 @@ import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js";
import { countQaSuiteFailedScenarios, type QaSuiteSummaryJson } from "./suite-summary.js";
import { closeQaWebSessions } from "./web-runtime.js";
type QaCrablineRuntime = typeof import("@openclaw/crabline");
type QaCrablineChannelDriverSmokeResult = Awaited<
ReturnType<typeof runOpenClawCrablineChannelDriverSmoke>
ReturnType<QaCrablineRuntime["runOpenClawCrablineChannelDriverSmoke"]>
>;
function resolveQaSuiteControlUiEnabled(params: {
explicit?: boolean;
@@ -501,13 +498,14 @@ function createQaSuiteReportNotes(params: {
fastMode: boolean;
concurrency: number;
isolatedWorkers?: boolean;
createCrablineChannelReportNotes?: QaCrablineRuntime["createOpenClawCrablineChannelReportNotes"];
}) {
return [
...params.transport.createReportNotes(params),
// Crabline reports completed generation paths through this filename-narrowed selection.
...createOpenClawCrablineChannelReportNotes(
...(params.createCrablineChannelReportNotes?.(
params.channelDriverSelection as OpenClawCrablineChannelDriverSelection | null | undefined,
),
) ?? []),
];
}
@@ -990,20 +988,31 @@ async function writeQaSuiteArtifacts(params: {
runtimePair?: [RuntimeId, RuntimeId];
writeEvidenceFile?: boolean;
runCrablineChannelDriverSmoke?: (
params: Parameters<typeof runOpenClawCrablineChannelDriverSmoke>[0],
params: Parameters<QaCrablineRuntime["runOpenClawCrablineChannelDriverSmoke"]>[0],
) => Promise<QaCrablineChannelDriverSmokeResult>;
}) {
const reportPath = path.join(params.outputDir, "qa-suite-report.md");
const summaryPath = path.join(params.outputDir, "qa-suite-summary.json");
const evidencePath = path.join(params.outputDir, QA_EVIDENCE_FILENAME);
const crablineChannelDriverSelection = params.channelDriverSelection;
const crablineChannelDriverSmoke: QaCrablineChannelDriverSmokeResult | undefined =
crablineChannelDriverSelection
? await (params.runCrablineChannelDriverSmoke ?? runOpenClawCrablineChannelDriverSmoke)({
outputDir: params.outputDir,
selection: crablineChannelDriverSelection,
})
: undefined;
// Non-Crabline package acceptance mounts this source without plugin-local
// dependencies. Keep the owner runtime outside every unrelated live path.
const crablineRuntime = crablineChannelDriverSelection
? await import("@openclaw/crabline")
: undefined;
let crablineChannelDriverSmoke: QaCrablineChannelDriverSmokeResult | undefined;
if (crablineChannelDriverSelection) {
const runCrablineChannelDriverSmoke =
params.runCrablineChannelDriverSmoke ??
crablineRuntime?.runOpenClawCrablineChannelDriverSmoke;
if (!runCrablineChannelDriverSmoke) {
throw new Error("Crabline runtime did not provide its channel-driver smoke helper.");
}
crablineChannelDriverSmoke = await runCrablineChannelDriverSmoke({
outputDir: params.outputDir,
selection: crablineChannelDriverSelection,
});
}
const crablineChannelDriverArtifactPaths = resolveQaCrablineChannelDriverArtifactPaths({
result: crablineChannelDriverSmoke,
selection: crablineChannelDriverSelection,
@@ -1029,6 +1038,7 @@ async function writeQaSuiteArtifacts(params: {
notes: createQaSuiteReportNotes({
...params,
channelDriverSelection: effectiveChannelDriverSelection,
createCrablineChannelReportNotes: crablineRuntime?.createOpenClawCrablineChannelReportNotes,
}),
});
const evidence =
+3 -3
View File
@@ -1458,6 +1458,9 @@ importers:
extensions/qa-lab:
dependencies:
'@openclaw/crabline':
specifier: 0.1.11
version: 0.1.11
'@openclaw/matrix':
specifier: workspace:*
version: link:../matrix
@@ -1489,9 +1492,6 @@ importers:
specifier: 4.4.3
version: 4.4.3
devDependencies:
'@openclaw/crabline':
specifier: 0.1.11
version: 0.1.11
'@openclaw/discord':
specifier: workspace:*
version: link:../discord
-6
View File
@@ -456,12 +456,6 @@ cp "$openclaw_package_dir/package.json" /app/package.json
node scripts/e2e/lib/npm-telegram-live/prepare-package.mjs \
/app/package.json \
/app/node_modules/openclaw/package.json
# QA Lab is mounted from source, so install its external runtime-only harness dependency
# separately from the package candidate under test.
crabline_version="$(
node -e 'process.stdout.write(require("./extensions/qa-lab/package.json").devDependencies["@openclaw/crabline"])'
)"
npm install -g "@openclaw/crabline@$crabline_version" --no-fund --no-audit
for deps_dir in "$openclaw_package_dir/node_modules" /npm-global/lib/node_modules; do
[ -d "$deps_dir" ] || continue
for dependency_dir in "$deps_dir"/*; do
-11
View File
@@ -267,17 +267,6 @@ describe("package Telegram live Docker E2E", () => {
expect(script).toContain("zod");
});
it("installs the mounted QA harness Crabline runtime dependency", () => {
const script = readFileSync(DOCKER_SCRIPT_PATH, "utf8");
expect(script).toContain(
'require("./extensions/qa-lab/package.json").devDependencies["@openclaw/crabline"]',
);
expect(script).toContain(
'npm install -g "@openclaw/crabline@$crabline_version" --no-fund --no-audit',
);
});
it("lets npm-specific credential aliases override shared QA env", () => {
expect(
testing.resolveCredentialSource({