mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(qa): isolate Crabline from package live lanes (#108472)
* fix(qa): isolate Crabline from package live lanes * fix(qa): narrow optional smoke runtime * ci(release): avoid unrelated Crabline install
This commit is contained in:
committed by
GitHub
parent
379aca9f35
commit
3ee408b7b5
@@ -5,6 +5,7 @@
|
||||
"description": "OpenClaw QA lab plugin with private debugger UI and scenario runner",
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"@openclaw/crabline": "0.1.11",
|
||||
"@openclaw/matrix": "workspace:*",
|
||||
"@copilotkit/aimock": "1.35.0",
|
||||
"@modelcontextprotocol/sdk": "1.29.0",
|
||||
@@ -17,7 +18,6 @@
|
||||
"zod": "4.4.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openclaw/crabline": "0.1.11",
|
||||
"@openclaw/discord": "workspace:*",
|
||||
"@openclaw/plugin-sdk": "workspace:*",
|
||||
"@openclaw/slack": "workspace:*",
|
||||
|
||||
@@ -3,11 +3,17 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { runQaFlowSuite, runQaTestFileScenarios } = vi.hoisted(() => ({
|
||||
const { crablineRuntimeLoads, runQaFlowSuite, runQaTestFileScenarios } = vi.hoisted(() => ({
|
||||
crablineRuntimeLoads: vi.fn(),
|
||||
runQaFlowSuite: vi.fn(),
|
||||
runQaTestFileScenarios: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@openclaw/crabline", async (importOriginal) => {
|
||||
crablineRuntimeLoads();
|
||||
return await importOriginal<typeof import("@openclaw/crabline")>();
|
||||
});
|
||||
|
||||
vi.mock("./suite.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("./suite.js")>()),
|
||||
runQaFlowSuite,
|
||||
@@ -108,6 +114,20 @@ describe("qa suite runtime launcher", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps Crabline out of unrelated live transport startup", async () => {
|
||||
expect(crablineRuntimeLoads).not.toHaveBeenCalled();
|
||||
|
||||
await runQaSuite({
|
||||
repoRoot: process.cwd(),
|
||||
providerMode: "mock-openai",
|
||||
channelDriver: "live",
|
||||
channelId: "telegram",
|
||||
scenarioIds: ["channel-chat-baseline"],
|
||||
});
|
||||
|
||||
expect(crablineRuntimeLoads).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("routes selected flow scenarios to the flow suite engine", async () => {
|
||||
const result = await runQaSuite({
|
||||
repoRoot: process.cwd(),
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
// Qa Lab plugin module implements suite launch behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
OPENCLAW_CRABLINE_DEFAULT_CHANNEL,
|
||||
resolveOpenClawCrablineChannelDriverSelection,
|
||||
} from "@openclaw/crabline";
|
||||
import { renderQaMarkdownReport, type QaReportScenario } from "openclaw/plugin-sdk/qa-runtime";
|
||||
import { isRepoRootRelativeRef, toRepoRelativePath } from "./cli-paths.js";
|
||||
import {
|
||||
@@ -132,10 +128,10 @@ function resolveRequestedScenarios(params: {
|
||||
});
|
||||
}
|
||||
|
||||
function resolveQaFlowChannelGroups(
|
||||
async function resolveQaFlowChannelGroups(
|
||||
runParams: QaSuiteRunParams | undefined,
|
||||
scenarios: readonly QaSeedScenarioWithSource[],
|
||||
): QaFlowChannelGroup[] {
|
||||
): Promise<QaFlowChannelGroup[]> {
|
||||
if (runParams?.channelDriver !== "crabline") {
|
||||
return [
|
||||
{
|
||||
@@ -145,6 +141,10 @@ function resolveQaFlowChannelGroups(
|
||||
},
|
||||
];
|
||||
}
|
||||
// Package-only live lanes mount the QA harness without its dev tree. Load
|
||||
// Crabline only for Crabline-owned runs so unrelated transports stay isolated.
|
||||
const { OPENCLAW_CRABLINE_DEFAULT_CHANNEL, resolveOpenClawCrablineChannelDriverSelection } =
|
||||
await import("@openclaw/crabline");
|
||||
const channels = resolveQaSuiteScenarioChannels({
|
||||
defaultChannel: OPENCLAW_CRABLINE_DEFAULT_CHANNEL,
|
||||
explicitChannel: runParams.channelDriverSelection?.channel,
|
||||
@@ -175,7 +175,9 @@ function resolveQaFlowChannelGroups(
|
||||
}));
|
||||
}
|
||||
|
||||
function resolveSuiteExecutionPlan(params: QaSuiteRunParams | undefined): QaSuiteExecutionPlan {
|
||||
async function resolveSuiteExecutionPlan(
|
||||
params: QaSuiteRunParams | undefined,
|
||||
): Promise<QaSuiteExecutionPlan> {
|
||||
const scenarioIds = params?.scenarioIds ?? [];
|
||||
if (scenarioIds.length === 0) {
|
||||
return { kind: "flow" };
|
||||
@@ -195,8 +197,9 @@ function resolveSuiteExecutionPlan(params: QaSuiteRunParams | undefined): QaSuit
|
||||
testFileScenariosByKind.set(scenario.execution.kind, scenarios);
|
||||
}
|
||||
const requiresFlowPartitions =
|
||||
resolveQaFlowChannelGroups(params, flowScenarios).filter((group) => group.scenarios.length > 0)
|
||||
.length > 1 ||
|
||||
(await resolveQaFlowChannelGroups(params, flowScenarios)).filter(
|
||||
(group) => group.scenarios.length > 0,
|
||||
).length > 1 ||
|
||||
(flowScenarios.length > 1 && flowScenarios.some(scenarioRequiresIsolatedQaSuiteWorker));
|
||||
if (testFileScenariosByKind.size === 0 && !requiresFlowPartitions) {
|
||||
return { kind: "flow" };
|
||||
@@ -536,9 +539,8 @@ async function runUnifiedQaSuite(params: {
|
||||
const testFilePartitionTasks: QaUnifiedPartitionTask[] = [];
|
||||
const scriptPartitionTasks: QaUnifiedPartitionTask[] = [];
|
||||
if (params.plan.flowScenarios.length > 0) {
|
||||
const channelGroups = resolveQaFlowChannelGroups(
|
||||
params.runParams,
|
||||
params.plan.flowScenarios,
|
||||
const channelGroups = (
|
||||
await resolveQaFlowChannelGroups(params.runParams, params.plan.flowScenarios)
|
||||
).filter((group) => group.scenarios.length > 0);
|
||||
const mixedChannelRun = channelGroups.length > 1;
|
||||
const runFlowSuite = await loadQaFlowSuiteRuntime();
|
||||
@@ -778,7 +780,7 @@ async function runUnifiedQaSuite(params: {
|
||||
|
||||
export async function runQaSuite(...args: [QaSuiteRunParams?]): Promise<QaSuiteRuntimeResult> {
|
||||
const runParams = args[0];
|
||||
const plan = resolveSuiteExecutionPlan(runParams);
|
||||
const plan = await resolveSuiteExecutionPlan(runParams);
|
||||
if (plan.kind === "unified") {
|
||||
const result = await runUnifiedQaSuite({
|
||||
runParams,
|
||||
|
||||
@@ -2,11 +2,7 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import {
|
||||
createOpenClawCrablineChannelReportNotes,
|
||||
runOpenClawCrablineChannelDriverSmoke,
|
||||
type OpenClawCrablineChannelDriverSelection,
|
||||
} from "@openclaw/crabline";
|
||||
import type { OpenClawCrablineChannelDriverSelection } from "@openclaw/crabline";
|
||||
import { disposeRegisteredAgentHarnesses } from "openclaw/plugin-sdk/agent-harness";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
@@ -100,8 +96,9 @@ import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js";
|
||||
import { countQaSuiteFailedScenarios, type QaSuiteSummaryJson } from "./suite-summary.js";
|
||||
import { closeQaWebSessions } from "./web-runtime.js";
|
||||
|
||||
type QaCrablineRuntime = typeof import("@openclaw/crabline");
|
||||
type QaCrablineChannelDriverSmokeResult = Awaited<
|
||||
ReturnType<typeof runOpenClawCrablineChannelDriverSmoke>
|
||||
ReturnType<QaCrablineRuntime["runOpenClawCrablineChannelDriverSmoke"]>
|
||||
>;
|
||||
function resolveQaSuiteControlUiEnabled(params: {
|
||||
explicit?: boolean;
|
||||
@@ -501,13 +498,14 @@ function createQaSuiteReportNotes(params: {
|
||||
fastMode: boolean;
|
||||
concurrency: number;
|
||||
isolatedWorkers?: boolean;
|
||||
createCrablineChannelReportNotes?: QaCrablineRuntime["createOpenClawCrablineChannelReportNotes"];
|
||||
}) {
|
||||
return [
|
||||
...params.transport.createReportNotes(params),
|
||||
// Crabline reports completed generation paths through this filename-narrowed selection.
|
||||
...createOpenClawCrablineChannelReportNotes(
|
||||
...(params.createCrablineChannelReportNotes?.(
|
||||
params.channelDriverSelection as OpenClawCrablineChannelDriverSelection | null | undefined,
|
||||
),
|
||||
) ?? []),
|
||||
];
|
||||
}
|
||||
|
||||
@@ -990,20 +988,31 @@ async function writeQaSuiteArtifacts(params: {
|
||||
runtimePair?: [RuntimeId, RuntimeId];
|
||||
writeEvidenceFile?: boolean;
|
||||
runCrablineChannelDriverSmoke?: (
|
||||
params: Parameters<typeof runOpenClawCrablineChannelDriverSmoke>[0],
|
||||
params: Parameters<QaCrablineRuntime["runOpenClawCrablineChannelDriverSmoke"]>[0],
|
||||
) => Promise<QaCrablineChannelDriverSmokeResult>;
|
||||
}) {
|
||||
const reportPath = path.join(params.outputDir, "qa-suite-report.md");
|
||||
const summaryPath = path.join(params.outputDir, "qa-suite-summary.json");
|
||||
const evidencePath = path.join(params.outputDir, QA_EVIDENCE_FILENAME);
|
||||
const crablineChannelDriverSelection = params.channelDriverSelection;
|
||||
const crablineChannelDriverSmoke: QaCrablineChannelDriverSmokeResult | undefined =
|
||||
crablineChannelDriverSelection
|
||||
? await (params.runCrablineChannelDriverSmoke ?? runOpenClawCrablineChannelDriverSmoke)({
|
||||
outputDir: params.outputDir,
|
||||
selection: crablineChannelDriverSelection,
|
||||
})
|
||||
: undefined;
|
||||
// Non-Crabline package acceptance mounts this source without plugin-local
|
||||
// dependencies. Keep the owner runtime outside every unrelated live path.
|
||||
const crablineRuntime = crablineChannelDriverSelection
|
||||
? await import("@openclaw/crabline")
|
||||
: undefined;
|
||||
let crablineChannelDriverSmoke: QaCrablineChannelDriverSmokeResult | undefined;
|
||||
if (crablineChannelDriverSelection) {
|
||||
const runCrablineChannelDriverSmoke =
|
||||
params.runCrablineChannelDriverSmoke ??
|
||||
crablineRuntime?.runOpenClawCrablineChannelDriverSmoke;
|
||||
if (!runCrablineChannelDriverSmoke) {
|
||||
throw new Error("Crabline runtime did not provide its channel-driver smoke helper.");
|
||||
}
|
||||
crablineChannelDriverSmoke = await runCrablineChannelDriverSmoke({
|
||||
outputDir: params.outputDir,
|
||||
selection: crablineChannelDriverSelection,
|
||||
});
|
||||
}
|
||||
const crablineChannelDriverArtifactPaths = resolveQaCrablineChannelDriverArtifactPaths({
|
||||
result: crablineChannelDriverSmoke,
|
||||
selection: crablineChannelDriverSelection,
|
||||
@@ -1029,6 +1038,7 @@ async function writeQaSuiteArtifacts(params: {
|
||||
notes: createQaSuiteReportNotes({
|
||||
...params,
|
||||
channelDriverSelection: effectiveChannelDriverSelection,
|
||||
createCrablineChannelReportNotes: crablineRuntime?.createOpenClawCrablineChannelReportNotes,
|
||||
}),
|
||||
});
|
||||
const evidence =
|
||||
|
||||
Generated
+3
-3
@@ -1458,6 +1458,9 @@ importers:
|
||||
|
||||
extensions/qa-lab:
|
||||
dependencies:
|
||||
'@openclaw/crabline':
|
||||
specifier: 0.1.11
|
||||
version: 0.1.11
|
||||
'@openclaw/matrix':
|
||||
specifier: workspace:*
|
||||
version: link:../matrix
|
||||
@@ -1489,9 +1492,6 @@ importers:
|
||||
specifier: 4.4.3
|
||||
version: 4.4.3
|
||||
devDependencies:
|
||||
'@openclaw/crabline':
|
||||
specifier: 0.1.11
|
||||
version: 0.1.11
|
||||
'@openclaw/discord':
|
||||
specifier: workspace:*
|
||||
version: link:../discord
|
||||
|
||||
@@ -456,12 +456,6 @@ cp "$openclaw_package_dir/package.json" /app/package.json
|
||||
node scripts/e2e/lib/npm-telegram-live/prepare-package.mjs \
|
||||
/app/package.json \
|
||||
/app/node_modules/openclaw/package.json
|
||||
# QA Lab is mounted from source, so install its external runtime-only harness dependency
|
||||
# separately from the package candidate under test.
|
||||
crabline_version="$(
|
||||
node -e 'process.stdout.write(require("./extensions/qa-lab/package.json").devDependencies["@openclaw/crabline"])'
|
||||
)"
|
||||
npm install -g "@openclaw/crabline@$crabline_version" --no-fund --no-audit
|
||||
for deps_dir in "$openclaw_package_dir/node_modules" /npm-global/lib/node_modules; do
|
||||
[ -d "$deps_dir" ] || continue
|
||||
for dependency_dir in "$deps_dir"/*; do
|
||||
|
||||
@@ -267,17 +267,6 @@ describe("package Telegram live Docker E2E", () => {
|
||||
expect(script).toContain("zod");
|
||||
});
|
||||
|
||||
it("installs the mounted QA harness Crabline runtime dependency", () => {
|
||||
const script = readFileSync(DOCKER_SCRIPT_PATH, "utf8");
|
||||
|
||||
expect(script).toContain(
|
||||
'require("./extensions/qa-lab/package.json").devDependencies["@openclaw/crabline"]',
|
||||
);
|
||||
expect(script).toContain(
|
||||
'npm install -g "@openclaw/crabline@$crabline_version" --no-fund --no-audit',
|
||||
);
|
||||
});
|
||||
|
||||
it("lets npm-specific credential aliases override shared QA env", () => {
|
||||
expect(
|
||||
testing.resolveCredentialSource({
|
||||
|
||||
Reference in New Issue
Block a user