ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-27 21:07:01 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(e2e): preserve docker cleanup failure artifacts

Browse Source
This commit is contained in:
Vincent Koc
2026-06-06 17:37:01 +02:00
parent ffea7fa647
commit 69a406118c
4 changed files with 312 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/docker-e2e-rerun.mjs
+51 -37
View File
@@ -161,6 +161,27 @@ function ghWorkflowCommand(lanes, ref, workflow, reuseInputs = {}) {
return fields.join(" ");
}
function failureName(failure) {
return failure.name || failure.lane || "";
}
function failedEntryFromRecord(failure, file, ref, workflow, reuseInputs) {
const lane = failureName(failure);
const targetable = failure.targetable !== false;
return {
ghWorkflowCommand: targetable
? failure.ghWorkflowCommand || ghWorkflowCommand([lane], ref, workflow, reuseInputs)
: "",
lane,
localRerunCommand: failure.rerunCommand,
logFile: failure.logFile,
reuseInputs,
source: file,
status: failure.status,
targetable,
};
}
function detectRepo() {
return run("gh", ["repo", "view", "--json", "nameWithOwner", "--jq", ".nameWithOwner"]).trim();
}
@@ -183,31 +204,18 @@ function failedLaneEntriesFromJson(file, ref, workflow) {
const source = path.basename(file);
if (source === "failures.json" && Array.isArray(parsed.lanes)) {
return parsed.lanes
.filter((lane) => lane.name)
.map((lane) => ({
ghWorkflowCommand:
lane.ghWorkflowCommand || ghWorkflowCommand([lane.name], ref, workflow, reuseInputs),
lane: lane.name,
localRerunCommand: lane.rerunCommand,
logFile: lane.logFile,
reuseInputs,
source: file,
status: lane.status,
}));
.filter((lane) => failureName(lane))
.map((lane) => failedEntryFromRecord(lane, file, ref, workflow, reuseInputs));
}
const lanes = Array.isArray(parsed.lanes) ? parsed.lanes : [];
return lanes
.filter((lane) => lane.status !== 0 && lane.name)
.map((lane) => ({
ghWorkflowCommand: ghWorkflowCommand([lane.name], ref, workflow, reuseInputs),
lane: lane.name,
localRerunCommand: lane.rerunCommand,
logFile: lane.logFile,
reuseInputs,
source: file,
status: lane.status,
}));
const failures =
Array.isArray(parsed.failures) && parsed.failures.length > 0
? parsed.failures
: lanes.filter((lane) => lane.status !== 0);
return failures
.filter((lane) => failureName(lane))
.map((lane) => failedEntryFromRecord(lane, file, ref, workflow, reuseInputs));
}
function mergeByLane(entries) {
@@ -275,23 +283,29 @@ function printEntries(entries, ref, workflow, runValue) {
console.log("No failed Docker E2E lanes found.");
return;
}
console.log(`Failed lanes: ${entries.map((entry) => entry.lane).join(", ")}`);
console.log("");
console.log("Combined GitHub rerun:");
console.log(
ghWorkflowCommand(
entries.map((entry) => entry.lane),
ref,
workflow,
commonReuseInputs(entries),
),
);
console.log("");
console.log("Per-lane GitHub reruns:");
for (const entry of entries) {
const workflowEntries = entries.filter((entry) => entry.targetable !== false);
console.log(`Failed Docker E2E entries: ${entries.map((entry) => entry.lane).join(", ")}`);
if (workflowEntries.length > 0) {
console.log("");
console.log("Combined GitHub rerun:");
console.log(
`- ${entry.lane}: ${entry.ghWorkflowCommand || ghWorkflowCommand([entry.lane], ref, workflow)}`,
ghWorkflowCommand(
workflowEntries.map((entry) => entry.lane),
ref,
workflow,
commonReuseInputs(workflowEntries),
),
);
console.log("");
console.log("Per-lane GitHub reruns:");
for (const entry of workflowEntries) {
console.log(
`- ${entry.lane}: ${entry.ghWorkflowCommand || ghWorkflowCommand([entry.lane], ref, workflow)}`,
);
}
} else {
console.log("");
console.log("No targetable failed Docker E2E lanes found.");
}
console.log("");
console.log("Local rerun starting points:");
scripts/test-docker-all.mjs
+132 -8
View File
@@ -40,6 +40,7 @@ const DEFAULT_LANE_TIMEOUT_MS = 120 * 60 * 1000;
const DEFAULT_LANE_START_STAGGER_MS = 2_000;
const DEFAULT_STATUS_INTERVAL_MS = 30_000;
const DEFAULT_PREFLIGHT_RUN_TIMEOUT_MS = 60_000;
const CLEANUP_SMOKE_NAME = "cleanup-smoke";
export const SHELL_CAPTURE_MAX_CHARS = 1024 * 1024;
export const LOG_TAIL_MAX_BYTES = 1024 * 1024;
const DEFAULT_TIMINGS_FILE = path.join(ROOT_DIR, ".artifacts/docker-tests/lane-timings.json");
@@ -456,8 +457,10 @@ async function writeFailureIndex(logDir, summary) {
const failures = Array.isArray(summary.failures)
? summary.failures
: (summary.lanes ?? []).filter((lane) => lane.status !== 0);
const workflowRerunFailures = failures.filter((failure) => failure.targetable !== false);
const lanes = failures.map((failure) => ({
ghWorkflowCommand: githubWorkflowRerunCommand([failure.name], ref),
ghWorkflowCommand:
failure.targetable === false ? undefined : githubWorkflowRerunCommand([failure.name], ref),
image: failure.image,
imageKind: failure.imageKind,
lane: failure.name,
@@ -466,13 +469,14 @@ async function writeFailureIndex(logDir, summary) {
noOutputTimedOut: failure.noOutputTimedOut,
rerunCommand: failure.rerunCommand,
status: failure.status,
targetable: failure.targetable,
timedOut: failure.timedOut,
}));
const failureIndex = {
combinedGhWorkflowCommand:
lanes.length > 0
workflowRerunFailures.length > 0
? githubWorkflowRerunCommand(
lanes.map((lane) => lane.lane),
workflowRerunFailures.map((failure) => failure.name),
ref,
)
: undefined,
@@ -712,6 +716,85 @@ async function runForeground(label, command, env) {
}
}
async function recordCleanupSmokeFailure(error, baseEnv, logDir, command, startedAtMs) {
const status = 1;
const logFile = path.join(logDir, `${CLEANUP_SMOKE_NAME}.log`);
const message = error instanceof Error ? error.message : String(error);
await fs.promises.writeFile(
logFile,
[
`==> [${CLEANUP_SMOKE_NAME}] command: ${command}`,
`==> [${CLEANUP_SMOKE_NAME}] status: ${status}`,
`==> [${CLEANUP_SMOKE_NAME}] error: ${message}`,
]
.filter(Boolean)
.join("\n"),
);
return {
command,
attempts: [
{
attempt: 1,
elapsedSeconds: phaseElapsedSeconds(startedAtMs),
finishedAt: new Date().toISOString(),
noOutputTimedOut: false,
startedAt: new Date(startedAtMs).toISOString(),
status,
timedOut: false,
},
],
elapsedSeconds: phaseElapsedSeconds(startedAtMs),
finishedAt: new Date().toISOString(),
image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE,
logFile,
name: CLEANUP_SMOKE_NAME,
noOutputTimedOut: false,
rerunCommand: command,
startedAt: new Date(startedAtMs).toISOString(),
status,
targetable: false,
timedOut: false,
};
}
async function runCleanupSmoke(baseEnv, logDir, command, startedAtMs) {
const logFile = path.join(logDir, `${CLEANUP_SMOKE_NAME}.log`);
const result = await runShellCommand({
command,
env: baseEnv,
label: CLEANUP_SMOKE_NAME,
logFile,
});
if (result.status === 0) {
return undefined;
}
return {
command,
attempts: [
{
attempt: 1,
elapsedSeconds: phaseElapsedSeconds(startedAtMs),
finishedAt: new Date().toISOString(),
noOutputTimedOut: result.noOutputTimedOut,
startedAt: new Date(startedAtMs).toISOString(),
status: result.status,
timedOut: result.timedOut,
},
],
elapsedSeconds: phaseElapsedSeconds(startedAtMs),
finishedAt: new Date().toISOString(),
image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE,
logFile,
name: CLEANUP_SMOKE_NAME,
noOutputTimedOut: result.noOutputTimedOut,
rerunCommand: command,
startedAt: new Date(startedAtMs).toISOString(),
status: result.status,
targetable: false,
timedOut: result.timedOut,
};
}
async function runForegroundGroup(entries, env) {
const failures = [];
for (const entry of entries) {
@@ -1469,17 +1552,58 @@ async function main() {
}
if (profile === DEFAULT_PROFILE && selectedLaneNames.length === 0) {
await runPhase(phases, "cleanup-smoke", {}, async () => {
await runForeground(
"Run cleanup smoke after parallel lanes",
"pnpm test:docker:cleanup",
const cleanupSmokeCommand = "pnpm test:docker:cleanup";
const cleanupStartedAtMs = Date.now();
let cleanupFailure;
try {
await runPhase(phases, CLEANUP_SMOKE_NAME, {}, async () => {
cleanupFailure = await runCleanupSmoke(
baseEnv,
logDir,
cleanupSmokeCommand,
cleanupStartedAtMs,
);
if (cleanupFailure) {
throw new Error(
`Run cleanup smoke after parallel lanes failed with status ${cleanupFailure.status}`,
);
}
});
} catch (error) {
cleanupFailure ??= await recordCleanupSmokeFailure(
error,
baseEnv,
logDir,
cleanupSmokeCommand,
cleanupStartedAtMs,
);
});
}
if (cleanupFailure) {
failures.push(cleanupFailure);
}
} else {
console.log("==> Cleanup smoke after parallel lanes: skipped for selected/release lanes");
}
await writeTimingStore(timingStore, allResults);
if (failures.length > 0) {
await writeRunSummary(logDir, {
chunk: releaseChunk || undefined,
failures,
image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE,
images: {
bare: baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE,
functional: baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE,
},
lanes: allResults,
phases,
profile,
selectedLanes: selectedLaneNames.length > 0 ? selectedLaneNames : undefined,
startedAt: runStartedAt,
status: "failed",
});
await printFailureSummary(failures, tailLines);
process.exit(1);
}
await writeRunSummary(logDir, {
chunk: releaseChunk || undefined,
failures,
test/scripts/docker-all-scheduler.test.ts
+80 -1
View File
@@ -1,6 +1,6 @@
// Docker All Scheduler tests cover docker all scheduler script behavior.
import { spawnSync } from "node:child_process";
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { setTimeout as delay } from "node:timers/promises";
@@ -177,6 +177,85 @@ describe("scripts/test-docker-all scheduler", () => {
}
});
posixIt("writes Docker run artifacts when cleanup smoke fails", () => {
const root = mkdtempSync(`${tmpdir()}/openclaw-docker-all-cleanup-`);
const logDir = path.join(root, "logs");
const packageTgz = path.join(root, "openclaw-current.tgz");
const fakePnpm = path.join(root, "pnpm");
writeFileSync(packageTgz, "fake package\n", "utf8");
writeFileSync(
fakePnpm,
`#!/usr/bin/env node
const command = process.argv.slice(2).join(" ");
if (command === "test:docker:cleanup") {
console.error("cleanup smoke failed intentionally");
process.exit(42);
}
process.exit(0);
`,
"utf8",
);
chmodSync(fakePnpm, 0o755);
try {
const result = spawnSync(process.execPath, ["scripts/test-docker-all.mjs"], {
cwd: process.cwd(),
encoding: "utf8",
env: {
...process.env,
OPENCLAW_CURRENT_PACKAGE_TGZ: packageTgz,
OPENCLAW_DOCKER_ALL_BUILD: "0",
OPENCLAW_DOCKER_ALL_LIVE_MODE: "skip",
OPENCLAW_DOCKER_ALL_LOG_DIR: logDir,
OPENCLAW_DOCKER_ALL_PARALLELISM: "16",
OPENCLAW_DOCKER_ALL_PREFLIGHT: "0",
OPENCLAW_DOCKER_ALL_START_STAGGER_MS: "0",
OPENCLAW_DOCKER_ALL_STATUS_INTERVAL_MS: "0",
OPENCLAW_DOCKER_ALL_TAIL_PARALLELISM: "16",
OPENCLAW_DOCKER_ALL_TIMINGS: "0",
PATH: `${root}${path.delimiter}${process.env.PATH ?? ""}`,
},
});
expect(result.status).toBe(1);
expect(result.stderr).toContain("cleanup smoke failed intentionally");
const summary = JSON.parse(readFileSync(path.join(logDir, "summary.json"), "utf8"));
expect(summary.status).toBe("failed");
expect(summary.failures).toHaveLength(1);
expect(summary.failures[0]).toMatchObject({
name: "cleanup-smoke",
rerunCommand: "pnpm test:docker:cleanup",
status: 42,
targetable: false,
});
expect(summary.lanes.some((lane: { name?: string }) => lane.name === "cleanup-smoke")).toBe(
false,
);
expect(summary.phases.at(-1)).toMatchObject({
name: "cleanup-smoke",
status: "failed",
});
const failureIndex = JSON.parse(readFileSync(path.join(logDir, "failures.json"), "utf8"));
expect(failureIndex.status).toBe("failed");
expect(failureIndex.combinedGhWorkflowCommand).toBeUndefined();
expect(failureIndex.lanes[0]?.ghWorkflowCommand).toBeUndefined();
expect(failureIndex.lanes).toEqual([
expect.objectContaining({
lane: "cleanup-smoke",
rerunCommand: "pnpm test:docker:cleanup",
status: 42,
targetable: false,
}),
]);
const cleanupLog = readFileSync(path.join(logDir, "cleanup-smoke.log"), "utf8");
expect(cleanupLog).toContain("cleanup smoke failed intentionally");
} finally {
rmSync(root, { force: true, recursive: true });
}
});
it("allows an overweight lane to start alone under low parallelism", () => {
expect(
canStartSchedulerLane(
test/scripts/docker-e2e-helper-cli.test.ts
+49
View File
@@ -1,5 +1,8 @@
// Docker E2E Helper Cli tests cover docker e2e helper cli script behavior.
import { spawnSync } from "node:child_process";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
function runHelper(script: string, ...args: string[]) {
@@ -62,4 +65,50 @@ describe("Docker E2E helper CLIs", () => {
"node scripts/docker-e2e-rerun.mjs <run-id|summary.json|failures.json>",
);
});
it.each(["summary.json", "failures.json"])(
"prints local cleanup reruns without synthesizing Docker lane reruns from %s",
(fileName) => {
const root = mkdtempSync(`${tmpdir()}/openclaw-docker-e2e-rerun-`);
try {
const cleanupFailure = {
lane: "cleanup-smoke",
logFile: "cleanup-smoke.log",
name: "cleanup-smoke",
rerunCommand: "pnpm test:docker:cleanup",
status: 42,
targetable: false,
};
const payload =
fileName === "summary.json"
? {
failures: [cleanupFailure],
lanes: [
{
name: "gateway-network",
status: 0,
},
],
status: "failed",
}
: {
lanes: [cleanupFailure],
status: "failed",
};
const file = path.join(root, fileName);
writeFileSync(file, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
const result = runHelper("scripts/docker-e2e-rerun.mjs", file, "--ref", "abc123");
expect(result.status).toBe(0);
expect(result.stderr).toBe("");
expect(result.stdout).toContain("Failed Docker E2E entries: cleanup-smoke");
expect(result.stdout).toContain("No targetable failed Docker E2E lanes found.");
expect(result.stdout).toContain("- cleanup-smoke: pnpm test:docker:cleanup");
expect(result.stdout).not.toContain("docker_lanes='cleanup-smoke'");
} finally {
rmSync(root, { force: true, recursive: true });
}
},
);
});