openclaw/scripts/docker-e2e-rerun.mts

#!/usr/bin/env node
// Builds cheap rerun commands from a Docker E2E GitHub run or local summary.
// For GitHub runs, the script downloads Docker E2E artifacts, reads
// summary/failures JSON, and prints targeted workflow commands for failed
// lanes, repacking the exact artifact target and reusing GHCR-backed prepared
// image refs when artifacts expose them.
import { spawnSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { isRecord } from "@openclaw/normalization-core/record-coerce";
import { readDockerE2eJsonArtifact } from "./lib/docker-e2e-json-artifacts.mts";

const DEFAULT_WORKFLOW = "openclaw-live-and-e2e-checks-reusable.yml";

const REUSE_INPUT_KEYS = [
  "bareImage",
  "functionalImage",
  "publishedUpgradeSurvivorBaseline",
  "publishedUpgradeSurvivorBaselines",
  "publishedUpgradeSurvivorScenarios",
  "allowUnreleasedChangelog",
] as const;

type ReuseInputKey = (typeof REUSE_INPUT_KEYS)[number];
type ReuseInputs = Partial<Record<ReuseInputKey, string>>;

function usage() {
  return [
    "Usage:",
    "  node --import tsx scripts/docker-e2e-rerun.mts <run-id|summary.json|failures.json> [--repo owner/repo] [--dir output-dir] [--workflow workflow.yml] [--ref ref]",
  ].join("\n");
}

function parseArgs(argv: string[]) {
  const options = {
    dir: "",
    help: false,
    input: "",
    ref: "",
    repo: "",
    workflow: DEFAULT_WORKFLOW,
  };
  for (let index = 0; index < argv.length; index += 1) {
    const arg = argv[index]!;
    if (arg === "--help" || arg === "-h") {
      options.help = true;
    } else if (arg === "--repo") {
      options.repo = argv[(index += 1)] ?? "";
    } else if (arg?.startsWith("--repo=")) {
      options.repo = arg.slice("--repo=".length);
    } else if (arg === "--dir") {
      options.dir = argv[(index += 1)] ?? "";
    } else if (arg?.startsWith("--dir=")) {
      options.dir = arg.slice("--dir=".length);
    } else if (arg === "--workflow") {
      options.workflow = argv[(index += 1)] ?? "";
    } else if (arg?.startsWith("--workflow=")) {
      options.workflow = arg.slice("--workflow=".length);
    } else if (arg === "--ref") {
      options.ref = argv[(index += 1)] ?? "";
    } else if (arg?.startsWith("--ref=")) {
      options.ref = arg.slice("--ref=".length);
    } else if (!options.input) {
      options.input = arg;
    } else {
      throw new Error(`unknown argument: ${arg}\n${usage()}`);
    }
  }
  if (options.help) {
    return options;
  }
  if (!options.input || !options.workflow) {
    throw new Error(usage());
  }
  return options;
}

function run(
  command: string,
  args: string[],
  options: { stdio?: "inherit" | ["ignore", "pipe", "pipe"] } = {},
): string {
  const result = spawnSync(command, args, {
    encoding: "utf8",
    stdio: options.stdio ?? ["ignore", "pipe", "pipe"],
  });
  if (result.status !== 0) {
    throw new Error(
      `${command} ${args.join(" ")} failed with ${result.status ?? result.signal}\n${result.stderr}`,
    );
  }
  return result.stdout;
}

function readJson(file: string) {
  const value = readDockerE2eJsonArtifact(file);
  return isRecord(value) ? value : {};
}

function shellQuote(value: string): string {
  return `'${value.replaceAll("'", "'\\''")}'`;
}

function laneNeedsReleasePath(lane: string): boolean {
  return /^bundled-channel(?:-|$)/u.test(lane);
}

function maybeGhcrImage(value: unknown): string {
  return typeof value === "string" && value.startsWith("ghcr.io/") ? value : "";
}

const TRUSTED_WORKFLOW_INPUTS = new Map<string, ReuseInputKey>([
  ["docker_e2e_bare_image", "bareImage"],
  ["docker_e2e_functional_image", "functionalImage"],
  ["published_upgrade_survivor_baseline", "publishedUpgradeSurvivorBaseline"],
  ["published_upgrade_survivor_baselines", "publishedUpgradeSurvivorBaselines"],
  ["published_upgrade_survivor_scenarios", "publishedUpgradeSurvivorScenarios"],
  ["allow_unreleased_changelog", "allowUnreleasedChangelog"],
]);

const WORKFLOW_INPUT_RE = /(?:^|\s)-f\s+([a-z0-9_]+)=('([^']*)'|[^\s]+)/gu;

function trustedReuseInputsFromCommand(command: unknown): ReuseInputs {
  const text = typeof command === "string" ? command : "";
  if (!/^\s*gh\s+workflow\s+run\s/u.test(text)) {
    return {};
  }
  const inputs: ReuseInputs = {};
  for (const match of text.matchAll(WORKFLOW_INPUT_RE)) {
    const input = match[1];
    const target = input ? TRUSTED_WORKFLOW_INPUTS.get(input) : undefined;
    const value = (match[3] ?? match[2] ?? "").replace(/^'/u, "").replace(/'$/u, "");
    if (!target || !value) {
      continue;
    }
    let normalized = value;
    if (target === "bareImage" || target === "functionalImage") {
      normalized = maybeGhcrImage(value);
    } else if (target === "allowUnreleasedChangelog" && value !== "true") {
      normalized = "";
    }
    if (normalized) {
      inputs[target] = normalized;
    }
  }
  return inputs;
}

function reuseInputsFromJson(parsed: Record<string, unknown>): ReuseInputs {
  const images = isRecord(parsed.images) ? parsed.images : {};
  const bareImage = maybeGhcrImage(images.bare);
  const functionalImage = maybeGhcrImage(images.functional);
  const allowUnreleasedChangelog = parsed.allowUnreleasedChangelog === true ? "true" : undefined;
  return {
    ...(allowUnreleasedChangelog ? { allowUnreleasedChangelog } : {}),
    ...(bareImage ? { bareImage } : {}),
    ...(functionalImage ? { functionalImage } : {}),
  };
}

function artifactTargetRef(
  parsed: Record<string, unknown>,
  file: string,
  required: boolean,
): string {
  const github = isRecord(parsed.github) ? parsed.github : {};
  const values = [parsed.ref, github.selectedSha].filter(
    (value) => value !== undefined && value !== null && value !== "",
  );
  if (
    !values.every(
      (value): value is string => typeof value === "string" && /^[a-f0-9]{40}$/u.test(value),
    )
  ) {
    if (required) {
      throw new Error(`${file} has an invalid artifact target ref; expected a full commit SHA`);
    }
    return "";
  }
  const refs = [...new Set(values)];
  if (refs.length > 1) {
    if (required) {
      throw new Error(`${file} has conflicting artifact target refs: ${refs.join(", ")}`);
    }
    return "";
  }
  return refs[0] || "";
}

function discardMismatchedPreparedImages(entry: FailedEntry, explicitRef: string): FailedEntry {
  if (!explicitRef || entry.artifactRef === explicitRef) {
    return entry;
  }
  const reuseInputs = Object.fromEntries(
    Object.entries(entry.reuseInputs ?? {}).filter(
      ([key]) => key !== "bareImage" && key !== "functionalImage",
    ),
  );
  return { ...entry, reuseInputs };
}

function sameReuseInputs(left: ReuseInputs | undefined, right: ReuseInputs | undefined): boolean {
  return REUSE_INPUT_KEYS.every((key) => (left?.[key] || "") === (right?.[key] || ""));
}

function reuseInputsKey(inputs: ReuseInputs | undefined): string {
  return JSON.stringify(REUSE_INPUT_KEYS.map((key) => inputs?.[key] || ""));
}

function commonReuseInputs(entries: FailedEntry[]): ReuseInputs {
  const first = entries[0]?.reuseInputs;
  return first && entries.every((entry) => sameReuseInputs(first, entry.reuseInputs)) ? first : {};
}

function groupByReuseInputs(entries: FailedEntry[]): FailedEntry[][] {
  const groups = new Map<string, FailedEntry[]>();
  for (const entry of entries) {
    const key = reuseInputsKey(entry.reuseInputs);
    const group = groups.get(key);
    if (group) {
      group.push(entry);
    } else {
      groups.set(key, [entry]);
    }
  }
  return [...groups.values()];
}

function ghWorkflowCommand(
  lanes: string[],
  ref: string,
  workflow: string,
  reuseInputs: ReuseInputs = {},
): string {
  const workflowRef = process.env.OPENCLAW_DOCKER_E2E_WORKFLOW_REF;
  const releasePath = lanes.some(laneNeedsReleasePath);
  const fields = [
    "gh workflow run",
    shellQuote(workflow),
    ...(workflowRef ? ["--ref", shellQuote(workflowRef)] : []),
    "-f",
    `ref=${shellQuote(ref)}`,
    "-f",
    "include_repo_e2e=false",
    "-f",
    `include_release_path_suites=${releasePath ? "true" : "false"}`,
    "-f",
    "include_openwebui=false",
    "-f",
    `docker_lanes=${shellQuote(lanes.join(" "))}`,
    "-f",
    "include_live_suites=false",
    "-f",
    "live_models_only=false",
  ];
  if (reuseInputs.bareImage) {
    fields.push("-f", `docker_e2e_bare_image=${shellQuote(reuseInputs.bareImage)}`);
  }
  if (reuseInputs.functionalImage) {
    fields.push("-f", `docker_e2e_functional_image=${shellQuote(reuseInputs.functionalImage)}`);
  }
  if (reuseInputs.bareImage || reuseInputs.functionalImage) {
    fields.push("-f", "shared_image_policy=existing-only");
  }
  if (reuseInputs.allowUnreleasedChangelog === "true") {
    fields.push("-f", "allow_unreleased_changelog=true");
  }
  if (reuseInputs.publishedUpgradeSurvivorBaseline) {
    fields.push(
      "-f",
      `published_upgrade_survivor_baseline=${shellQuote(
        reuseInputs.publishedUpgradeSurvivorBaseline,
      )}`,
    );
  }
  if (reuseInputs.publishedUpgradeSurvivorBaselines) {
    fields.push(
      "-f",
      `published_upgrade_survivor_baselines=${shellQuote(
        reuseInputs.publishedUpgradeSurvivorBaselines,
      )}`,
    );
  }
  if (reuseInputs.publishedUpgradeSurvivorScenarios) {
    fields.push(
      "-f",
      `published_upgrade_survivor_scenarios=${shellQuote(
        reuseInputs.publishedUpgradeSurvivorScenarios,
      )}`,
    );
  }
  return fields.join(" ");
}

function failureName(failure: Record<string, unknown>): string {
  if (typeof failure.name === "string" && failure.name) {
    return failure.name;
  }
  return typeof failure.lane === "string" ? failure.lane : "";
}

function failedEntryFromRecord(
  failure: Record<string, unknown>,
  artifactRef: string,
  reuseInputs: ReuseInputs,
) {
  const lane = failureName(failure);
  const targetable = failure.targetable !== false;
  const workflowInputs = {
    ...trustedReuseInputsFromCommand(failure.ghWorkflowCommand),
    ...reuseInputs,
  };
  return {
    artifactRef,
    lane,
    localRerunCommand: typeof failure.rerunCommand === "string" ? failure.rerunCommand : "",
    reuseInputs: workflowInputs,
    targetable,
  };
}

type FailedEntry = ReturnType<typeof failedEntryFromRecord>;

function mergeReuseInputs(left: ReuseInputs = {}, right: ReuseInputs = {}): ReuseInputs {
  const merged: ReuseInputs = { ...left };
  for (const key of REUSE_INPUT_KEYS) {
    const value = right[key];
    if (value) {
      merged[key] = value;
    }
  }
  return merged;
}

function detectRepo(): string {
  return run("gh", ["repo", "view", "--json", "nameWithOwner", "--jq", ".nameWithOwner"]).trim();
}

function findFiles(rootDir: string, basenames: Set<string>, out: string[] = []): string[] {
  for (const entry of fs.readdirSync(rootDir, { withFileTypes: true })) {
    const file = path.join(rootDir, entry.name);
    if (entry.isDirectory()) {
      findFiles(file, basenames, out);
    } else if (basenames.has(entry.name)) {
      out.push(file);
    }
  }
  return out;
}

function failedLaneEntriesFromJson(file: string, explicitRef = ""): FailedEntry[] {
  const parsed = readJson(file);
  const reuseInputs = reuseInputsFromJson(parsed);
  const source = path.basename(file);
  let failures;
  const lanes = Array.isArray(parsed.lanes) ? parsed.lanes.filter(isRecord) : [];
  if (source === "failures.json") {
    failures = lanes.filter((lane) => failureName(lane));
  } else {
    const recordedFailures = Array.isArray(parsed.failures) ? parsed.failures.filter(isRecord) : [];
    failures =
      recordedFailures.length > 0 ? recordedFailures : lanes.filter((lane) => lane.status !== 0);
    failures = failures.filter((lane) => failureName(lane));
  }
  const needsTargetRef = !explicitRef && failures.some((failure) => failure.targetable !== false);
  const artifactRef = artifactTargetRef(parsed, file, needsTargetRef);
  return failures.map((failure) =>
    discardMismatchedPreparedImages(
      failedEntryFromRecord(failure, artifactRef, reuseInputs),
      explicitRef,
    ),
  );
}

function mergeArtifactRefs(left: string, right: string, lane: string): string {
  if (left && right && left !== right) {
    throw new Error(`lane ${lane} has mixed artifact target refs: ${left}, ${right}`);
  }
  return left || right || "";
}

function mergeByLane(entries: FailedEntry[], explicitRef = ""): FailedEntry[] {
  const byLane = new Map<string, FailedEntry>();
  for (const entry of entries) {
    const existing = byLane.get(entry.lane);
    if (existing) {
      byLane.set(entry.lane, {
        ...existing,
        ...entry,
        artifactRef:
          explicitRef || mergeArtifactRefs(existing.artifactRef, entry.artifactRef, entry.lane),
        localRerunCommand: existing.localRerunCommand || entry.localRerunCommand,
        reuseInputs: mergeReuseInputs(existing.reuseInputs, entry.reuseInputs),
        targetable: existing.targetable && entry.targetable,
      });
    } else {
      byLane.set(entry.lane, { ...entry, artifactRef: explicitRef || entry.artifactRef });
    }
  }
  return [...byLane.values()].toSorted((left, right) => left.lane.localeCompare(right.lane));
}

function resolveTargetRef(entries: FailedEntry[], explicitRef: string): string {
  const targetable = entries.filter((entry) => entry.targetable);
  if (targetable.length === 0) {
    return "";
  }
  if (explicitRef) {
    if (!/^[a-f0-9]{40}$/u.test(explicitRef)) {
      throw new Error("--ref must be the exact lowercase 40-character target SHA");
    }
    return explicitRef;
  }
  const missing = targetable.filter((entry) => !entry.artifactRef);
  if (missing.length > 0) {
    throw new Error(
      `Docker E2E artifacts are missing an exact target ref for: ${missing.map((entry) => entry.lane).join(", ")}; pass --ref explicitly`,
    );
  }
  const refs = [...new Set(targetable.map((entry) => entry.artifactRef).filter(Boolean))];
  if (refs.length > 1) {
    throw new Error(`Docker E2E artifacts contain mixed target refs: ${refs.join(", ")}`);
  }
  return refs[0] || "";
}

function downloadDockerArtifacts(runId: string, repo: string, outputDir: string): string[] {
  fs.mkdirSync(outputDir, { recursive: true });
  const payload: unknown = JSON.parse(
    run("gh", [
      "api",
      `repos/${repo}/actions/runs/${runId}/artifacts?per_page=100`,
      "--jq",
      ".artifacts",
    ]),
  );
  const artifacts = Array.isArray(payload) ? payload.filter(isRecord) : [];
  const names = artifacts.flatMap((artifact) =>
    artifact.expired !== true &&
    typeof artifact.name === "string" &&
    artifact.name.startsWith("docker-e2e-")
      ? [artifact.name]
      : [],
  );
  if (names.length === 0) {
    throw new Error(`No docker-e2e-* artifacts found for run ${runId}`);
  }
  for (const [index, name] of names.entries()) {
    const artifactDir = path.join(
      outputDir,
      `${String(index).padStart(3, "0")}-${safePathSegment(name)}`,
    );
    run("gh", ["run", "download", runId, "--repo", repo, "--name", name, "--dir", artifactDir], {
      stdio: "inherit",
    });
  }
  return names;
}

function runInfo(runId: string, repo: string) {
  const value: unknown = JSON.parse(
    run("gh", [
      "run",
      "view",
      runId,
      "--repo",
      repo,
      "--json",
      "databaseId,headSha,headBranch,status,conclusion,url,workflowName",
    ]),
  );
  return {
    url: isRecord(value) && typeof value.url === "string" ? value.url : undefined,
    workflowName:
      isRecord(value) && typeof value.workflowName === "string" ? value.workflowName : undefined,
  };
}

function safePathSegment(value: string): string {
  return (
    value
      .replace(/[^a-zA-Z0-9_.-]+/gu, "-")
      .replace(/^-+|-+$/gu, "")
      .slice(0, 80) || "run"
  );
}

function defaultOutputDir(input: string): string {
  return fs.mkdtempSync(
    path.join(os.tmpdir(), `openclaw-docker-e2e-rerun-${safePathSegment(input)}-`),
  );
}

function printEntries(
  entries: FailedEntry[],
  ref: string,
  workflow: string,
  runValue?: ReturnType<typeof runInfo>,
): void {
  if (runValue) {
    console.log(`Run: ${runValue.url}`);
    console.log(`Workflow: ${runValue.workflowName}`);
  }
  console.log(`Ref: ${ref}`);
  console.log(
    "Targeted GitHub reruns repack the exact artifact target and reuse GHCR-backed prepared image refs when the downloaded artifacts expose them.",
  );
  if (entries.length === 0) {
    console.log("No failed Docker E2E lanes found.");
    return;
  }
  const workflowEntries = entries.filter((entry) => entry.targetable);
  console.log(`Failed Docker E2E entries: ${entries.map((entry) => entry.lane).join(", ")}`);
  if (workflowEntries.length > 0) {
    console.log("");
    const workflowGroups = groupByReuseInputs(workflowEntries);
    if (workflowGroups.length === 1) {
      console.log("Combined GitHub rerun:");
      console.log(
        ghWorkflowCommand(
          workflowEntries.map((entry) => entry.lane),
          ref,
          workflow,
          commonReuseInputs(workflowEntries),
        ),
      );
    } else {
      console.log("Combined GitHub reruns:");
      for (const group of workflowGroups) {
        const lanes = group.map((entry) => entry.lane);
        console.log(
          `- ${lanes.join(", ")}: ${ghWorkflowCommand(lanes, ref, workflow, group[0]?.reuseInputs)}`,
        );
      }
    }
    console.log("");
    console.log("Per-lane GitHub reruns:");
    for (const entry of workflowEntries) {
      console.log(
        `- ${entry.lane}: ${ghWorkflowCommand([entry.lane], ref, workflow, entry.reuseInputs)}`,
      );
    }
  } else {
    console.log("");
    console.log("No targetable failed Docker E2E lanes found.");
  }
  console.log("");
  console.log("Local rerun starting points:");
  for (const entry of entries) {
    if (entry.localRerunCommand) {
      console.log(`- ${entry.lane}: ${entry.localRerunCommand}`);
    }
  }
}

function main(): void {
  const options = parseArgs(process.argv.slice(2));
  if (options.help) {
    console.log(usage());
    return;
  }
  const isLocalJson = fs.existsSync(options.input) && fs.statSync(options.input).isFile();
  if (isLocalJson) {
    const entries = mergeByLane(failedLaneEntriesFromJson(options.input, options.ref), options.ref);
    const ref = resolveTargetRef(entries, options.ref);
    printEntries(entries, ref, options.workflow);
  } else {
    const repo = options.repo || detectRepo();
    const runLocal = runInfo(options.input, repo);
    const outputDir = options.dir || defaultOutputDir(options.input);
    const artifactNames = downloadDockerArtifacts(options.input, repo, outputDir);
    const files = findFiles(outputDir, new Set(["failures.json", "summary.json"]));
    const entries = mergeByLane(
      files.flatMap((file) => failedLaneEntriesFromJson(file, options.ref)),
      options.ref,
    );
    const ref = resolveTargetRef(entries, options.ref);
    console.log(`Artifacts: ${artifactNames.join(", ")}`);
    console.log(`Downloaded: ${outputDir}`);
    printEntries(entries, ref, options.workflow, runLocal);
  }
}

try {
  main();
} catch (error) {
  console.error(error instanceof Error ? error.message : String(error));
  process.exit(1);
}