// Docker E2E aggregate scheduler.
// Builds shared Docker images, prepares one OpenClaw npm tarball, assigns lanes
// to bare/functional images, and runs lanes through weighted resource pools.

import assert from "node:assert/strict";
import { execFileSync, spawn, type ChildProcess } from "node:child_process";
import fs from "node:fs";
import { mkdir, open, readFile } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { isRecord } from "@openclaw/normalization-core/record-coerce";
import {
  DEFAULT_E2E_BARE_IMAGE,
  DEFAULT_E2E_FUNCTIONAL_IMAGE,
  DEFAULT_LIVE_RETRIES,
  DEFAULT_PARALLELISM,
  DEFAULT_PROFILE,
  DEFAULT_RESOURCE_LIMITS,
  DEFAULT_TAIL_PARALLELISM,
  RELEASE_PATH_PROFILE,
  findLaneByName,
  laneResources,
  laneSummary,
  laneWeight,
  lanesNeedE2eImageKind,
  lanesNeedOpenClawPackage,
  normalizeReleaseProfile,
  parseLaneSelection,
  parseLiveMode,
  parseProfile,
  resolveDockerE2ePlan,
} from "./lib/docker-e2e-plan.mts";
import type { DockerE2eLane } from "./lib/docker-e2e-scenarios.mts";
import { sleep } from "./lib/sleep.mjs";
import {
  createPrepublishPluginRegistryArtifact,
  inspectNpmPackageTarball,
  validatePrepublishPluginRegistryArtifact,
} from "./prepublish-plugin-registry-artifact.mjs";

const SCRIPT_ROOT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const ROOT_DIR = path.resolve(process.env.OPENCLAW_DOCKER_E2E_REPO_ROOT || SCRIPT_ROOT_DIR);
const LIVE_DOCKER_DEFAULT_HARNESS_DIR =
  path.basename(SCRIPT_ROOT_DIR) === ".release-harness" && ROOT_DIR !== SCRIPT_ROOT_DIR
    ? ".release-harness"
    : ".";
const DEFAULT_FAILURE_TAIL_LINES = 80;
const DEFAULT_LANE_TIMEOUT_MS = 120 * 60 * 1000;
const DEFAULT_LANE_START_STAGGER_MS = 2_000;
const DEFAULT_STATUS_INTERVAL_MS = 30_000;
const DEFAULT_PREFLIGHT_RUN_TIMEOUT_MS = 60_000;
const CLEANUP_SMOKE_NAME = "cleanup-smoke";
export const SHELL_CAPTURE_MAX_CHARS = 1024 * 1024;
export const LOG_TAIL_MAX_BYTES = 1024 * 1024;
const SHELL_TIMEOUT_KILL_GRACE_MS = 10_000;
const SHELL_POST_FORCE_KILL_WAIT_MS = 1_000;
const SHELL_PROCESS_GROUP_EXIT_POLL_MS = 25;
const MAX_TIMER_TIMEOUT_MS = 2_147_000_000;
const DEFAULT_TIMINGS_FILE = path.join(ROOT_DIR, ".artifacts/docker-tests/lane-timings.json");
const DEFAULT_GITHUB_WORKFLOW = "openclaw-live-and-e2e-checks-reusable.yml";
const CANDIDATE_ENV_KEYS =
  "OPENCLAW_DOCKER_E2E_SELECTED_SHA OPENCLAW_CURRENT_PACKAGE_TGZ OPENCLAW_CURRENT_PACKAGE_VERSION OPENCLAW_CURRENT_PACKAGE_SHA256".split(
    " ",
  );
const REGISTRY_ENV_KEYS =
  "OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_CANDIDATE_VERSION OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_MANIFEST_SHA256".split(
    " ",
  );

type SchedulerLimits = ReturnType<typeof parseSchedulerOptions>;
type DockerCandidatePlan = ReturnType<typeof resolveDockerE2ePlan>["plan"];

type SchedulerActiveState = {
  count: number;
  resources: Map<string, number>;
  weight: number;
};

type SchedulerLane = Pick<DockerE2eLane, "name"> &
  Partial<Pick<DockerE2eLane, "resources" | "weight">>;

type TimingStore = Awaited<ReturnType<typeof loadTimingStore>>;

type ShellCommandResult = Omit<ReturnType<typeof shellCommandSkippedForShutdown>, "signal"> & {
  signal: NodeJS.Signals | null;
};
type ShellCaptureResult = Omit<ReturnType<typeof shellCaptureSkippedForShutdown>, "signal"> & {
  signal: NodeJS.Signals | null;
};

type ShellCommandOptions = {
  command: string;
  env: NodeJS.ProcessEnv;
  label: string;
  logFile?: string;
  noOutputTimeoutMs?: number;
  timeoutKillGraceMs?: number;
  timeoutMs?: number;
};

type ShellCaptureOptions = Omit<ShellCommandOptions, "logFile" | "noOutputTimeoutMs"> &
  Required<Pick<ShellCommandOptions, "timeoutMs">>;

type LaneResult = Omit<Awaited<ReturnType<typeof runLane>>, "imageKind"> & {
  imageKind?: DockerE2eLane["e2eImageKind"];
  targetable?: boolean;
};

type RunLaneSummary = Pick<LaneResult, "name" | "status"> & Partial<LaneResult>;

type RunSummary = Record<string, unknown> & {
  failures?: RunLaneSummary[];
  github?: ReturnType<typeof githubRunSummary>;
  images?: { bare?: string; functional?: string };
  lanes?: RunLaneSummary[];
  status?: string;
};

type LanePoolOptions = SchedulerLimits & {
  failFast: boolean;
  poolLabel: string;
  startStaggerMs: number;
  statusIntervalMs: number;
  timeoutMs: number;
};

type ForegroundEntry = {
  command: string;
  env?: NodeJS.ProcessEnv;
  label: string;
  phaseDetails?: Record<string, unknown>;
  phases?: Array<Record<string, unknown>>;
};

type ShutdownSignal = "SIGINT" | "SIGKILL" | "SIGTERM";

const IS_MAIN = (() => {
  const entrypoint = process.argv[1];
  if (!entrypoint) {
    return false;
  }
  try {
    // Node resolves ESM URLs through symlinks, but argv keeps the invoked path.
    return fs.realpathSync(entrypoint) === fs.realpathSync(fileURLToPath(import.meta.url));
  } catch {
    return false;
  }
})();

function dockerAllUsage() {
  return [
    "Usage: node scripts/test-docker-all.mjs [--plan-json | --prepare-only=<manifest>]",
    "",
    "Options:",
    "  --plan-json    Print the resolved Docker E2E plan as JSON and exit.",
    "  --prepare-only Prepare one immutable candidate manifest and exit.",
    "  -h, --help     Show this help.",
    "",
    "Lane selection and scheduler settings are configured with OPENCLAW_DOCKER_ALL_* env vars.",
  ].join("\n");
}

export function parseDockerAllCliArgs(argv: readonly string[]) {
  const options: { help: boolean; planJson: boolean; prepareOnly?: string } = {
    help: false,
    planJson: false,
  };
  for (const arg of argv) {
    if (arg === "--plan-json") {
      options.planJson = true;
    } else if (arg.startsWith("--prepare-only=")) {
      options.prepareOnly = arg.slice("--prepare-only=".length);
      if (!options.prepareOnly) {
        throw new Error(`--prepare-only requires a manifest path\n\n${dockerAllUsage()}`);
      }
    } else if (arg === "--help" || arg === "-h") {
      options.help = true;
    } else {
      throw new Error(`unknown argument: ${arg}\n\n${dockerAllUsage()}`);
    }
  }
  assert(!(options.planJson && options.prepareOnly), "conflicting plan/prep options");
  return options;
}

let cliOptions: ReturnType<typeof parseDockerAllCliArgs> = {
  help: false,
  planJson: false,
};
if (IS_MAIN) {
  try {
    cliOptions = parseDockerAllCliArgs(process.argv.slice(2));
  } catch (error) {
    console.error(error instanceof Error ? error.message : String(error));
    process.exit(1);
  }
  if (cliOptions.help) {
    console.log(dockerAllUsage());
    process.exit(0);
  }
}

function parsePositiveInt(raw: string | undefined, fallback: number, label: string) {
  if (raw === undefined || raw === "") {
    return fallback;
  }
  const text = raw.trim();
  if (!/^\d+$/u.test(text)) {
    throw new Error(`${label} must be a positive integer. Got: ${JSON.stringify(raw)}`);
  }
  const parsed = Number(text);
  if (!Number.isSafeInteger(parsed) || parsed < 1) {
    throw new Error(`${label} must be a positive integer. Got: ${JSON.stringify(raw)}`);
  }
  return parsed;
}

function parseNonNegativeInt(raw: string | undefined, fallback: number, label: string) {
  if (raw === undefined || raw === "") {
    return fallback;
  }
  const text = raw.trim();
  if (!/^\d+$/u.test(text)) {
    throw new Error(`${label} must be a non-negative integer. Got: ${JSON.stringify(raw)}`);
  }
  const parsed = Number(text);
  if (!Number.isSafeInteger(parsed) || parsed < 0) {
    throw new Error(`${label} must be a non-negative integer. Got: ${JSON.stringify(raw)}`);
  }
  return parsed;
}

function parseBool(raw: string | undefined, fallback: boolean) {
  if (raw === undefined || raw === "") {
    return fallback;
  }
  return !/^(?:0|false|no)$/i.test(raw);
}

function normalizeReleaseProfileEnv(raw: string | undefined) {
  const profile = raw?.trim();
  if (!profile) {
    return normalizeReleaseProfile(undefined);
  }
  if (profile === "minimum" || profile === "beta" || profile === "stable" || profile === "full") {
    return normalizeReleaseProfile(profile);
  }
  throw new Error(
    `release profile must be one of: beta, stable, full. Got: ${JSON.stringify(raw)}`,
  );
}

function numericTimerValueMs(valueMs: unknown) {
  const value = Number(valueMs);
  return Number.isFinite(value) ? Math.floor(value) : undefined;
}

function resolveDockerSchedulerTimeoutMs(
  valueMs: unknown,
  fallbackMs: unknown = MAX_TIMER_TIMEOUT_MS,
) {
  const value = numericTimerValueMs(valueMs) ?? numericTimerValueMs(fallbackMs);
  return Math.min(Math.max(value ?? MAX_TIMER_TIMEOUT_MS, 1), MAX_TIMER_TIMEOUT_MS);
}

function resolveOptionalTimerTimeoutMs(valueMs: unknown) {
  const value = numericTimerValueMs(valueMs);
  if (value === undefined || value <= 0) {
    return undefined;
  }
  return resolveDockerSchedulerTimeoutMs(value);
}

function resourceLimitsSummary(resourceLimits: Record<string, number>) {
  return Object.entries(resourceLimits)
    .map(([resource, limit]) => `${resource}=${String(limit)}`)
    .join(" ");
}

export function describeDockerSchedulerLimits(parallelism: number, options: SchedulerLimits) {
  return `parallelism=${parallelism} weightLimit=${options.weightLimit} resources=${resourceLimitsSummary(
    options.resourceLimits,
  )}`;
}

function parseSchedulerOptions(env: NodeJS.ProcessEnv, parallelism: number) {
  const weightLimit = parsePositiveInt(
    env.OPENCLAW_DOCKER_ALL_WEIGHT_LIMIT,
    parallelism,
    "OPENCLAW_DOCKER_ALL_WEIGHT_LIMIT",
  );
  const resourceLimits: Record<string, number> = {};
  for (const [resource, fallback] of Object.entries(DEFAULT_RESOURCE_LIMITS)) {
    const envName = `OPENCLAW_DOCKER_ALL_${resource.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_LIMIT`;
    resourceLimits[resource] = parsePositiveInt(
      env[envName],
      Math.min(parallelism, fallback),
      envName,
    );
  }
  return {
    resourceLimits,
    weightLimit,
  };
}

export function canStartSchedulerLane(
  candidate: SchedulerLane,
  active: SchedulerActiveState,
  parallelism: number,
  options: SchedulerLimits,
) {
  const weight = Math.max(1, candidate.weight ?? 1);
  if (active.count >= parallelism) {
    return false;
  }

  const exceedsWeightLimit = active.weight + weight > options.weightLimit;
  const resources = [...new Set(["docker", ...(candidate.resources ?? [])])];
  const exceedsResourceLimit = resources.some((resource) => {
    const limit = options.resourceLimits[resource] ?? options.weightLimit;
    const current = active.resources.get(resource) ?? 0;
    return current + weight > limit;
  });

  if (!exceedsWeightLimit && !exceedsResourceLimit) {
    return true;
  }

  return active.count === 0;
}

function timingSeconds(timings: Record<string, unknown>, poolLane: DockerE2eLane) {
  const timing = timings[poolLane.name];
  const fromStore = isRecord(timing) ? timing.durationSeconds : undefined;
  if (typeof fromStore === "number" && Number.isFinite(fromStore) && fromStore > 0) {
    return fromStore;
  }
  return poolLane.estimateSeconds ?? 0;
}

function orderLanes(poolLanes: DockerE2eLane[], timingStore?: unknown) {
  const timings = isRecord(timingStore) && isRecord(timingStore.lanes) ? timingStore.lanes : {};
  return poolLanes
    .map((poolLane, index) => ({
      index,
      poolLane,
      seconds: timingSeconds(timings, poolLane),
    }))
    .toSorted((a, b) => b.seconds - a.seconds || a.index - b.index)
    .map(({ poolLane }) => poolLane);
}

function utcStampForPath() {
  return new Date().toISOString().replaceAll("-", "").replaceAll(":", "").replace(/\..*$/, "Z");
}

function utcStamp() {
  return new Date().toISOString().replace(/\..*$/, "Z");
}

function appendExtension(env: NodeJS.ProcessEnv, extension: string) {
  const current = env.OPENCLAW_DOCKER_BUILD_EXTENSIONS ?? env.OPENCLAW_EXTENSIONS ?? "";
  const tokens = current.split(/\s+/).filter(Boolean);
  if (!tokens.includes(extension)) {
    tokens.push(extension);
  }
  env.OPENCLAW_DOCKER_BUILD_EXTENSIONS = tokens.join(" ");
}

function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv {
  const env = {
    ...process.env,
    ...extra,
  };
  const pathEntries = [
    env.PATH,
    env.PNPM_HOME,
    env.npm_execpath ? path.dirname(env.npm_execpath) : undefined,
    path.dirname(process.execPath),
  ]
    .flatMap((entry) => (entry ? entry.split(path.delimiter) : []))
    .filter(Boolean);
  env.PATH = [...new Set(pathEntries)].join(path.delimiter);
  return env;
}

function gitOutput(repoRoot: string, args: string[]) {
  return execFileSync("git", args, { cwd: repoRoot, encoding: "utf8" }).trim();
}

function rootPackageVersion(repoRoot: string) {
  return JSON.parse(fs.readFileSync(path.join(repoRoot, "package.json"), "utf8")).version as string;
}

function readCompleteTuple(env: NodeJS.ProcessEnv, keys: readonly string[], label: string) {
  const entries = keys.flatMap((key) => (env[key] ? [[key, env[key]]] : []));
  const complete = entries.length === keys.length;
  assert(!entries.length || complete, `${label} fields must be complete`);
  return complete ? Object.fromEntries(entries) : undefined;
}

function validateRegistryEnvironment(baseEnv: NodeJS.ProcessEnv, plan: DockerCandidatePlan) {
  validatePrepublishPluginRegistryArtifact({
    artifactDir: baseEnv.OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR!,
    expectedCandidateVersion: baseEnv.OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_CANDIDATE_VERSION!,
    expectedManifestSha256: baseEnv.OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_MANIFEST_SHA256!,
    expectedSourceSha: baseEnv.OPENCLAW_DOCKER_E2E_SELECTED_SHA!,
    requiredPackages: plan.requiredPrepublishPluginPackages,
  });
}

export function validateDockerCandidateEnvironment(
  baseEnv: NodeJS.ProcessEnv,
  plan: DockerCandidatePlan,
  repoRoot = ROOT_DIR,
) {
  const strictCandidate = CANDIDATE_ENV_KEYS.slice(2).some((key) => baseEnv[key]);
  if (!strictCandidate || !plan.needs.package) {
    baseEnv.OPENCLAW_CURRENT_PACKAGE_TGZ &&= path.resolve(baseEnv.OPENCLAW_CURRENT_PACKAGE_TGZ);
    const registryDir = baseEnv.OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR;
    if (!plan.needs.prepublishPluginRegistry || !registryDir) {
      delete baseEnv.OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR;
      return;
    }
    baseEnv.OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR = path.resolve(registryDir);
    validateRegistryEnvironment(baseEnv, plan);
    return;
  }
  const candidate = readCompleteTuple(baseEnv, CANDIDATE_ENV_KEYS, "Docker candidate")!;
  const registry = readCompleteTuple(baseEnv, REGISTRY_ENV_KEYS, "Docker candidate registry");
  const packagePath = candidate.OPENCLAW_CURRENT_PACKAGE_TGZ;
  if (
    !path.isAbsolute(packagePath) ||
    gitOutput(repoRoot, ["rev-parse", "HEAD"]) !== candidate.OPENCLAW_DOCKER_E2E_SELECTED_SHA
  ) {
    throw new Error("Docker candidate path must be absolute and selected SHA must equal HEAD");
  }
  const packed = inspectNpmPackageTarball(packagePath);
  if (
    packed.packageJson.name !== "openclaw" ||
    packed.packageJson.version !== rootPackageVersion(repoRoot) ||
    packed.packageJson.version !== candidate.OPENCLAW_CURRENT_PACKAGE_VERSION ||
    packed.sha256 !== candidate.OPENCLAW_CURRENT_PACKAGE_SHA256
  ) {
    throw new Error("Docker candidate package identity differs from the immutable tuple");
  }
  if (registry) {
    const registryDir = registry.OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR;
    assert(path.isAbsolute(registryDir), "Docker candidate registry path must be absolute");
    validateRegistryEnvironment(baseEnv, plan);
  } else if (plan.needs.prepublishPluginRegistry) {
    throw new Error("Docker plan requires a prepublish plugin registry tuple");
  }
}

function shellQuote(value: string) {
  return `'${value.replaceAll("'", "'\\''")}'`;
}

function maybeGhcrImage(value: unknown) {
  return typeof value === "string" && value.startsWith("ghcr.io/") ? value : "";
}

export function githubWorkflowRerunCommand(
  laneNames: readonly string[],
  ref: string,
  env: NodeJS.ProcessEnv = process.env,
) {
  const workflowRef = env.OPENCLAW_DOCKER_E2E_WORKFLOW_REF || undefined;
  const releasePath = env.OPENCLAW_DOCKER_ALL_PROFILE === RELEASE_PATH_PROFILE;
  const allowUnreleasedChangelog = env.OPENCLAW_DOCKER_E2E_ALLOW_UNRELEASED_CHANGELOG === "true";
  const bareImage = maybeGhcrImage(env.OPENCLAW_DOCKER_E2E_BARE_IMAGE);
  const functionalImage = maybeGhcrImage(env.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE);
  const fields = [
    "gh workflow run",
    shellQuote(env.OPENCLAW_DOCKER_E2E_WORKFLOW || DEFAULT_GITHUB_WORKFLOW),
    ...(workflowRef ? ["--ref", shellQuote(workflowRef)] : []),
    "-f",
    `ref=${shellQuote(ref)}`,
    "-f",
    "include_repo_e2e=false",
    "-f",
    `include_release_path_suites=${releasePath ? "true" : "false"}`,
    "-f",
    "include_openwebui=false",
    "-f",
    `docker_lanes=${shellQuote(laneNames.join(" "))}`,
    "-f",
    "include_live_suites=false",
    "-f",
    "live_models_only=false",
  ];
  if (allowUnreleasedChangelog) {
    fields.push("-f", "allow_unreleased_changelog=true");
  }
  if (env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC) {
    fields.push(
      "-f",
      `published_upgrade_survivor_baseline=${shellQuote(env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC)}`,
    );
  }
  if (env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS) {
    fields.push(
      "-f",
      `published_upgrade_survivor_baselines=${shellQuote(env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS)}`,
    );
  }
  if (env.OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS) {
    fields.push(
      "-f",
      `published_upgrade_survivor_scenarios=${shellQuote(env.OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS)}`,
    );
  }
  if (bareImage) {
    fields.push("-f", `docker_e2e_bare_image=${shellQuote(bareImage)}`);
  }
  if (functionalImage) {
    fields.push("-f", `docker_e2e_functional_image=${shellQuote(functionalImage)}`);
  }
  if (bareImage || functionalImage) {
    fields.push("-f", "shared_image_policy=existing-only");
  }
  return fields.join(" ");
}

export function buildLaneRerunCommand(name: string, baseEnv: NodeJS.ProcessEnv) {
  const poolLane = findLaneByName(name);
  const build = name.startsWith("live-") ? "1" : "0";
  const image = poolLane ? e2eImageForLane(poolLane, baseEnv) : baseEnv.OPENCLAW_DOCKER_E2E_IMAGE;
  const env: Array<readonly [string, string | undefined]> = [
    ["OPENCLAW_DOCKER_ALL_LANES", name],
    ["OPENCLAW_DOCKER_ALL_BUILD", build],
    ["OPENCLAW_DOCKER_ALL_PREFLIGHT", "0"],
    ["OPENCLAW_SKIP_DOCKER_BUILD", "1"],
    ["OPENCLAW_DOCKER_E2E_IMAGE", image || DEFAULT_E2E_FUNCTIONAL_IMAGE],
    ["OPENCLAW_DOCKER_E2E_BARE_IMAGE", baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE],
    ["OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE", baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE],
    ...CANDIDATE_ENV_KEYS.map((key) => [key, baseEnv[key]] as const),
    ...REGISTRY_ENV_KEYS.map((key) => [key, baseEnv[key]] as const),
    ["OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC", baseEnv.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC],
    ["OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS", baseEnv.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS],
    ["OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS", baseEnv.OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS],
  ];
  if (baseEnv.OPENCLAW_DOCKER_ALL_PNPM_COMMAND) {
    env.push(["OPENCLAW_DOCKER_ALL_PNPM_COMMAND", baseEnv.OPENCLAW_DOCKER_ALL_PNPM_COMMAND]);
  }
  return `${env
    .filter(
      (entry): entry is readonly [string, string] => entry[1] !== undefined && entry[1] !== "",
    )
    .map(([key, value]) => `${key}=${shellQuote(value)}`)
    .join(" ")} pnpm test:docker:all`;
}

function liveDockerHarnessScriptCommand(script: string) {
  return `bash -c 'harness="\${OPENCLAW_DOCKER_E2E_TRUSTED_HARNESS_DIR:-${LIVE_DOCKER_DEFAULT_HARNESS_DIR}}"; OPENCLAW_LIVE_DOCKER_REPO_ROOT="\${OPENCLAW_DOCKER_E2E_REPO_ROOT:-$PWD}" bash "$harness/scripts/${script}"'`;
}

async function loadTimingStore(file: string, enabled: boolean) {
  if (!enabled) {
    return { enabled: false, file, lanes: {}, version: 1 };
  }
  const raw = await readFile(file, "utf8").catch(() => "");
  if (!raw.trim()) {
    return { enabled: true, file, lanes: {}, version: 1 };
  }
  try {
    const parsed: unknown = JSON.parse(raw);
    return {
      enabled: true,
      file,
      lanes: isRecord(parsed) && isRecord(parsed.lanes) ? parsed.lanes : {},
      version: 1,
    };
  } catch (error) {
    console.warn(
      `WARN: ignoring unreadable Docker lane timings ${file}: ${
        error instanceof Error ? error.message : String(error)
      }`,
    );
    return { enabled: true, file, lanes: {}, version: 1 };
  }
}

async function writeTimingStore(timingStore: TimingStore, results: LaneResult[]) {
  if (!timingStore.enabled || results.length === 0) {
    return;
  }
  const next = {
    lanes: { ...timingStore.lanes },
    updatedAt: new Date().toISOString(),
    version: 1,
  };
  for (const result of results) {
    if (!result || typeof result.elapsedSeconds !== "number") {
      continue;
    }
    next.lanes[result.name] = {
      durationSeconds: result.elapsedSeconds,
      status: result.status,
      timedOut: result.timedOut,
      updatedAt: new Date().toISOString(),
    };
  }
  await mkdir(path.dirname(timingStore.file), { recursive: true });
  await fs.promises.writeFile(timingStore.file, `${JSON.stringify(next, null, 2)}\n`);
  timingStore.lanes = next.lanes;
  console.log(`==> Docker lane timings: ${timingStore.file}`);
}

function githubRunSummary(env: NodeJS.ProcessEnv) {
  return {
    ref: env.GITHUB_REF_NAME || undefined,
    repository: env.GITHUB_REPOSITORY || undefined,
    runId: env.GITHUB_RUN_ID || undefined,
    runUrl:
      env.GITHUB_SERVER_URL && env.GITHUB_REPOSITORY && env.GITHUB_RUN_ID
        ? `${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}`
        : undefined,
    selectedSha: env.OPENCLAW_DOCKER_E2E_SELECTED_SHA || undefined,
    sha: env.GITHUB_SHA || undefined,
    workflow: env.GITHUB_WORKFLOW || undefined,
  };
}

export async function writeRunSummary(
  logDir: string,
  summary: RunSummary,
  env: NodeJS.ProcessEnv = process.env,
) {
  const file = path.join(logDir, "summary.json");
  const payload = {
    ...summary,
    // Summary reruns do not carry failure-index commands, so preserve this exact package intent.
    allowUnreleasedChangelog:
      env.OPENCLAW_DOCKER_E2E_ALLOW_UNRELEASED_CHANGELOG === "true" ? true : undefined,
    packageArtifactName: env.OPENCLAW_DOCKER_E2E_PACKAGE_ARTIFACT_NAME || undefined,
    finishedAt: new Date().toISOString(),
    github: githubRunSummary(env),
    version: 1,
  };
  await fs.promises.writeFile(file, `${JSON.stringify(payload, null, 2)}\n`);
  await writeFailureIndex(logDir, payload, env);
  console.log(`==> Docker run summary: ${file}`);
}

async function writeFailureIndex(logDir: string, summary: RunSummary, env: NodeJS.ProcessEnv) {
  const ref =
    summary.github?.selectedSha ||
    env.OPENCLAW_DOCKER_E2E_SELECTED_SHA ||
    summary.github?.sha ||
    summary.github?.ref ||
    env.GITHUB_SHA ||
    "HEAD";
  const failures = Array.isArray(summary.failures)
    ? summary.failures
    : (summary.lanes ?? []).filter((lane) => lane.status !== 0);
  const workflowRerunFailures = failures.filter((failure) => failure.targetable !== false);
  const lanes = failures.map((failure) => ({
    ghWorkflowCommand:
      failure.targetable === false
        ? undefined
        : githubWorkflowRerunCommand([failure.name], ref, env),
    image: failure.image,
    imageKind: failure.imageKind,
    lane: failure.name,
    logFile: failure.logFile,
    name: failure.name,
    noOutputTimedOut: failure.noOutputTimedOut,
    rerunCommand: failure.rerunCommand,
    status: failure.status,
    targetable: failure.targetable,
    timedOut: failure.timedOut,
  }));
  const failureIndex = {
    combinedGhWorkflowCommand:
      workflowRerunFailures.length > 0
        ? githubWorkflowRerunCommand(
            workflowRerunFailures.map((failure) => failure.name),
            ref,
            env,
          )
        : undefined,
    generatedAt: new Date().toISOString(),
    lanes,
    note: "Targeted GitHub reruns repack the exact selected ref and reuse only GHCR-backed shared images when the generated command includes docker_e2e_*_image inputs.",
    images: summary.images,
    packageArtifactName: env.OPENCLAW_DOCKER_E2E_PACKAGE_ARTIFACT_NAME || undefined,
    ref,
    runUrl: summary.github?.runUrl,
    status: summary.status,
    version: 1,
    workflow: env.OPENCLAW_DOCKER_E2E_WORKFLOW || DEFAULT_GITHUB_WORKFLOW,
  };
  await fs.promises.writeFile(
    path.join(logDir, "failures.json"),
    `${JSON.stringify(failureIndex, null, 2)}\n`,
  );
}

function phaseElapsedSeconds(startedAtMs: number) {
  return Math.round((Date.now() - startedAtMs) / 1000);
}

function laneAttempt(
  attempt: number,
  startedAtMs: number,
  result: Pick<ShellCommandResult, "noOutputTimedOut" | "status" | "timedOut">,
) {
  return {
    attempt,
    elapsedSeconds: phaseElapsedSeconds(startedAtMs),
    finishedAt: new Date().toISOString(),
    noOutputTimedOut: result.noOutputTimedOut,
    startedAt: new Date(startedAtMs).toISOString(),
    status: result.status,
    timedOut: result.timedOut,
  };
}

function cleanupSmokeResult(
  baseEnv: NodeJS.ProcessEnv,
  logFile: string,
  command: string,
  startedAtMs: number,
  result: Pick<ShellCommandResult, "noOutputTimedOut" | "status" | "timedOut">,
) {
  return {
    command,
    attempts: [laneAttempt(1, startedAtMs, result)],
    elapsedSeconds: phaseElapsedSeconds(startedAtMs),
    finishedAt: new Date().toISOString(),
    image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE,
    logFile,
    name: CLEANUP_SMOKE_NAME,
    noOutputTimedOut: result.noOutputTimedOut,
    rerunCommand: command,
    startedAt: new Date(startedAtMs).toISOString(),
    status: result.status,
    targetable: false,
    timedOut: result.timedOut,
  };
}

async function runPhase<T>(
  phases: Array<Record<string, unknown>>,
  name: string,
  details: Record<string, unknown>,
  fn: () => Promise<T>,
): Promise<T> {
  const startedAtMs = Date.now();
  const startedAt = new Date(startedAtMs).toISOString();
  let status: "failed" | "passed" = "passed";
  let errorMessage: string | undefined;
  try {
    const result = await fn();
    return result;
  } catch (error) {
    status = "failed";
    errorMessage = error instanceof Error ? error.message : String(error);
    throw error;
  } finally {
    const phase = {
      ...details,
      name,
      startedAt,
      status,
      ...(errorMessage === undefined ? {} : { error: errorMessage }),
      elapsedSeconds: phaseElapsedSeconds(startedAtMs),
      finishedAt: new Date().toISOString(),
    };
    phases.push(phase);
    console.log(`==> Phase ${phase.status}: ${name} ${phase.elapsedSeconds}s`);
  }
}

function printLaneManifest(
  label: string,
  poolLanes: DockerE2eLane[],
  timingStore: TimingStore,
): void {
  console.log(`==> ${label} lanes (${poolLanes.length})`);
  for (const [index, poolLane] of poolLanes.entries()) {
    const seconds = timingSeconds(timingStore.lanes, poolLane);
    const estimate = seconds > 0 ? ` last=${Math.round(seconds)}s` : "";
    console.log(`  ${index + 1}. ${laneSummary(poolLane)}${estimate}`);
  }
}

export function dockerPreflightContainerNames(raw: string): string[] {
  return raw.split(/\r?\n/).flatMap((line) => {
    const name = line.trim().split(/\s+/, 1)[0];
    return name &&
      /^(?:openclaw-[a-z0-9-]+-e2e-\d+|openclaw-openwebui(?:-gateway)?-\d+)$/u.test(name)
      ? [name]
      : [];
  });
}

export function resolveDockerPreflightPlatform(arch: NodeJS.Architecture = process.arch) {
  return arch === "arm64" ? "linux/arm64" : "linux/amd64";
}

export function dockerPreflightSmokeCommand(arch: NodeJS.Architecture = process.arch) {
  const platform = resolveDockerPreflightPlatform(arch);
  return `docker run --rm --platform ${shellQuote(platform)} alpine:3.20 true`;
}

export function runShellCommand({
  command,
  env,
  label,
  logFile,
  timeoutMs,
  noOutputTimeoutMs,
  timeoutKillGraceMs = SHELL_TIMEOUT_KILL_GRACE_MS,
}: ShellCommandOptions) {
  if (activeChildrenShutdownPromise) {
    return activeChildrenShutdownPromise.then(() => shellCommandSkippedForShutdown());
  }
  return new Promise<ShellCommandResult>((resolve) => {
    const resolvedTimeoutMs = resolveOptionalTimerTimeoutMs(timeoutMs);
    const resolvedNoOutputTimeoutMs = resolveOptionalTimerTimeoutMs(noOutputTimeoutMs);
    const resolvedTimeoutKillGraceMs = resolveDockerSchedulerTimeoutMs(
      timeoutKillGraceMs,
      SHELL_TIMEOUT_KILL_GRACE_MS,
    );
    const pipeOutput = Boolean(logFile || resolvedNoOutputTimeoutMs);
    const child = spawn("bash", ["-c", command], {
      cwd: ROOT_DIR,
      detached: process.platform !== "win32",
      env,
      stdio: pipeOutput ? ["ignore", "pipe", "pipe"] : "inherit",
    });
    activeChildren.set(child, resolvedTimeoutKillGraceMs);
    let timedOut = false;
    let noOutputTimedOut = false;
    let killTimer: ReturnType<typeof setTimeout> | undefined;
    let killAt: number | undefined;
    const stream = logFile ? fs.createWriteStream(logFile, { flags: "a" }) : undefined;
    let noOutputTimer: ReturnType<typeof setTimeout> | undefined;
    const terminateForTimeout = (message: string, options: { noOutput?: boolean } = {}) => {
      if (timedOut) {
        return;
      }
      timedOut = true;
      noOutputTimedOut = options.noOutput === true;
      if (stream) {
        stream.write(`\n==> [${label}] ${message}; sending SIGTERM\n`);
      } else {
        console.error(`==> [${label}] ${message}; sending SIGTERM`);
      }
      terminateChild(child, "SIGTERM");
      killAt = Date.now() + resolvedTimeoutKillGraceMs;
      killTimer = setTimeout(() => terminateChild(child, "SIGKILL"), resolvedTimeoutKillGraceMs);
      killTimer.unref?.();
    };
    const resetNoOutputTimer = () => {
      if (!resolvedNoOutputTimeoutMs || timedOut) {
        return;
      }
      if (noOutputTimer) {
        clearTimeout(noOutputTimer);
      }
      noOutputTimer = setTimeout(() => {
        terminateForTimeout(`no output for ${resolvedNoOutputTimeoutMs}ms`, { noOutput: true });
      }, resolvedNoOutputTimeoutMs);
      noOutputTimer.unref?.();
    };
    const timeoutTimer =
      resolvedTimeoutMs !== undefined
        ? setTimeout(() => {
            terminateForTimeout(`timeout after ${resolvedTimeoutMs}ms`);
          }, resolvedTimeoutMs)
        : undefined;
    timeoutTimer?.unref?.();

    if (stream) {
      stream.write(`==> [${label}] command: ${command}\n`);
      stream.write(`==> [${label}] started: ${utcStamp()}\n`);
    }
    if (pipeOutput && child.stdout && child.stderr) {
      const writeOutput = (target: NodeJS.WriteStream, chunk: Uint8Array) => {
        resetNoOutputTimer();
        if (stream) {
          stream.write(chunk);
        } else {
          target.write(chunk);
        }
      };
      child.stdout.on("data", (chunk: Uint8Array) => writeOutput(process.stdout, chunk));
      child.stderr.on("data", (chunk: Uint8Array) => writeOutput(process.stderr, chunk));
      resetNoOutputTimer();
    }

    child.on("close", (status, signal) => {
      if (timeoutTimer) {
        clearTimeout(timeoutTimer);
      }
      if (noOutputTimer) {
        clearTimeout(noOutputTimer);
      }
      const finish = () => {
        if (killTimer) {
          clearTimeout(killTimer);
        }
        killAt = undefined;
        activeChildren.delete(child);
        const exitCode = typeof status === "number" ? status : signal ? 128 : 1;
        if (stream) {
          stream.write(
            `\n==> [${label}] finished: ${utcStamp()} status=${exitCode}${
              noOutputTimedOut ? " noOutputTimedOut=true" : ""
            }\n`,
          );
          stream.end();
        }
        resolve({ signal, status: exitCode, timedOut, noOutputTimedOut });
      };
      if (timedOut) {
        void finishTimedOutShellProcessTree(child, killAt, resolvedTimeoutKillGraceMs).then(
          finish,
          finish,
        );
        return;
      }
      finish();
    });
  });
}

export function appendBoundedShellCapture(
  current: string,
  chunk: unknown,
  maxChars = SHELL_CAPTURE_MAX_CHARS,
) {
  const combined = `${current}${String(chunk)}`;
  if (combined.length <= maxChars) {
    return { text: combined, truncated: false };
  }
  return { text: combined.slice(-maxChars), truncated: true };
}

export function runShellCaptureCommand({
  command,
  env,
  label,
  timeoutMs,
  timeoutKillGraceMs = SHELL_TIMEOUT_KILL_GRACE_MS,
}: ShellCaptureOptions) {
  if (activeChildrenShutdownPromise) {
    return activeChildrenShutdownPromise.then(() => shellCaptureSkippedForShutdown(label));
  }
  return new Promise<ShellCaptureResult>((resolve) => {
    const resolvedTimeoutMs = resolveOptionalTimerTimeoutMs(timeoutMs);
    const resolvedTimeoutKillGraceMs = resolveDockerSchedulerTimeoutMs(
      timeoutKillGraceMs,
      SHELL_TIMEOUT_KILL_GRACE_MS,
    );
    const child = spawn("bash", ["-c", command], {
      cwd: ROOT_DIR,
      detached: process.platform !== "win32",
      env,
      stdio: ["ignore", "pipe", "pipe"],
    });
    activeChildren.set(child, resolvedTimeoutKillGraceMs);
    let stdout = "";
    let stderr = "";
    let stdoutTruncated = false;
    let stderrTruncated = false;
    let timedOut = false;
    let killTimer: ReturnType<typeof setTimeout> | undefined;
    let killAt: number | undefined;
    const timeoutTimer =
      resolvedTimeoutMs !== undefined
        ? setTimeout(() => {
            timedOut = true;
            terminateChild(child, "SIGTERM");
            killAt = Date.now() + resolvedTimeoutKillGraceMs;
            killTimer = setTimeout(
              () => terminateChild(child, "SIGKILL"),
              resolvedTimeoutKillGraceMs,
            );
            killTimer.unref?.();
          }, resolvedTimeoutMs)
        : undefined;
    timeoutTimer?.unref?.();
    child.stdout.on("data", (chunk: Buffer) => {
      const next = appendBoundedShellCapture(stdout, chunk);
      stdout = next.text;
      stdoutTruncated ||= next.truncated;
    });
    child.stderr.on("data", (chunk: Buffer) => {
      const next = appendBoundedShellCapture(stderr, chunk);
      stderr = next.text;
      stderrTruncated ||= next.truncated;
    });
    child.on("close", (status, signal) => {
      if (timeoutTimer) {
        clearTimeout(timeoutTimer);
      }
      const finish = () => {
        if (killTimer) {
          clearTimeout(killTimer);
        }
        killAt = undefined;
        activeChildren.delete(child);
        const exitCode = typeof status === "number" ? status : signal ? 128 : 1;
        resolve({
          label,
          signal,
          status: exitCode,
          stderr,
          stderrTruncated,
          stdout,
          stdoutTruncated,
          timedOut,
        });
      };
      if (timedOut) {
        void finishTimedOutShellProcessTree(child, killAt, resolvedTimeoutKillGraceMs).then(
          finish,
          finish,
        );
        return;
      }
      finish();
    });
  });
}

async function runForeground(label: string, command: string, env: NodeJS.ProcessEnv) {
  console.log(`==> ${label}`);
  const result = await runShellCommand({ command, env, label });
  if (result.status !== 0) {
    throw new Error(`${label} failed with status ${result.status}`);
  }
}

export async function runCleanupSmokePhase(
  baseEnv: NodeJS.ProcessEnv,
  logDir: string,
  phases: Array<Record<string, unknown>>,
) {
  const command = "pnpm test:docker:cleanup";
  const logFile = path.join(logDir, `${CLEANUP_SMOKE_NAME}.log`);
  const startedAtMs = Date.now();
  let failure: LaneResult | undefined;
  try {
    await runPhase(phases, CLEANUP_SMOKE_NAME, {}, async () => {
      const result = await runShellCommand({
        command,
        env: baseEnv,
        label: CLEANUP_SMOKE_NAME,
        logFile,
      });
      if (result.status !== 0) {
        failure = cleanupSmokeResult(baseEnv, logFile, command, startedAtMs, result);
      }
      if (failure) {
        throw new Error(
          `Run cleanup smoke after parallel lanes failed with status ${failure.status}`,
        );
      }
    });
  } catch (error) {
    if (!failure) {
      const status = 1;
      const message = error instanceof Error ? error.message : String(error);
      await fs.promises.writeFile(
        logFile,
        [
          `==> [${CLEANUP_SMOKE_NAME}] command: ${command}`,
          `==> [${CLEANUP_SMOKE_NAME}] status: ${status}`,
          `==> [${CLEANUP_SMOKE_NAME}] error: ${message}`,
        ].join("\n"),
      );
      failure = cleanupSmokeResult(baseEnv, logFile, command, startedAtMs, {
        noOutputTimedOut: false,
        status,
        timedOut: false,
      });
    }
  }
  return failure;
}

async function runForegroundGroup(entries: ForegroundEntry[], env: NodeJS.ProcessEnv) {
  const failures: Array<{ entry: ForegroundEntry; error: unknown }> = [];
  for (const entry of entries) {
    try {
      const { command, label, phaseDetails = {}, phases } = entry;
      const entryEnv = { ...env, ...entry.env };
      if (phases) {
        await runPhase(phases, `build:${label}`, phaseDetails, async () => {
          await runForeground(label, command, entryEnv);
        });
      } else {
        await runForeground(label, command, entryEnv);
      }
    } catch (error) {
      failures.push({ entry, error });
    }
  }
  if (failures.length > 0) {
    throw new Error(
      failures
        .map(
          ({ entry, error }) =>
            `${entry.label}: ${error instanceof Error ? error.message : String(error)}`,
        )
        .join("\n"),
    );
  }
}

async function runDockerPreflight(
  baseEnv: NodeJS.ProcessEnv,
  options: { cleanup: boolean; enabled: boolean; runTimeoutMs: number },
) {
  if (!options.enabled) {
    console.log("==> Docker preflight: skipped");
    return;
  }
  console.log("==> Docker preflight");
  const version = await runShellCaptureCommand({
    command: "docker version --format '{{.Server.Version}}'",
    env: baseEnv,
    label: "docker-version",
    timeoutMs: 20_000,
  });
  if (version.status !== 0) {
    throw new Error(
      `Docker preflight failed: docker version status=${version.status}\n${version.stderr}${version.stdout}`,
    );
  }
  console.log(`==> Docker server: ${version.stdout.trim()}`);

  if (options.cleanup) {
    const stale = await runShellCaptureCommand({
      command:
        "docker ps -a --filter status=created --filter status=exited --filter status=dead --format '{{.Names}} {{.Status}}'",
      env: baseEnv,
      label: "docker-stale-list",
      timeoutMs: 20_000,
    });
    if (stale.status === 0) {
      const names = dockerPreflightContainerNames(stale.stdout);
      if (names.length > 0) {
        console.log(`==> Docker preflight cleanup: ${names.join(", ")}`);
        const cleanup = await runShellCommand({
          command: `docker rm -f ${names.map(shellQuote).join(" ")}`,
          env: baseEnv,
          label: "docker-stale-cleanup",
          timeoutMs: 90_000,
        });
        if (cleanup.status !== 0) {
          throw new Error(`Docker preflight cleanup failed with status ${cleanup.status}`);
        }
      }
    }
  }

  const startedAt = Date.now();
  const run = await runShellCommand({
    command: dockerPreflightSmokeCommand(),
    env: baseEnv,
    label: "docker-run-smoke",
    timeoutMs: options.runTimeoutMs,
  });
  const elapsedSeconds = Math.round((Date.now() - startedAt) / 1000);
  if (run.status !== 0) {
    throw new Error(
      `Docker preflight failed: ${dockerPreflightSmokeCommand()} status=${run.status} elapsed=${elapsedSeconds}s`,
    );
  }
  console.log(`==> Docker preflight run: ${elapsedSeconds}s`);
}

async function prepareOpenClawPackage(baseEnv: NodeJS.ProcessEnv, logDir: string) {
  const existing = baseEnv.OPENCLAW_CURRENT_PACKAGE_TGZ;
  if (existing) {
    const packageTgz = path.resolve(existing);
    baseEnv.OPENCLAW_CURRENT_PACKAGE_TGZ = packageTgz;
    baseEnv.OPENCLAW_BUNDLED_CHANNEL_HOST_BUILD = "0";
    baseEnv.OPENCLAW_NPM_ONBOARD_HOST_BUILD = "0";
    console.log(`==> OpenClaw package: ${packageTgz}`);
    return;
  }

  const packDir = path.join(logDir, "openclaw-package");
  await mkdir(packDir, { recursive: true });
  const packageTgz = path.join(packDir, "openclaw-current.tgz");
  await runForeground(
    "Prepare OpenClaw package once",
    `node ${shellQuote(path.join(ROOT_DIR, "scripts/package-openclaw-for-docker.mjs"))} --source-dir ${shellQuote(ROOT_DIR)} --allow-unreleased-changelog --output-dir ${shellQuote(packDir)} --output-name openclaw-current.tgz`,
    baseEnv,
  );
  await fs.promises.access(packageTgz);
  // Preserve current-tree package intent in generated GitHub reruns; otherwise a local QA
  // failure would retry through the strict release path and fail before reaching its lane.
  baseEnv.OPENCLAW_DOCKER_E2E_ALLOW_UNRELEASED_CHANGELOG = "true";
  baseEnv.OPENCLAW_CURRENT_PACKAGE_TGZ = packageTgz;
  baseEnv.OPENCLAW_BUNDLED_CHANNEL_HOST_BUILD = "0";
  baseEnv.OPENCLAW_NPM_ONBOARD_HOST_BUILD = "0";
  console.log(`==> OpenClaw package: ${baseEnv.OPENCLAW_CURRENT_PACKAGE_TGZ}`);
}

async function prepareDockerCandidate(
  plan: DockerCandidatePlan,
  logDir: string,
  manifestPath: string,
) {
  const sourceSha = gitOutput(ROOT_DIR, ["rev-parse", "HEAD"]);
  let candidate = null;
  if (plan.needs.package) {
    if (gitOutput(ROOT_DIR, ["status", "--porcelain=v1"])) {
      throw new Error("repository has working-tree changes; refusing to prepare Docker candidate");
    }
    const candidateEnv = commandEnv();
    for (const key of [...CANDIDATE_ENV_KEYS, ...REGISTRY_ENV_KEYS]) {
      delete candidateEnv[key];
    }
    await prepareOpenClawPackage(candidateEnv, logDir);
    const packagePath = candidateEnv.OPENCLAW_CURRENT_PACKAGE_TGZ!;
    const packed = inspectNpmPackageTarball(packagePath);
    const version = rootPackageVersion(ROOT_DIR);
    if (packed.packageJson.name !== "openclaw" || packed.packageJson.version !== version) {
      throw new Error("packed Docker candidate name or version differs from the root package");
    }
    let registry = null;
    if (plan.needs.prepublishPluginRegistry) {
      const registryDir = path.join(logDir, "prepublish-plugin-registry");
      fs.rmSync(registryDir, { force: true, recursive: true });
      const artifact = createPrepublishPluginRegistryArtifact({
        repoRoot: ROOT_DIR,
        outputDir: registryDir,
        sourceSha,
        candidateVersion: version,
        requiredPackages: plan.requiredPrepublishPluginPackages,
      });
      registry = {
        dir: registryDir,
        candidateVersion: version,
        manifestSha256: artifact.manifestSha256,
      };
    }
    candidate = {
      package: { path: packagePath, name: packed.packageJson.name, version, sha256: packed.sha256 },
      registry,
    };
  }
  await mkdir(path.dirname(manifestPath), { recursive: true });
  fs.writeFileSync(
    manifestPath,
    `${JSON.stringify({ schema: "openclaw.qa-docker-candidate/v1", schemaVersion: 1, sourceSha, candidate }, null, 2)}\n`,
  );
}

function e2eImageForLane(poolLane: DockerE2eLane, baseEnv: NodeJS.ProcessEnv) {
  if (poolLane.e2eImageKind === "bare") {
    return baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE;
  }
  if (poolLane.e2eImageKind === "functional") {
    return baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE;
  }
  return undefined;
}

type DockerLaneEnv = {
  [key: string]: string | undefined;
  OPENCLAW_DOCKER_CACHE_HOME_DIR: string;
  OPENCLAW_DOCKER_CLI_TOOLS_DIR: string;
};

function laneEnv(
  poolLane: DockerE2eLane,
  baseEnv: NodeJS.ProcessEnv,
  logDir: string,
  cacheKey: string | undefined,
): DockerLaneEnv {
  const name = poolLane.name;
  const cacheName = cacheKey || name;
  const env: DockerLaneEnv = {
    ...baseEnv,
    OPENCLAW_DOCKER_CACHE_HOME_DIR:
      process.env.OPENCLAW_DOCKER_CACHE_HOME_DIR ?? path.join(logDir, `${cacheName}-cache`),
    OPENCLAW_DOCKER_CLI_TOOLS_DIR:
      process.env.OPENCLAW_DOCKER_CLI_TOOLS_DIR ?? path.join(logDir, `${cacheName}-cli-tools`),
  };
  env.OPENCLAW_DOCKER_ALL_LANE_NAME = name;
  const image = e2eImageForLane(poolLane, baseEnv);
  if (image) {
    env.OPENCLAW_DOCKER_E2E_IMAGE = image;
  }
  if (poolLane.e2eImageKind) {
    env.OPENCLAW_DOCKER_E2E_IMAGE_KIND = poolLane.e2eImageKind;
  }
  return env;
}

async function runLane(
  lane: DockerE2eLane,
  baseEnv: NodeJS.ProcessEnv,
  logDir: string,
  fallbackTimeoutMs: number,
) {
  const { name } = lane;
  const timeoutMs = lane.timeoutMs ?? fallbackTimeoutMs;
  const noOutputTimeoutMs = lane.noOutputTimeoutMs;
  const logFile = path.join(logDir, `${name}.log`);
  const env = laneEnv(lane, baseEnv, logDir, lane.cacheKey);
  const pnpmCommand = env.OPENCLAW_DOCKER_ALL_PNPM_COMMAND?.trim();
  const command = pnpmCommand
    ? lane.command.replace(/(^|\s)pnpm(?=\s)/g, `$1${shellQuote(pnpmCommand)}`)
    : lane.command;
  await mkdir(env.OPENCLAW_DOCKER_CLI_TOOLS_DIR, { recursive: true });
  await mkdir(env.OPENCLAW_DOCKER_CACHE_HOME_DIR, { recursive: true });
  await fs.promises.writeFile(
    logFile,
    [
      `==> [${name}] cli tools dir: ${env.OPENCLAW_DOCKER_CLI_TOOLS_DIR}`,
      `==> [${name}] cache dir: ${env.OPENCLAW_DOCKER_CACHE_HOME_DIR}`,
      `==> [${name}] timeout: ${timeoutMs}ms`,
      `==> [${name}] no output timeout: ${noOutputTimeoutMs ?? 0}ms`,
      `==> [${name}] retries: ${lane.retries ?? 0}`,
      `==> [${name}] e2e image kind: ${lane.e2eImageKind ?? "none"}`,
      `==> [${name}] e2e image: ${env.OPENCLAW_DOCKER_E2E_IMAGE ?? ""}`,
      "",
    ].join("\n"),
  );
  console.log(`==> [${name}] start`);
  const startedAt = Date.now();
  const startedAtIso = new Date(startedAt).toISOString();
  let result: ShellCommandResult;
  const attempts: ReturnType<typeof laneAttempt>[] = [];
  const maxAttempts = 1 + Math.max(0, lane.retries ?? 0);
  for (let attempt = 1; ; attempt += 1) {
    const attemptStartedAt = Date.now();
    if (attempt > 1) {
      await fs.promises.appendFile(logFile, `\n==> [${name}] retry attempt ${attempt}\n`);
      console.log(`==> [${name}] retry ${attempt}/${maxAttempts}`);
    }
    result = await runShellCommand({
      command,
      env,
      label: name,
      logFile,
      timeoutMs,
      noOutputTimeoutMs,
    });
    attempts.push(laneAttempt(attempt, attemptStartedAt, result));
    if (result.status === 0 || attempt >= maxAttempts) {
      break;
    }
    const retryable =
      result.timedOut || (await laneLogMatchesRetryPattern(logFile, lane.retryPatterns));
    if (!retryable) {
      break;
    }
  }
  const elapsedSeconds = Math.round((Date.now() - startedAt) / 1000);
  if (result.status === 0) {
    console.log(`==> [${name}] pass ${elapsedSeconds}s`);
  } else {
    const timeoutLabel = result.timedOut ? " timeout" : "";
    console.error(
      `==> [${name}] fail${timeoutLabel} status=${result.status} ${elapsedSeconds}s log=${logFile}`,
    );
  }
  return {
    command,
    attempts,
    finishedAt: new Date().toISOString(),
    image: env.OPENCLAW_DOCKER_E2E_IMAGE,
    imageKind: lane.e2eImageKind,
    logFile,
    name,
    elapsedSeconds,
    rerunCommand: buildLaneRerunCommand(name, baseEnv),
    startedAt: startedAtIso,
    status: result.status,
    noOutputTimedOut: result.noOutputTimedOut,
    timedOut: result.timedOut,
  };
}

async function runLanePool(
  poolLanes: DockerE2eLane[],
  baseEnv: NodeJS.ProcessEnv,
  logDir: string,
  parallelism: number,
  options: LanePoolOptions,
) {
  const failures: LaneResult[] = [];
  const results: LaneResult[] = [];
  const pending = [...poolLanes];
  const running = new Map<symbol, Promise<{ id: symbol; result: LaneResult }>>();
  const active = {
    count: 0,
    resources: new Map<string, number>(),
    weight: 0,
  } satisfies SchedulerActiveState;
  const activeLanes = new Map<string, number>();
  let lastLaneStartAt = 0;
  let laneStartQueue: Promise<void> = Promise.resolve();
  const statusTimer =
    options.statusIntervalMs > 0
      ? setInterval(() => {
          const runningSummary = [...activeLanes]
            .map(([name, startedAt]) => `${name}:${Math.round((Date.now() - startedAt) / 1000)}s`)
            .join(", ");
          const resources = [...active.resources.entries()]
            .map(([resource, value]) => `${resource}=${value}`)
            .join(" ");
          console.log(
            `==> [${options.poolLabel}] active=${active.count} pending=${pending.length} ${resources}${
              runningSummary ? ` lanes=${runningSummary}` : ""
            }`,
          );
        }, options.statusIntervalMs)
      : undefined;
  statusTimer?.unref?.();

  async function waitForLaneStartSlot() {
    if (options.startStaggerMs <= 0) {
      return;
    }
    const previous = laneStartQueue;
    let releaseNext = () => {};
    laneStartQueue = new Promise<void>((resolve) => {
      releaseNext = resolve;
    });
    await previous;
    const waitMs = Math.max(0, lastLaneStartAt + options.startStaggerMs - Date.now());
    if (waitMs > 0) {
      await sleep(waitMs);
    }
    lastLaneStartAt = Date.now();
    releaseNext();
  }

  function canStartLane(candidate: DockerE2eLane) {
    return canStartSchedulerLane(candidate, active, parallelism, options);
  }

  function reserve(candidate: DockerE2eLane) {
    const weight = laneWeight(candidate);
    active.count += 1;
    active.weight += weight;
    for (const resource of laneResources(candidate)) {
      active.resources.set(resource, (active.resources.get(resource) ?? 0) + weight);
    }
  }

  function release(candidate: DockerE2eLane) {
    const weight = laneWeight(candidate);
    active.count -= 1;
    active.weight -= weight;
    for (const resource of laneResources(candidate)) {
      const next = (active.resources.get(resource) ?? 0) - weight;
      if (next > 0) {
        active.resources.set(resource, next);
      } else {
        active.resources.delete(resource);
      }
    }
  }

  async function startLane(poolLane: DockerE2eLane) {
    await waitForLaneStartSlot();
    reserve(poolLane);
    activeLanes.set(poolLane.name, Date.now());
    const id = Symbol(poolLane.name);
    const promise = runLane(poolLane, baseEnv, logDir, options.timeoutMs)
      .then((result) => ({ id, result }))
      .finally(() => {
        activeLanes.delete(poolLane.name);
        release(poolLane);
      });
    running.set(id, promise);
  }

  try {
    while (pending.length > 0 || running.size > 0) {
      let started = false;
      if (!options.failFast || failures.length === 0) {
        for (let index = 0; index < pending.length;) {
          const candidate = pending[index];
          if (!candidate) {
            break;
          }
          if (!canStartLane(candidate)) {
            index += 1;
            continue;
          }
          pending.splice(index, 1);
          await startLane(candidate);
          started = true;
        }
      }

      if (started) {
        continue;
      }
      if (running.size === 0) {
        const blocked = pending.map(laneSummary).join(", ");
        throw new Error(
          `No Docker lanes fit scheduler limits (${describeDockerSchedulerLimits(
            parallelism,
            options,
          )}): ${blocked}. Tune OPENCLAW_DOCKER_ALL_PARALLELISM, OPENCLAW_DOCKER_ALL_WEIGHT_LIMIT, or OPENCLAW_DOCKER_ALL_<RESOURCE>_LIMIT.`,
        );
      }

      const { id, result } = await Promise.race(running.values());
      running.delete(id);
      results.push(result);
      if (result.status !== 0) {
        failures.push(result);
      }
      if (options.failFast && failures.length > 0) {
        const remainingResults = await Promise.all(running.values());
        running.clear();
        for (const remaining of remainingResults) {
          results.push(remaining.result);
          if (remaining.result.status !== 0) {
            failures.push(remaining.result);
          }
        }
        break;
      }
    }
  } finally {
    if (statusTimer) {
      clearInterval(statusTimer);
    }
  }

  return { failures, results };
}

export async function tailFile(file: string, lines: number, maxBytes = LOG_TAIL_MAX_BYTES) {
  let content: string;
  try {
    const handle = await open(file, "r");
    try {
      const stat = await handle.stat();
      const bytesToRead = Math.min(Math.max(1, maxBytes), stat.size);
      const buffer = Buffer.alloc(bytesToRead);
      await handle.read(buffer, 0, bytesToRead, stat.size - bytesToRead);
      content = buffer.toString("utf8");
    } finally {
      await handle.close().catch(() => {});
    }
  } catch {
    content = "";
  }
  const tail = content.split(/\r?\n/).slice(-lines).join("\n");
  return tail.trimEnd();
}

async function laneLogMatchesRetryPattern(logFile: string, patterns: RegExp[]) {
  if (!patterns || patterns.length === 0) {
    return false;
  }
  const tail = await tailFile(logFile, 160);
  return patterns.some((pattern) => pattern.test(tail));
}

async function printFailureSummary(failures: LaneResult[], tailLines: number) {
  console.error(`ERROR: ${failures.length} Docker lane(s) failed.`);
  for (const failure of failures) {
    console.error(`---- ${failure.name} failed (status=${failure.status}): ${failure.logFile}`);
    const tail = await tailFile(failure.logFile, tailLines);
    if (tail) {
      console.error(tail);
    }
  }
}

const activeChildren = new Map<ChildProcess, number>();
let activeChildrenShutdownPromise: Promise<unknown> | undefined;

function shellCommandSkippedForShutdown(signal: ShutdownSignal | null = null) {
  return {
    noOutputTimedOut: false,
    signal,
    status: 143,
    timedOut: false,
  };
}

function shellCaptureSkippedForShutdown(label: string, signal: ShutdownSignal | null = null) {
  return {
    label,
    signal,
    status: 143,
    stderr: "",
    stderrTruncated: false,
    stdout: "",
    stdoutTruncated: false,
    timedOut: false,
  };
}

function shellProcessGroupAlive(child: ChildProcess) {
  if (process.platform === "win32" || !child.pid) {
    return false;
  }
  try {
    process.kill(-child.pid, 0);
    return true;
  } catch (error) {
    return error instanceof Error && "code" in error && error.code === "EPERM";
  }
}

async function waitForShellProcessGroupExit(child: ChildProcess, timeoutMs: number) {
  const deadlineAt = Date.now() + timeoutMs;
  while (Date.now() < deadlineAt) {
    if (!shellProcessGroupAlive(child)) {
      return true;
    }
    await new Promise((resolvePoll) => {
      setTimeout(resolvePoll, SHELL_PROCESS_GROUP_EXIT_POLL_MS);
    });
  }
  return !shellProcessGroupAlive(child);
}

async function finishTimedOutShellProcessTree(
  child: ChildProcess,
  killAt: number | undefined,
  timeoutKillGraceMs: number,
) {
  if (!shellProcessGroupAlive(child)) {
    return;
  }
  const graceRemainingMs =
    killAt === undefined ? timeoutKillGraceMs : Math.max(0, killAt - Date.now());
  if (graceRemainingMs > 0) {
    await waitForShellProcessGroupExit(child, graceRemainingMs);
  }
  if (shellProcessGroupAlive(child)) {
    terminateChild(child, "SIGKILL");
  }
  await waitForShellProcessGroupExit(child, SHELL_POST_FORCE_KILL_WAIT_MS);
}

function terminateChild(child: ChildProcess, signal: ShutdownSignal) {
  if (process.platform !== "win32" && child.pid) {
    try {
      process.kill(-child.pid, signal);
      return;
    } catch {
      // Fall back to killing the direct child below.
    }
  }
  child.kill(signal);
}

function terminateActiveChildren(signal: ShutdownSignal) {
  for (const child of activeChildren.keys()) {
    terminateChild(child, signal);
  }
}

async function shutdownActiveChildren(signal: ShutdownSignal, exitCode: number) {
  if (activeChildrenShutdownPromise) {
    terminateActiveChildren("SIGKILL");
    return activeChildrenShutdownPromise;
  }
  const children = [...activeChildren.entries()];
  terminateActiveChildren(signal);
  activeChildrenShutdownPromise = Promise.all(
    children.map(([child, timeoutKillGraceMs]) =>
      finishTimedOutShellProcessTree(child, Date.now() + timeoutKillGraceMs, timeoutKillGraceMs),
    ),
  ).finally(() => {
    process.exit(exitCode);
  });
  return activeChildrenShutdownPromise;
}

process.on("SIGINT", () => {
  void shutdownActiveChildren("SIGINT", 130);
});
process.on("SIGTERM", () => {
  void shutdownActiveChildren("SIGTERM", 143);
});

async function main() {
  const runStartedAt = new Date().toISOString();
  const phases: Array<Record<string, unknown>> = [];
  const parallelism = parsePositiveInt(
    process.env.OPENCLAW_DOCKER_ALL_PARALLELISM,
    DEFAULT_PARALLELISM,
    "OPENCLAW_DOCKER_ALL_PARALLELISM",
  );
  const tailParallelism = parsePositiveInt(
    process.env.OPENCLAW_DOCKER_ALL_TAIL_PARALLELISM,
    Math.min(parallelism, DEFAULT_TAIL_PARALLELISM),
    "OPENCLAW_DOCKER_ALL_TAIL_PARALLELISM",
  );
  const tailLines = parsePositiveInt(
    process.env.OPENCLAW_DOCKER_ALL_FAILURE_TAIL_LINES,
    DEFAULT_FAILURE_TAIL_LINES,
    "OPENCLAW_DOCKER_ALL_FAILURE_TAIL_LINES",
  );
  const laneTimeoutMs = parsePositiveInt(
    process.env.OPENCLAW_DOCKER_ALL_LANE_TIMEOUT_MS,
    DEFAULT_LANE_TIMEOUT_MS,
    "OPENCLAW_DOCKER_ALL_LANE_TIMEOUT_MS",
  );
  const laneStartStaggerMs = parseNonNegativeInt(
    process.env.OPENCLAW_DOCKER_ALL_START_STAGGER_MS,
    DEFAULT_LANE_START_STAGGER_MS,
    "OPENCLAW_DOCKER_ALL_START_STAGGER_MS",
  );
  const statusIntervalMs = parseNonNegativeInt(
    process.env.OPENCLAW_DOCKER_ALL_STATUS_INTERVAL_MS,
    DEFAULT_STATUS_INTERVAL_MS,
    "OPENCLAW_DOCKER_ALL_STATUS_INTERVAL_MS",
  );
  const preflightRunTimeoutMs = parsePositiveInt(
    process.env.OPENCLAW_DOCKER_ALL_PREFLIGHT_RUN_TIMEOUT_MS,
    DEFAULT_PREFLIGHT_RUN_TIMEOUT_MS,
    "OPENCLAW_DOCKER_ALL_PREFLIGHT_RUN_TIMEOUT_MS",
  );
  const failFast = parseBool(process.env.OPENCLAW_DOCKER_ALL_FAIL_FAST, true);
  const dryRun = parseBool(process.env.OPENCLAW_DOCKER_ALL_DRY_RUN, false);
  const preflightEnabled = parseBool(process.env.OPENCLAW_DOCKER_ALL_PREFLIGHT, true);
  const preflightCleanup = parseBool(process.env.OPENCLAW_DOCKER_ALL_PREFLIGHT_CLEANUP, true);
  const timingsEnabled = parseBool(process.env.OPENCLAW_DOCKER_ALL_TIMINGS, true);
  const buildEnabled = parseBool(process.env.OPENCLAW_DOCKER_ALL_BUILD, true);
  const allowFrozenTargetScenarioOmissions = parseBool(
    process.env.OPENCLAW_ALLOW_FROZEN_TARGET_SCENARIO_OMISSIONS,
    false,
  );
  const planJson =
    cliOptions.planJson || parseBool(process.env.OPENCLAW_DOCKER_ALL_PLAN_JSON, false);
  const planReleaseAll = parseBool(process.env.OPENCLAW_DOCKER_ALL_PLAN_RELEASE_ALL, false);
  const profile = parseProfile(process.env.OPENCLAW_DOCKER_ALL_PROFILE);
  const releaseProfile = normalizeReleaseProfileEnv(
    process.env.OPENCLAW_DOCKER_ALL_RELEASE_PROFILE || process.env.OPENCLAW_RELEASE_PROFILE,
  );
  const releaseChunk = process.env.OPENCLAW_DOCKER_ALL_CHUNK || process.env.DOCKER_E2E_CHUNK || "";
  const includeOpenWebUI = parseBool(
    process.env.OPENCLAW_DOCKER_ALL_INCLUDE_OPENWEBUI ?? process.env.INCLUDE_OPENWEBUI,
    true,
  );
  const selectedLaneNamesRaw =
    process.env.OPENCLAW_DOCKER_ALL_LANES || process.env.DOCKER_E2E_LANES || "";
  const selectedLaneNames = parseLaneSelection(selectedLaneNamesRaw);
  if (selectedLaneNamesRaw && selectedLaneNames.length === 0) {
    throw new Error("OPENCLAW_DOCKER_ALL_LANES must include at least one lane name");
  }
  const liveMode = parseLiveMode(process.env.OPENCLAW_DOCKER_ALL_LIVE_MODE);
  const liveRetries = parseNonNegativeInt(
    process.env.OPENCLAW_DOCKER_ALL_LIVE_RETRIES,
    DEFAULT_LIVE_RETRIES,
    "OPENCLAW_DOCKER_ALL_LIVE_RETRIES",
  );
  const timingsFile = path.resolve(
    process.env.OPENCLAW_DOCKER_ALL_TIMINGS_FILE || DEFAULT_TIMINGS_FILE,
  );
  const runId = process.env.OPENCLAW_DOCKER_ALL_RUN_ID || utcStampForPath();
  const logDir = path.resolve(
    process.env.OPENCLAW_DOCKER_ALL_LOG_DIR ||
      path.join(ROOT_DIR, ".artifacts/docker-tests", runId),
  );

  const baseEnv = commandEnv({
    OPENCLAW_DOCKER_E2E_BARE_IMAGE:
      process.env.OPENCLAW_DOCKER_E2E_BARE_IMAGE ||
      process.env.OPENCLAW_DOCKER_E2E_IMAGE ||
      DEFAULT_E2E_BARE_IMAGE,
    OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE:
      process.env.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE ||
      process.env.OPENCLAW_DOCKER_E2E_IMAGE ||
      DEFAULT_E2E_FUNCTIONAL_IMAGE,
  });
  baseEnv.OPENCLAW_DOCKER_E2E_IMAGE =
    process.env.OPENCLAW_DOCKER_E2E_IMAGE || baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE;
  const writeSummary = (summary: RunSummary) => writeRunSummary(logDir, summary, baseEnv);
  appendExtension(baseEnv, "matrix");
  appendExtension(baseEnv, "acpx");
  appendExtension(baseEnv, "codex");

  const timingStore = await loadTimingStore(timingsFile, timingsEnabled);
  const { omittedUnsupportedLaneNames, orderedLanes, orderedTailLanes, plan, scheduledLanes } =
    resolveDockerE2ePlan({
      includeOpenWebUI,
      liveMode,
      liveRetries,
      orderLanes,
      planReleaseAll: planJson && planReleaseAll,
      profile,
      releaseChunk,
      releaseProfile,
      selectedLaneNames,
      timingStore,
      upgradeSurvivorBaselines: process.env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS,
      upgradeSurvivorScenarios: process.env.OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS,
      upgradeSurvivorTargetRoot: process.env.OPENCLAW_UPGRADE_SURVIVOR_TARGET_ROOT,
      allowFrozenTargetScenarioOmissions,
      candidatePackageRoot: ROOT_DIR,
    });
  if (omittedUnsupportedLaneNames.length > 0 && !allowFrozenTargetScenarioOmissions) {
    throw new Error(
      `frozen target scenario omissions require trusted workflow opt-in: ${omittedUnsupportedLaneNames.join(", ")}`,
    );
  }
  if (scheduledLanes.length === 0 && omittedUnsupportedLaneNames.length === 0) {
    throw new Error(
      [
        "resolved zero Docker lanes",
        `profile=${profile}`,
        `releaseChunk=${releaseChunk || "<none>"}`,
        `releaseProfile=${releaseProfile}`,
        `liveMode=${liveMode}`,
        `includeOpenWebUI=${includeOpenWebUI ? "1" : "0"}`,
        `selectedLanes=${selectedLaneNames.length > 0 ? selectedLaneNames.join(",") : "<none>"}`,
      ].join("; "),
    );
  }
  const omittedUnsupportedLanes =
    omittedUnsupportedLaneNames.length > 0 ? omittedUnsupportedLaneNames : undefined;

  if (planJson) {
    process.stdout.write(`${JSON.stringify(plan, null, 2)}\n`);
    return;
  }
  if (cliOptions.prepareOnly) {
    await prepareDockerCandidate(plan, logDir, path.resolve(cliOptions.prepareOnly));
    return;
  }

  await mkdir(logDir, { recursive: true });
  console.log(`==> Docker test logs: ${logDir}`);
  console.log(`==> Profile: ${profile}${releaseChunk ? ` chunk=${releaseChunk}` : ""}`);
  if (profile === RELEASE_PATH_PROFILE) {
    console.log(`==> Release profile: ${releaseProfile}`);
  }
  console.log(`==> Parallelism: ${parallelism}`);
  console.log(`==> Tail parallelism: ${tailParallelism}`);
  console.log(`==> Lane timeout: ${laneTimeoutMs}ms`);
  console.log(`==> Live mode: ${liveMode}`);
  console.log(`==> Live retries: ${liveRetries}`);
  console.log(`==> Lane start stagger: ${laneStartStaggerMs}ms`);
  console.log(`==> Status interval: ${statusIntervalMs}ms`);
  console.log(`==> Fail fast: ${failFast ? "yes" : "no"}`);
  console.log(`==> Dry run: ${dryRun ? "yes" : "no"}`);
  console.log(
    `==> Docker preflight: ${preflightEnabled ? "yes" : "no"}${
      preflightCleanup ? " cleanup=yes" : " cleanup=no"
    }`,
  );
  console.log(`==> Build shared Docker images: ${buildEnabled ? "yes" : "no"}`);
  console.log(`==> Docker E2E bare image: ${baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE}`);
  console.log(`==> Docker E2E functional image: ${baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE}`);
  if (profile === RELEASE_PATH_PROFILE) {
    console.log(`==> Include Open WebUI: ${includeOpenWebUI ? "yes" : "no"}`);
  }
  if (selectedLaneNames.length > 0) {
    console.log(`==> Selected lanes: ${selectedLaneNames.join(", ")}`);
  }
  console.log(`==> Docker lane timings: ${timingStore.enabled ? timingsFile : "disabled"}`);
  console.log(`==> Live-test bundled plugins: ${baseEnv.OPENCLAW_DOCKER_BUILD_EXTENSIONS}`);
  const schedulerOptions = parseSchedulerOptions(process.env, parallelism);
  const tailSchedulerOptions = parseSchedulerOptions(process.env, tailParallelism);
  console.log(
    `==> Scheduler: weight=${schedulerOptions.weightLimit} ${resourceLimitsSummary(schedulerOptions.resourceLimits)}`,
  );
  console.log(
    `==> Tail scheduler: weight=${tailSchedulerOptions.weightLimit} ${resourceLimitsSummary(tailSchedulerOptions.resourceLimits)}`,
  );
  printLaneManifest("Main", orderedLanes, timingStore);
  printLaneManifest("Tail", orderedTailLanes, timingStore);
  if (omittedUnsupportedLanes) {
    console.log(
      `==> Docker lanes omitted: target lacks ${omittedUnsupportedLanes.join(", ")} scenario support`,
    );
  }
  if (dryRun) {
    console.log("==> Dry run complete");
    return;
  }
  validateDockerCandidateEnvironment(baseEnv, plan);

  // Planning can report unsupported scenarios, but execution cannot pass when
  // frozen-target omissions leave no selected lane to run.
  if (scheduledLanes.length === 0) {
    await writeSummary({
      chunk: releaseChunk || undefined,
      failures: [],
      lanes: [],
      omittedUnsupportedLanes,
      phases,
      profile,
      selectedLanes: selectedLaneNames.length > 0 ? selectedLaneNames : undefined,
      startedAt: runStartedAt,
      status: "failed",
    });
    throw new Error(
      `resolved zero runnable Docker lanes; frozen target does not support: ${omittedUnsupportedLaneNames.join(", ")}`,
    );
  }

  await runPhase(
    phases,
    "docker-preflight",
    { cleanup: preflightCleanup, enabled: preflightEnabled },
    async () => {
      await runDockerPreflight(baseEnv, {
        cleanup: preflightCleanup,
        enabled: preflightEnabled,
        runTimeoutMs: preflightRunTimeoutMs,
      });
    },
  );
  if (lanesNeedOpenClawPackage(scheduledLanes)) {
    await runPhase(phases, "prepare-openclaw-package", {}, async () => {
      await prepareOpenClawPackage(baseEnv, logDir);
    });
  } else {
    console.log("==> OpenClaw package: not needed for selected lanes");
  }

  if (buildEnabled) {
    const buildEntries: ForegroundEntry[] = [];
    if (scheduledLanes.some((poolLane) => poolLane.needsLiveImage)) {
      buildEntries.push({
        command: liveDockerHarnessScriptCommand("test-live-build-docker.sh"),
        label: "shared live-test image once",
        phaseDetails: { imageKind: "live" },
        phases,
      });
    }
    if (lanesNeedE2eImageKind(scheduledLanes, "bare")) {
      buildEntries.push({
        command: "pnpm test:docker:e2e-build",
        env: {
          OPENCLAW_DOCKER_E2E_IMAGE: baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE,
          OPENCLAW_DOCKER_E2E_TARGET: "bare",
        },
        label: `shared bare Docker E2E image once: ${baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE}`,
        phaseDetails: { image: baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE, imageKind: "bare" },
        phases,
      });
    }
    if (lanesNeedE2eImageKind(scheduledLanes, "functional")) {
      buildEntries.push({
        command: "pnpm test:docker:e2e-build",
        env: {
          OPENCLAW_DOCKER_E2E_IMAGE: baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE,
          OPENCLAW_DOCKER_E2E_TARGET: "functional",
        },
        label: `shared functional Docker E2E image once: ${baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE}`,
        phaseDetails: {
          image: baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE,
          imageKind: "functional",
        },
        phases,
      });
    }
    await runForegroundGroup(buildEntries, baseEnv);
  } else {
    console.log(`==> Shared Docker image builds: skipped`);
  }

  const options = {
    ...schedulerOptions,
    failFast,
    poolLabel: "main",
    startStaggerMs: laneStartStaggerMs,
    statusIntervalMs,
    timeoutMs: laneTimeoutMs,
  } satisfies LanePoolOptions;
  const mainResult = await runPhase(phases, "main-lane-pool", { lanes: orderedLanes.length }, () =>
    runLanePool(orderedLanes, baseEnv, logDir, parallelism, options),
  );
  const failures = [...mainResult.failures];
  const allResults = [...mainResult.results];
  await writeTimingStore(timingStore, mainResult.results);
  if (failFast && failures.length > 0) {
    await writeSummary({
      chunk: releaseChunk || undefined,
      failures,
      image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE,
      images: {
        bare: baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE,
        functional: baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE,
      },
      lanes: allResults,
      omittedUnsupportedLanes,
      phases,
      profile,
      selectedLanes: selectedLaneNames.length > 0 ? selectedLaneNames : undefined,
      startedAt: runStartedAt,
      status: "failed",
    });
    await printFailureSummary(failures, tailLines);
    process.exit(1);
  }

  if (orderedTailLanes.length > 0) {
    console.log("==> Running provider-sensitive Docker tail lanes");
    const tailResult = await runPhase(
      phases,
      "tail-lane-pool",
      { lanes: orderedTailLanes.length },
      () =>
        runLanePool(orderedTailLanes, baseEnv, logDir, tailParallelism, {
          ...options,
          ...tailSchedulerOptions,
          poolLabel: "tail",
        }),
    );
    failures.push(...tailResult.failures);
    allResults.push(...tailResult.results);
    await writeTimingStore(timingStore, tailResult.results);
  } else {
    console.log("==> Provider-sensitive Docker tail lanes: none");
  }
  if (failures.length > 0) {
    await writeSummary({
      chunk: releaseChunk || undefined,
      failures,
      image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE,
      images: {
        bare: baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE,
        functional: baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE,
      },
      lanes: allResults,
      omittedUnsupportedLanes,
      phases,
      profile,
      selectedLanes: selectedLaneNames.length > 0 ? selectedLaneNames : undefined,
      startedAt: runStartedAt,
      status: "failed",
    });
    await printFailureSummary(failures, tailLines);
    process.exit(1);
  }

  if (profile === DEFAULT_PROFILE && selectedLaneNames.length === 0) {
    const cleanupFailure = await runCleanupSmokePhase(baseEnv, logDir, phases);
    if (cleanupFailure) {
      failures.push(cleanupFailure);
    }
  } else {
    console.log("==> Cleanup smoke after parallel lanes: skipped for selected/release lanes");
  }
  await writeTimingStore(timingStore, allResults);
  if (failures.length > 0) {
    await writeSummary({
      chunk: releaseChunk || undefined,
      failures,
      image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE,
      images: {
        bare: baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE,
        functional: baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE,
      },
      lanes: allResults,
      omittedUnsupportedLanes,
      phases,
      profile,
      selectedLanes: selectedLaneNames.length > 0 ? selectedLaneNames : undefined,
      startedAt: runStartedAt,
      status: "failed",
    });
    await printFailureSummary(failures, tailLines);
    process.exit(1);
  }
  await writeSummary({
    chunk: releaseChunk || undefined,
    failures,
    image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE,
    images: {
      bare: baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE,
      functional: baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE,
    },
    lanes: allResults,
    omittedUnsupportedLanes,
    phases,
    profile,
    selectedLanes: selectedLaneNames.length > 0 ? selectedLaneNames : undefined,
    startedAt: runStartedAt,
    status: "passed",
  });
  console.log("==> Docker test suite passed");
}

if (IS_MAIN) {
  await main().catch((error: unknown) => {
    console.error(error instanceof Error ? error.message : String(error));
    process.exit(1);
  });
}