diff --git a/scripts/AGENTS.md b/scripts/AGENTS.md index 1b7e256e5131..f7b1bd269ef7 100644 --- a/scripts/AGENTS.md +++ b/scripts/AGENTS.md @@ -18,6 +18,11 @@ This directory owns local tooling, script wrappers, and generated-artifact helpe - Metadata-only or explicitly narrow commands may skip the lock when the existing helper logic says that is safe. - If you change the lock heuristics, add or update the narrow tests under `test/scripts/`. +## PR Prepare Gates + +- `scripts/pr prepare-gates` holds the heavy-check lock for its whole local gate block (`scripts/pr-gates-lock.mjs`), so concurrent gate runs across `.worktrees` queue as units instead of dying on child lock timeouts or vitest no-output watchdog kills. +- `OPENCLAW_PR_GATES_REMOTE=testbox` runs the full-suite `pnpm test` gate on a Blacksmith Testbox through `scripts/crabbox-wrapper.mjs` (same delegation as `check:changed`); `pnpm build`/`pnpm check` stay local. The `tbx_` lease id and Actions run URL land in `.local/gates.env` (`REMOTE_GATES_*`) and `.local/prep.md`. Use it for reviewed trusted code when a loaded host makes the local 88-shard run stall-kill; contributor/fork code stays on secretless CI or sanitized AWS unless a maintainer explicitly approves credentialed execution. + ## Generated Outputs - If a script writes generated artifacts, keep the source-of-truth generator, the package script, and the matching verification/check command aligned. diff --git a/scripts/pr-gates-lock.mjs b/scripts/pr-gates-lock.mjs new file mode 100644 index 000000000000..aa589a91ecc1 --- /dev/null +++ b/scripts/pr-gates-lock.mjs @@ -0,0 +1,75 @@ +// Holds the shared local heavy-check lock for a whole scripts/pr gate block so +// concurrent gate runs across .worktrees serialize before their first command +// instead of dying mid-test on child lock timeouts or no-output watchdog kills. +import fs from "node:fs"; +import { + acquireLocalHeavyCheckLockSync, + resolveLocalHeavyCheckEnv, +} from "./lib/local-heavy-check-runtime.mjs"; + +// A queued gate block legitimately waits out another full gate run; the +// 10-minute per-command default in the lock runtime is far too short for that. +const DEFAULT_GATE_LOCK_TIMEOUT_MS = 2 * 60 * 60 * 1000; +const PARENT_WATCH_INTERVAL_MS = 2_000; + +function parseArgs(argv) { + const args = { statusFile: "" }; + for (let index = 0; index < argv.length; index += 1) { + if (argv[index] === "--status-file") { + args.statusFile = argv[index + 1] ?? ""; + index += 1; + continue; + } + throw new Error(`Unknown option: ${argv[index]}`); + } + if (!args.statusFile) { + throw new Error("Usage: node scripts/pr-gates-lock.mjs --status-file "); + } + return args; +} + +function main() { + const { statusFile } = parseArgs(process.argv.slice(2)); + const baseEnv = resolveLocalHeavyCheckEnv(process.env); + const env = baseEnv.OPENCLAW_HEAVY_CHECK_LOCK_TIMEOUT_MS + ? baseEnv + : { ...baseEnv, OPENCLAW_HEAVY_CHECK_LOCK_TIMEOUT_MS: String(DEFAULT_GATE_LOCK_TIMEOUT_MS) }; + + const parentPid = process.ppid; + const release = acquireLocalHeavyCheckLockSync({ + cwd: process.cwd(), + env, + toolName: "pr-gates", + }); + let released = false; + const releaseOnce = () => { + if (released) { + return; + } + released = true; + release(); + }; + + process.on("exit", releaseOnce); + for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) { + process.on(signal, () => { + releaseOnce(); + process.exit(0); + }); + } + + fs.writeFileSync(statusFile, "acquired\n"); + + // The owner-pid reclaim in the lock runtime already covers a SIGKILLed + // holder; this watch releases promptly when the gate shell dies mid-block. + // ppid 1 also counts as dead: orphans reparent to init/launchd, and a + // helper that started orphaned has no gate block to hold the lock for. + setInterval(() => { + if (process.ppid !== parentPid || process.ppid === 1) { + releaseOnce(); + process.exit(0); + } + }, PARENT_WATCH_INTERVAL_MS); +} + +main(); diff --git a/scripts/pr-lib/gates.sh b/scripts/pr-lib/gates.sh index 9a2c09c86cd5..42687078d653 100644 --- a/scripts/pr-lib/gates.sh +++ b/scripts/pr-lib/gates.sh @@ -31,6 +31,175 @@ pin_worktree_bundled_plugins_dir() { export OPENCLAW_BUNDLED_PLUGINS_DIR="${OPENCLAW_BUNDLED_PLUGINS_DIR:-$PWD/extensions}" } +resolve_pr_gates_remote_mode() { + case "${OPENCLAW_PR_GATES_REMOTE:-}" in + "") + printf 'local\n' + ;; + testbox) + printf 'testbox\n' + ;; + *) + echo "Unsupported OPENCLAW_PR_GATES_REMOTE=${OPENCLAW_PR_GATES_REMOTE} (supported: testbox)." >&2 + return 1 + ;; + esac +} + +PR_GATES_LOCK_PID="" +PR_GATES_LOCK_STATUS_FILE="" + +acquire_pr_gates_lock() { + # Serialize whole gate blocks across .worktrees on the shared heavy-check + # lock; a queued gate run waits here, before its first command, instead of + # dying on child lock timeouts or shard no-output watchdog kills mid-test. + if [ "${OPENCLAW_TEST_HEAVY_CHECK_LOCK_HELD:-}" = "1" ]; then + return 0 + fi + + PR_GATES_LOCK_STATUS_FILE=$(mktemp) + # Use the canonical helper: the PR branch under test may predate it. + local scripts_dir="${script_parent_dir:-}" + if [ -z "$scripts_dir" ]; then + scripts_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) + fi + node "$scripts_dir/pr-gates-lock.mjs" --status-file "$PR_GATES_LOCK_STATUS_FILE" & + PR_GATES_LOCK_PID=$! + while [ ! -s "$PR_GATES_LOCK_STATUS_FILE" ]; do + if ! kill -0 "$PR_GATES_LOCK_PID" 2>/dev/null; then + wait "$PR_GATES_LOCK_PID" 2>/dev/null || true + PR_GATES_LOCK_PID="" + echo "Failed to acquire the shared local heavy-check lock for prepare gates." + exit 1 + fi + sleep 0.2 + done + # Same held-lock contract check-changed uses for its children: gate stages + # must not re-acquire the lock the block holder already owns. + export OPENCLAW_TEST_HEAVY_CHECK_LOCK_HELD=1 + export OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD=1 + export OPENCLAW_OXLINT_SKIP_LOCK=1 +} + +prepare_local_gate_workspace() { + pin_worktree_bundled_plugins_dir + acquire_pr_gates_lock + bootstrap_deps_if_needed +} + +release_pr_gates_lock() { + if [ -z "${PR_GATES_LOCK_PID:-}" ]; then + return 0 + fi + kill "$PR_GATES_LOCK_PID" 2>/dev/null || true + wait "$PR_GATES_LOCK_PID" 2>/dev/null || true + PR_GATES_LOCK_PID="" + rm -f "$PR_GATES_LOCK_STATUS_FILE" + PR_GATES_LOCK_STATUS_FILE="" + unset OPENCLAW_TEST_HEAVY_CHECK_LOCK_HELD OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD OPENCLAW_OXLINT_SKIP_LOCK +} + +run_remote_testbox_full_test_gate() { + local label="$1" + local log_file="$2" + local lease_label="$3" + # Same Blacksmith Testbox delegation shape check:changed uses; the worktree's + # own wrapper syncs this prep tree (the canonical copy would sync the primary + # checkout instead). + run_quiet_logged "$label" "$log_file" \ + node scripts/crabbox-wrapper.mjs run \ + --provider blacksmith-testbox \ + --blacksmith-org openclaw \ + --blacksmith-workflow .github/workflows/ci-check-testbox.yml \ + --blacksmith-job check \ + --blacksmith-ref main \ + --idle-timeout 90m \ + --ttl 240m \ + --timing-json \ + --label "$lease_label" \ + -- env CI=1 PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN=install corepack pnpm test +} + +read_remote_testbox_gate_stamp() { + # crabbox --timing-json emits one single-line JSON report on stderr; pick the + # last successful blacksmith-testbox report in the gate log as the stamp. + local log_file="$1" + jq -c -R ' + fromjson? + | select(type == "object") + | select(.provider == "blacksmith-testbox" and .exitCode == 0 and ((.leaseId // "") | startswith("tbx_"))) + ' "$log_file" | tail -n 1 +} + +read_remote_testbox_gate_run_url() { + # The delegated timing report currently omits actionsRunUrl, while the same + # run prints the canonical Actions URL as a separate line. + local log_file="$1" + local pr_url="${PR_URL:-}" + local expected_repo="${pr_url#https://github.com/}" + expected_repo="${expected_repo%%/pull/*}" + if [ -z "$expected_repo" ] || [ "$expected_repo" = "$pr_url" ]; then + expected_repo="openclaw/openclaw" + fi + local url_prefix="https://github.com/$expected_repo/actions/runs/" + local marker="GitHub Actions run: $url_prefix" + awk -v marker="$marker" -v url_prefix="$url_prefix" ' + index($0, marker) { + suffix = substr($0, index($0, marker) + length(marker)) + if (match(suffix, /^[0-9]+/)) { + print url_prefix substr(suffix, RSTART, RLENGTH) + exit + } + } + ' "$log_file" +} + +require_remote_testbox_gate_stamp() { + # Runs inside $(...): report to stderr and fail the substitution so set -e + # aborts the caller with the message visible. + local log_file="$1" + local stamp + stamp=$(read_remote_testbox_gate_stamp "$log_file") + if [ -z "$stamp" ]; then + echo "Remote testbox gate passed but no successful blacksmith-testbox timing stamp was found in $log_file." >&2 + return 1 + fi + local actions_run_url + actions_run_url=$(read_remote_testbox_gate_run_url "$log_file") + if [ -n "$actions_run_url" ] && [ "$(printf '%s\n' "$stamp" | jq -r '.actionsRunUrl // empty')" = "" ]; then + stamp=$(printf '%s\n' "$stamp" | jq -c --arg actionsRunUrl "$actions_run_url" '. + {actionsRunUrl: $actionsRunUrl}') + fi + printf '%s\n' "$stamp" +} + +write_gates_env_stamp() { + local pr="$1" + local docs_only="$2" + local changelog_required="$3" + local gates_mode="$4" + local last_verified_head="$5" + local full_gates_head="$6" + local hosted_gates_head="$7" + local remote_provider="$8" + local remote_lease_id="$9" + local remote_run_url="${10}" + + # Security: shell-escape values to prevent command injection when sourced. + printf '%s=%q\n' \ + PR_NUMBER "$pr" \ + DOCS_ONLY "$docs_only" \ + CHANGELOG_REQUIRED "$changelog_required" \ + GATES_MODE "$gates_mode" \ + LAST_VERIFIED_HEAD_SHA "$last_verified_head" \ + FULL_GATES_HEAD_SHA "$full_gates_head" \ + HOSTED_GATES_HEAD_SHA "$hosted_gates_head" \ + REMOTE_GATES_PROVIDER "$remote_provider" \ + REMOTE_GATES_LEASE_ID "$remote_lease_id" \ + REMOTE_GATES_RUN_URL "$remote_run_url" \ + GATES_PASSED_AT "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + > .local/gates.env +} + run_prepare_push_retry_gates() { local docs_only="${1:-false}" @@ -40,17 +209,72 @@ run_prepare_push_retry_gates() { return 1 fi - pin_worktree_bundled_plugins_dir - bootstrap_deps_if_needed + local gates_remote_mode + gates_remote_mode=$(resolve_pr_gates_remote_mode) + + prepare_local_gate_workspace run_quiet_logged "pnpm build (lease-retry)" ".local/lease-retry-build.log" pnpm build run_quiet_logged "pnpm check (lease-retry)" ".local/lease-retry-check.log" pnpm check - if [ "$docs_only" != "true" ]; then + + # The retry rebased the prep head, so the pre-push gates.env stamp no longer + # describes what these gates just verified; rewrite it for the new head so + # prep.md and prep.env do not attribute stale evidence to the pushed commit. + local retry_head + retry_head=$(git rev-parse HEAD) + local gates_mode="full" + local full_gates_head="$retry_head" + local remote_gates_provider="" + local remote_gates_lease_id="" + local remote_gates_run_url="" + + if [ "$docs_only" = "true" ]; then + release_pr_gates_lock + gates_mode="docs_only" + # No test ran: carry the prior full-gates proof and how it was produced. + full_gates_head="${FULL_GATES_HEAD_SHA:-}" + remote_gates_provider="${REMOTE_GATES_PROVIDER:-}" + remote_gates_lease_id="${REMOTE_GATES_LEASE_ID:-}" + remote_gates_run_url="${REMOTE_GATES_RUN_URL:-}" + elif [ "$gates_remote_mode" = "testbox" ]; then + release_pr_gates_lock + gates_mode="remote_testbox" + run_remote_testbox_full_test_gate \ + "pnpm test (lease-retry, blacksmith-testbox)" \ + ".local/lease-retry-test.log" \ + "pr-${PR_NUMBER:-unknown}-gates-lease-retry" + local retry_stamp + retry_stamp=$(require_remote_testbox_gate_stamp ".local/lease-retry-test.log") + remote_gates_provider="blacksmith-testbox" + remote_gates_lease_id=$(printf '%s\n' "$retry_stamp" | jq -r '.leaseId') + remote_gates_run_url=$(printf '%s\n' "$retry_stamp" | jq -r '.actionsRunUrl // ""') + echo "Remote testbox lease-retry gate stamp: $remote_gates_lease_id${remote_gates_run_url:+ ($remote_gates_run_url)}" + else run_quiet_logged "pnpm test (lease-retry)" ".local/lease-retry-test.log" pnpm test + release_pr_gates_lock fi + + write_gates_env_stamp \ + "${PR_NUMBER:-}" \ + "$docs_only" \ + "${CHANGELOG_REQUIRED:-false}" \ + "$gates_mode" \ + "$retry_head" \ + "$full_gates_head" \ + "" \ + "$remote_gates_provider" \ + "$remote_gates_lease_id" \ + "$remote_gates_run_url" } prepare_gates() { local pr="$1" + local gates_remote_mode + gates_remote_mode=$(resolve_pr_gates_remote_mode) + if [ "$gates_remote_mode" = "testbox" ] && [ "${OPENCLAW_TESTBOX:-}" = "1" ]; then + echo "OPENCLAW_PR_GATES_REMOTE=testbox conflicts with OPENCLAW_TESTBOX=1; hosted exact-head gates already own remote proof." + exit 2 + fi + enter_worktree "$pr" false checkout_prep_branch "$pr" @@ -120,11 +344,19 @@ prepare_gates() { current_head=$(git rev-parse HEAD) local previous_last_verified_head="" local previous_full_gates_head="" + local remote_gates_provider="" + local remote_gates_lease_id="" + local remote_gates_run_url="" if [ -s .local/gates.env ]; then # shellcheck disable=SC1091 source .local/gates.env previous_last_verified_head="${LAST_VERIFIED_HEAD_SHA:-}" previous_full_gates_head="${FULL_GATES_HEAD_SHA:-}" + # Carried alongside FULL_GATES_HEAD_SHA: they describe how that full-suite + # proof was produced; a fresh full run below overwrites them. + remote_gates_provider="${REMOTE_GATES_PROVIDER:-}" + remote_gates_lease_id="${REMOTE_GATES_LEASE_ID:-}" + remote_gates_run_url="${REMOTE_GATES_RUN_URL:-}" fi local gates_mode="full" @@ -140,6 +372,9 @@ prepare_gates() { if [ "${OPENCLAW_TESTBOX:-}" = "1" ]; then gates_mode="hosted_exact_head" + remote_gates_provider="" + remote_gates_lease_id="" + remote_gates_run_url="" if [ "$changelog_only" = "true" ]; then run_quiet_logged "git diff --check" ".local/gates-diff-check.log" git diff --check origin/main...HEAD fi @@ -149,14 +384,35 @@ prepare_gates() { gates_mode="reused_docs_only" echo "Docs/changelog-only delta since last verified head $previous_last_verified_head; reusing prior gates." else - pin_worktree_bundled_plugins_dir - bootstrap_deps_if_needed + prepare_local_gate_workspace run_quiet_logged "pnpm build" ".local/gates-build.log" pnpm build run_quiet_logged "pnpm check" ".local/gates-check.log" pnpm check if [ "$docs_only" = "true" ]; then + release_pr_gates_lock gates_mode="docs_only" + previous_full_gates_head="" + remote_gates_provider="" + remote_gates_lease_id="" + remote_gates_run_url="" echo "Docs-only change detected with high confidence; skipping pnpm test." + elif [ "$gates_remote_mode" = "testbox" ]; then + # The full suite runs on a Blacksmith Testbox, so free the local lock + # for other heavy work while we wait on remote proof. + release_pr_gates_lock + gates_mode="remote_testbox" + echo "Running pnpm test on Blacksmith Testbox (OPENCLAW_PR_GATES_REMOTE=testbox)." + run_remote_testbox_full_test_gate \ + "pnpm test (blacksmith-testbox)" \ + ".local/gates-test.log" \ + "pr-$pr-gates" + local remote_stamp + remote_stamp=$(require_remote_testbox_gate_stamp ".local/gates-test.log") + remote_gates_provider="blacksmith-testbox" + remote_gates_lease_id=$(printf '%s\n' "$remote_stamp" | jq -r '.leaseId') + remote_gates_run_url=$(printf '%s\n' "$remote_stamp" | jq -r '.actionsRunUrl // ""') + echo "Remote testbox gate stamp: $remote_gates_lease_id${remote_gates_run_url:+ ($remote_gates_run_url)}" + previous_full_gates_head="$current_head" else gates_mode="full" if [ -n "${OPENCLAW_VITEST_MAX_WORKERS:-}" ]; then @@ -169,21 +425,25 @@ prepare_gates() { echo "Running pnpm test with host-aware scheduling defaults." run_quiet_logged "pnpm test" ".local/gates-test.log" pnpm test fi + release_pr_gates_lock + remote_gates_provider="" + remote_gates_lease_id="" + remote_gates_run_url="" previous_full_gates_head="$current_head" fi fi - # Security: shell-escape values to prevent command injection when sourced. - printf '%s=%q\n' \ - PR_NUMBER "$pr" \ - DOCS_ONLY "$docs_only" \ - CHANGELOG_REQUIRED "$changelog_required" \ - GATES_MODE "$gates_mode" \ - LAST_VERIFIED_HEAD_SHA "$current_head" \ - FULL_GATES_HEAD_SHA "${previous_full_gates_head:-}" \ - HOSTED_GATES_HEAD_SHA "$hosted_gates_head" \ - GATES_PASSED_AT "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - > .local/gates.env + write_gates_env_stamp \ + "$pr" \ + "$docs_only" \ + "$changelog_required" \ + "$gates_mode" \ + "$current_head" \ + "${previous_full_gates_head:-}" \ + "$hosted_gates_head" \ + "$remote_gates_provider" \ + "$remote_gates_lease_id" \ + "$remote_gates_run_url" echo "docs_only=$docs_only" echo "changelog_only=$changelog_only" diff --git a/scripts/pr-lib/prepare-core.sh b/scripts/pr-lib/prepare-core.sh index 6dd567ae339c..632073c12482 100644 --- a/scripts/pr-lib/prepare-core.sh +++ b/scripts/pr-lib/prepare-core.sh @@ -156,6 +156,10 @@ prepare_push() { push_prep_head_to_pr_branch "$pr" "$PR_HEAD" "$prep_head_sha" "$lease_sha" true "${DOCS_ONLY:-false}" "$push_result_env" # shellcheck disable=SC1090 source "$push_result_env" + # A lease retry reruns gates for the rebased head and rewrites gates.env; + # re-source so prep.md/prep.env carry the stamp for the head actually pushed. + # shellcheck disable=SC1091 + source .local/gates.env prep_head_sha="$PUSH_PREP_HEAD_SHA" local_prep_head_sha="$PUSH_LOCAL_PREP_HEAD_SHA" local mainline_base_sha @@ -194,6 +198,11 @@ prepare_push() { - Gate mode: ${GATES_MODE:-unknown}. - Verified the remote PR head tree matches the local prep head. EOF_PREP + if [ -n "${REMOTE_GATES_LEASE_ID:-}" ]; then + cat >> .local/prep.md < /private/var symlink; resolve so lock and + // owner paths compare canonically. + const dir = realpathSync(mkdtempSync(join(tmpdir(), prefix))); + tempDirs.push(dir); + return dir; +} + +function makeLockRepoDir(): string { + const dir = makeTempDir("openclaw-pr-gates-lock-"); + mkdirSync(join(dir, ".git"), { recursive: true }); + return dir; +} + +function heavyCheckLockDir(repoDir: string): string { + return join(repoDir, ".git", "openclaw-local-checks", "heavy-check.lock"); +} + +function sanitizedEnv(overrides: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + // check:changed and gate runs export these to children; drop ambient copies + // so lock and mode behavior under test only sees explicit overrides. + const env: NodeJS.ProcessEnv = { ...process.env }; + delete env.OPENCLAW_PR_GATES_REMOTE; + delete env.OPENCLAW_TESTBOX; + delete env.OPENCLAW_TEST_HEAVY_CHECK_LOCK_HELD; + delete env.OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD; + delete env.OPENCLAW_OXLINT_SKIP_LOCK; + delete env.OPENCLAW_HEAVY_CHECK_LOCK_TIMEOUT_MS; + delete env.OPENCLAW_HEAVY_CHECK_LOCK_POLL_MS; + return { ...env, ...overrides }; +} + +function runGatesBash(script: string, options: { cwd?: string; env?: NodeJS.ProcessEnv } = {}) { + return spawnSync( + "bash", + [ + "-c", + [ + "set -euo pipefail", + `script_parent_dir='${repoRoot}/scripts'`, + `source '${repoRoot}/scripts/pr-lib/common.sh'`, + `source '${repoRoot}/scripts/pr-lib/gates.sh'`, + script, + ].join("\n"), + ], + { + cwd: options.cwd ?? repoRoot, + encoding: "utf8", + env: sanitizedEnv(options.env), + }, + ); +} + +function spawnGateLockHolder(repoDir: string, statusFile: string, env: NodeJS.ProcessEnv = {}) { + const child = spawn(process.execPath, [gateLockHelperPath, "--status-file", statusFile], { + cwd: repoDir, + stdio: ["ignore", "ignore", "pipe"], + env: sanitizedEnv(env), + }); + children.push(child); + return child; +} + +function makeRetryRepo(): { repoDir: string; stubBin: string; headSha: string } { + const dir = makeTempDir("openclaw-pr-gates-retry-"); + const repoDir = join(dir, "repo"); + mkdirSync(repoDir); + for (const args of [ + ["init", "-q"], + [ + "-c", + "user.name=t", + "-c", + "user.email=t@example.com", + "commit", + "-q", + "--allow-empty", + "-m", + "retry head", + ], + ]) { + const result = spawnSync("git", args, { cwd: repoDir, encoding: "utf8" }); + expect(result.status).toBe(0); + } + mkdirSync(join(repoDir, ".local")); + + const stubBin = join(dir, "bin"); + mkdirSync(stubBin); + writeFileSync(join(stubBin, "pnpm"), "#!/bin/sh\nexit 0\n"); + chmodSync(join(stubBin, "pnpm"), 0o755); + // Route the crabbox wrapper to a canned timing report; everything else + // (the gate lock helper) still needs the real node. + writeFileSync( + join(stubBin, "node"), + [ + "#!/bin/sh", + 'case "$2" in', + `run) printf '{"provider":"blacksmith-testbox","leaseId":"tbx_retry","exitCode":0,"runStatus":"succeeded","actionsRunUrl":"https://example.test/runs/7"}\\n' >&2; exit 0;;`, + `*) exec '${process.execPath}' "$@";;`, + "esac", + ].join("\n"), + ); + chmodSync(join(stubBin, "node"), 0o755); + + const headSha = spawnSync("git", ["rev-parse", "HEAD"], { + cwd: repoDir, + encoding: "utf8", + }).stdout.trim(); + return { repoDir, stubBin, headSha }; +} + +async function waitFor(predicate: () => boolean, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate()) { + return true; + } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + return predicate(); +} + +async function waitForExit(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) { + return; + } + await new Promise((resolve) => { + child.once("exit", resolve); + }); +} + +afterEach(async () => { + for (const child of children.splice(0)) { + child.kill("SIGKILL"); + await waitForExit(child); + } + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } +}); + +describe("resolve_pr_gates_remote_mode", () => { + it.each([ + { value: undefined, expected: "local" }, + { value: "", expected: "local" }, + { value: "testbox", expected: "testbox" }, + ])("resolves OPENCLAW_PR_GATES_REMOTE=$value to $expected", ({ value, expected }) => { + const env: NodeJS.ProcessEnv = {}; + if (value !== undefined) { + env.OPENCLAW_PR_GATES_REMOTE = value; + } + const result = runGatesBash("resolve_pr_gates_remote_mode", { env }); + expect(result.status).toBe(0); + expect(result.stdout.trim()).toBe(expected); + }); + + it("rejects unsupported values", () => { + const result = runGatesBash("resolve_pr_gates_remote_mode", { + env: { OPENCLAW_PR_GATES_REMOTE: "azure" }, + }); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("Unsupported OPENCLAW_PR_GATES_REMOTE=azure"); + }); + + it("rejects the hosted-gates conflict before touching the worktree", () => { + const result = runGatesBash("prepare_gates 424242", { + env: { OPENCLAW_PR_GATES_REMOTE: "testbox", OPENCLAW_TESTBOX: "1" }, + }); + expect(result.status).toBe(2); + expect(result.stdout).toContain("conflicts with OPENCLAW_TESTBOX=1"); + }); +}); + +describe("remote testbox gate delegation", () => { + it("runs the full pnpm test through the worktree crabbox wrapper", () => { + const dir = makeTempDir("openclaw-pr-gates-remote-"); + const stubBin = join(dir, "bin"); + mkdirSync(stubBin); + writeFileSync( + join(stubBin, "node"), + [ + "#!/bin/sh", + "printf 'ARG:%s\\n' \"$@\"", + `printf '{"provider":"blacksmith-testbox","leaseId":"tbx_stub","exitCode":0,"runStatus":"passed"}\\n' >&2`, + ].join("\n"), + ); + chmodSync(join(stubBin, "node"), 0o755); + + const workDir = join(dir, "work"); + mkdirSync(workDir); + const result = runGatesBash( + "run_remote_testbox_full_test_gate 'pnpm test (blacksmith-testbox)' .local/gates-test.log pr-424242-gates\n" + + "grep '^ARG:' .local/gates-test.log | paste -sd ' ' -", + { + cwd: workDir, + env: { PATH: `${stubBin}:${process.env.PATH ?? ""}` }, + }, + ); + + expect(result.status).toBe(0); + const argLine = result.stdout + .split("\n") + .find((line) => line.includes("crabbox-wrapper.mjs")) + ?.replaceAll("ARG:", ""); + expect(argLine).toBe( + "scripts/crabbox-wrapper.mjs run " + + "--provider blacksmith-testbox " + + "--blacksmith-org openclaw " + + "--blacksmith-workflow .github/workflows/ci-check-testbox.yml " + + "--blacksmith-job check " + + "--blacksmith-ref main " + + "--idle-timeout 90m --ttl 240m --timing-json " + + "--label pr-424242-gates " + + "-- env CI=1 PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN=install corepack pnpm test", + ); + }); + + it("extracts the last successful blacksmith-testbox timing stamp", () => { + const dir = makeTempDir("openclaw-pr-gates-stamp-"); + const log = join(dir, "gates-test.log"); + writeFileSync( + log, + [ + "provider=blacksmith-testbox id=tbx_first sync=delegated auth=blacksmith", + "GitHub Actions run: https://github.com/openclaw/openclaw/actions/runs/1234", + '{"not":"a stamp"}', + "not json at all", + '{"provider":"blacksmith-testbox","leaseId":"tbx_first","exitCode":1,"runStatus":"failed"}', + '{"provider":"blacksmith-testbox","leaseId":"tbx_final","exitCode":0,"runStatus":"passed"}', + "GitHub Actions run: https://github.com/openclaw/openclaw/actions/runs/9999", + "GitHub Actions run: https://github.com/example/other/actions/runs/8888", + "", + ].join("\n"), + ); + + const result = runGatesBash( + `require_remote_testbox_gate_stamp '${log}' | jq -r '[.leaseId, .actionsRunUrl] | @tsv'`, + ); + expect(result.status).toBe(0); + expect(result.stdout.trim()).toBe( + "tbx_final\thttps://github.com/openclaw/openclaw/actions/runs/1234", + ); + }); + + it("fails when the gate log has no successful stamp", () => { + const dir = makeTempDir("openclaw-pr-gates-stamp-"); + const log = join(dir, "gates-test.log"); + writeFileSync( + log, + '{"provider":"blacksmith-testbox","leaseId":"tbx_only","exitCode":1,"runStatus":"failed"}\n', + ); + + const result = runGatesBash(`require_remote_testbox_gate_stamp '${log}'`); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("no successful blacksmith-testbox timing stamp"); + }); +}); + +describe("lease-retry gate stamp refresh", () => { + it("rewrites gates.env with the fresh remote stamp for the rebased head", () => { + const { repoDir, stubBin, headSha } = makeRetryRepo(); + const result = runGatesBash( + [ + "PR_NUMBER=4242", + "CHANGELOG_REQUIRED=false", + "REMOTE_GATES_PROVIDER=blacksmith-testbox", + "REMOTE_GATES_LEASE_ID=tbx_stale", + "REMOTE_GATES_RUN_URL=https://example.test/runs/1", + "FULL_GATES_HEAD_SHA=deadbeef", + "run_prepare_push_retry_gates false", + "cat .local/gates.env", + ].join("\n"), + { + cwd: repoDir, + env: { + PATH: `${stubBin}:${process.env.PATH ?? ""}`, + OPENCLAW_PR_GATES_REMOTE: "testbox", + }, + }, + ); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("GATES_MODE=remote_testbox"); + expect(result.stdout).toContain("REMOTE_GATES_LEASE_ID=tbx_retry"); + expect(result.stdout).toContain(`LAST_VERIFIED_HEAD_SHA=${headSha}`); + expect(result.stdout).toContain(`FULL_GATES_HEAD_SHA=${headSha}`); + expect(result.stdout).not.toContain("tbx_stale"); + }); + + it("clears a stale remote stamp when the retry test ran locally", () => { + const { repoDir, stubBin, headSha } = makeRetryRepo(); + const result = runGatesBash( + [ + "PR_NUMBER=4242", + "CHANGELOG_REQUIRED=false", + "REMOTE_GATES_PROVIDER=blacksmith-testbox", + "REMOTE_GATES_LEASE_ID=tbx_stale", + "run_prepare_push_retry_gates false", + "cat .local/gates.env", + ].join("\n"), + { + cwd: repoDir, + env: { PATH: `${stubBin}:${process.env.PATH ?? ""}` }, + }, + ); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("GATES_MODE=full"); + expect(result.stdout).toContain("REMOTE_GATES_LEASE_ID=''"); + expect(result.stdout).toContain(`FULL_GATES_HEAD_SHA=${headSha}`); + expect(result.stdout).not.toContain("tbx_stale"); + }); +}); + +describe("prepare gate stamp transitions", () => { + it("clears remote stamps when fresh docs-only gates do not reuse prior proof", () => { + const { repoDir } = makeRetryRepo(); + spawnSync("git", ["update-ref", "refs/remotes/origin/main", "HEAD"], { cwd: repoDir }); + mkdirSync(join(repoDir, "docs"), { recursive: true }); + writeFileSync(join(repoDir, "docs", "proof.md"), "fresh docs\n"); + spawnSync("git", ["add", "docs/proof.md"], { cwd: repoDir }); + spawnSync( + "git", + ["-c", "user.name=t", "-c", "user.email=t@example.com", "commit", "-qm", "docs"], + { cwd: repoDir }, + ); + writeFileSync(join(repoDir, ".local", "pr-meta.env"), "PR_AUTHOR=steipete\n"); + writeFileSync( + join(repoDir, ".local", "gates.env"), + [ + "LAST_VERIFIED_HEAD_SHA=deadbeef", + "FULL_GATES_HEAD_SHA=deadbeef", + "REMOTE_GATES_PROVIDER=blacksmith-testbox", + "REMOTE_GATES_LEASE_ID=tbx_stale", + "REMOTE_GATES_RUN_URL=https://example.test/runs/1", + "", + ].join("\n"), + ); + + const result = runGatesBash( + [ + "enter_worktree() { :; }", + "checkout_prep_branch() { :; }", + "path_is_docsish() { return 0; }", + "changelog_required_for_changed_files() { return 1; }", + "prepare_local_gate_workspace() { :; }", + "run_quiet_logged() { :; }", + "release_pr_gates_lock() { :; }", + "prepare_gates 4242", + "cat .local/gates.env", + ].join("\n"), + { cwd: repoDir }, + ); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("GATES_MODE=docs_only"); + expect(result.stdout).toContain("FULL_GATES_HEAD_SHA=''"); + expect(result.stdout).toContain("REMOTE_GATES_LEASE_ID=''"); + expect(result.stdout).not.toContain("tbx_stale"); + }); + + it("clears remote stamps when hosted exact-head gates replace remote proof", () => { + const { repoDir } = makeRetryRepo(); + spawnSync("git", ["update-ref", "refs/remotes/origin/main", "HEAD"], { cwd: repoDir }); + writeFileSync(join(repoDir, "changed.ts"), "export {};\n"); + spawnSync("git", ["add", "changed.ts"], { cwd: repoDir }); + spawnSync( + "git", + ["-c", "user.name=t", "-c", "user.email=t@example.com", "commit", "-qm", "change"], + { cwd: repoDir }, + ); + writeFileSync(join(repoDir, ".local", "pr-meta.env"), "PR_AUTHOR=steipete\n"); + writeFileSync( + join(repoDir, ".local", "gates.env"), + [ + "LAST_VERIFIED_HEAD_SHA=deadbeef", + "FULL_GATES_HEAD_SHA=deadbeef", + "REMOTE_GATES_PROVIDER=blacksmith-testbox", + "REMOTE_GATES_LEASE_ID=tbx_stale", + "REMOTE_GATES_RUN_URL=https://example.test/runs/1", + "", + ].join("\n"), + ); + + const result = runGatesBash( + [ + "enter_worktree() { :; }", + "checkout_prep_branch() { :; }", + "path_is_docsish() { return 1; }", + "changelog_required_for_changed_files() { return 1; }", + "run_hosted_prepare_gates() { :; }", + "prepare_gates 4242", + "cat .local/gates.env", + ].join("\n"), + { cwd: repoDir, env: { OPENCLAW_TESTBOX: "1" } }, + ); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("GATES_MODE=hosted_exact_head"); + expect(result.stdout).toContain("REMOTE_GATES_LEASE_ID=''"); + expect(result.stdout).not.toContain("tbx_stale"); + }); +}); + +describe("pr-gates-lock helper", () => { + it("acquires the shared heavy-check lock and releases it on SIGTERM", async () => { + const repoDir = makeLockRepoDir(); + const statusFile = join(repoDir, "status"); + const holder = spawnGateLockHolder(repoDir, statusFile); + + expect(await waitFor(() => existsSync(statusFile), 5_000)).toBe(true); + expect(existsSync(heavyCheckLockDir(repoDir))).toBe(true); + + holder.kill("SIGTERM"); + await waitForExit(holder); + expect(await waitFor(() => !existsSync(heavyCheckLockDir(repoDir)), 5_000)).toBe(true); + }); + + it("queues behind an existing holder and acquires after it exits", async () => { + const repoDir = makeLockRepoDir(); + const firstStatus = join(repoDir, "status-first"); + const secondStatus = join(repoDir, "status-second"); + + const first = spawnGateLockHolder(repoDir, firstStatus); + expect(await waitFor(() => existsSync(firstStatus), 5_000)).toBe(true); + + const second = spawnGateLockHolder(repoDir, secondStatus, { + OPENCLAW_HEAVY_CHECK_LOCK_POLL_MS: "50", + }); + await new Promise((resolve) => setTimeout(resolve, 400)); + expect(existsSync(secondStatus)).toBe(false); + + first.kill("SIGTERM"); + await waitForExit(first); + expect(await waitFor(() => existsSync(secondStatus), 5_000)).toBe(true); + + second.kill("SIGTERM"); + await waitForExit(second); + expect(await waitFor(() => !existsSync(heavyCheckLockDir(repoDir)), 5_000)).toBe(true); + }); + + it("fails instead of holding when the wait times out", async () => { + const repoDir = makeLockRepoDir(); + const lockDir = heavyCheckLockDir(repoDir); + mkdirSync(lockDir, { recursive: true }); + // Owner pid must be alive or the helper reclaims the stale lock. + writeFileSync( + join(lockDir, "owner.json"), + `${JSON.stringify({ pid: process.pid, tool: "test-holder", cwd: repoDir })}\n`, + ); + + const statusFile = join(repoDir, "status"); + const holder = spawnGateLockHolder(repoDir, statusFile, { + OPENCLAW_HEAVY_CHECK_LOCK_TIMEOUT_MS: "200", + OPENCLAW_HEAVY_CHECK_LOCK_POLL_MS: "50", + }); + await waitForExit(holder); + + expect(holder.exitCode).not.toBe(0); + expect(existsSync(statusFile)).toBe(false); + }); + + it("releases the lock when the parent process dies", async () => { + const repoDir = makeLockRepoDir(); + const statusFile = join(repoDir, "status"); + const parent = spawn( + "bash", + [ + "-c", + `node '${gateLockHelperPath}' --status-file '${statusFile}' 2>/dev/null & ` + + `while [ ! -s '${statusFile}' ]; do sleep 0.05; done`, + ], + { cwd: repoDir, stdio: "ignore", env: sanitizedEnv() }, + ); + children.push(parent); + await waitForExit(parent); + + expect(existsSync(statusFile)).toBe(true); + expect(await waitFor(() => !existsSync(heavyCheckLockDir(repoDir)), 8_000)).toBe(true); + }); +}); + +describe("gates.sh gate lock plumbing", () => { + it("acquires the block lock before dependency bootstrap", () => { + const result = runGatesBash( + [ + "events=$(mktemp)", + 'pin_worktree_bundled_plugins_dir() { echo pin >> "$events"; }', + 'acquire_pr_gates_lock() { echo lock >> "$events"; }', + 'bootstrap_deps_if_needed() { echo bootstrap >> "$events"; }', + "prepare_local_gate_workspace", + 'cat "$events"', + ].join("\n"), + ); + + expect(result.status).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual(["pin", "lock", "bootstrap"]); + }); + + it("exports the held-lock contract while holding and clears it on release", () => { + const repoDir = makeLockRepoDir(); + const result = runGatesBash( + [ + "acquire_pr_gates_lock", + 'echo "held=${OPENCLAW_TEST_HEAVY_CHECK_LOCK_HELD:-unset},${OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD:-unset},${OPENCLAW_OXLINT_SKIP_LOCK:-unset}"', + "jq -r .tool .git/openclaw-local-checks/heavy-check.lock/owner.json", + "release_pr_gates_lock", + 'echo "released=${OPENCLAW_TEST_HEAVY_CHECK_LOCK_HELD:-unset}"', + '[ -d .git/openclaw-local-checks/heavy-check.lock ] && echo "lock=held" || echo "lock=free"', + ].join("\n"), + { cwd: repoDir }, + ); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("held=1,1,1"); + expect(result.stdout).toContain("pr-gates"); + expect(result.stdout).toContain("released=unset"); + expect(result.stdout).toContain("lock=free"); + }); + + it("skips acquisition when a parent already holds the lock", () => { + const repoDir = makeLockRepoDir(); + const result = runGatesBash( + [ + "acquire_pr_gates_lock", + '[ -d .git/openclaw-local-checks/heavy-check.lock ] && echo "lock=held" || echo "lock=free"', + 'echo "helper_pid=${PR_GATES_LOCK_PID:-none}"', + ].join("\n"), + { cwd: repoDir, env: { OPENCLAW_TEST_HEAVY_CHECK_LOCK_HELD: "1" } }, + ); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("lock=free"); + expect(result.stdout).toContain("helper_pid=none"); + }); + + it("fails the gate run when the lock wait times out", () => { + const repoDir = makeLockRepoDir(); + const lockDir = heavyCheckLockDir(repoDir); + mkdirSync(lockDir, { recursive: true }); + writeFileSync( + join(lockDir, "owner.json"), + `${JSON.stringify({ pid: process.pid, tool: "test-holder", cwd: repoDir })}\n`, + ); + + const result = runGatesBash("acquire_pr_gates_lock", { + cwd: repoDir, + env: { + OPENCLAW_HEAVY_CHECK_LOCK_TIMEOUT_MS: "200", + OPENCLAW_HEAVY_CHECK_LOCK_POLL_MS: "50", + }, + }); + + expect(result.status).toBe(1); + expect(result.stdout).toContain( + "Failed to acquire the shared local heavy-check lock for prepare gates.", + ); + }); +});