fix(pr): prevent same-PR operations from overlapping (#103669)

* fix(pr): serialize operations per pull request

* fix(pr): tighten supervisor error handling
This commit is contained in:
Peter Steinberger
2026-07-10 13:34:03 +01:00
committed by GitHub
parent 505f8ae6a3
commit ea6aa1dc75
9 changed files with 2733 additions and 47 deletions
+1
View File
@@ -20,6 +20,7 @@ This directory owns local tooling, script wrappers, and generated-artifact helpe
## PR Prepare Gates
- `scripts/pr` serializes review, prepare, and merge operations per PR across linked worktrees; `scripts/pr gc` skips active or indeterminate locks. A successful command return is the trusted synchronous-completion contract: every PR-state-mutating child must be joined before returning, and such work must never daemonize or explicitly escape both the operation group and lock-notification FD. Failed, interrupted, or controller-lost operations stay locked because detached children cannot be disproved; after verifying no child tools remain, use the reported exact-OID `scripts/pr lock-recover` command. Never bypass or delete these refs manually.
- `scripts/pr prepare-gates` holds the heavy-check lock for its whole local gate block (`scripts/pr-gates-lock.mjs`), so concurrent gate runs across `.worktrees` queue as units instead of dying on child lock timeouts or vitest no-output watchdog kills.
- `OPENCLAW_PR_GATES_REMOTE=testbox` runs the full-suite `pnpm test` gate on a Blacksmith Testbox through `scripts/crabbox-wrapper.mjs` (same delegation as `check:changed`); `pnpm build`/`pnpm check` stay local. The `tbx_` lease id and Actions run URL land in `.local/gates.env` (`REMOTE_GATES_*`) and `.local/prep.md`. Use it for reviewed trusted code when a loaded host makes the local 88-shard run stall-kill; contributor/fork code stays on secretless CI or sanitized AWS unless a maintainer explicitly approves credentialed execution.
+70 -3
View File
@@ -21,6 +21,32 @@ if common_git_dir=$(git -C "$script_parent_dir" rev-parse --path-format=absolute
fi
fi
is_locked_pr_command() {
case "$1" in
review-init | review-checkout-main | review-checkout-pr | review-claim | review-guard | review-artifacts-init | review-validate-artifacts | review-tests | prepare-init | prepare-validate-commit | prepare-gates | prepare-push | prepare-sync-head | prepare-run | merge-verify | merge-run) return 0 ;;
*) return 1 ;;
esac
}
is_supervised_pr_process() {
[ "${OPENCLAW_PR_DEDICATED_PROCESS_GROUP:-}" = "1" ] &&
[ "${OPENCLAW_PR_LOCK_NOTIFY_FD:-}" = "3" ] &&
[ "${OPENCLAW_PR_LOCK_SUPERVISOR_PID:-}" = "$PPID" ]
}
if [ "${1-}" = "gc" ] || is_locked_pr_command "${1-}"; then
if is_supervised_pr_process; then
# Do not leak the one-shot marker to tools or nested wrapper calls.
unset OPENCLAW_PR_DEDICATED_PROCESS_GROUP
else
unset OPENCLAW_PR_DEDICATED_PROCESS_GROUP
unset OPENCLAW_PR_LOCK_NOTIFY_FD
unset OPENCLAW_PR_LOCK_SUPERVISOR_PID
command -v node >/dev/null 2>&1 || { echo "Missing required command: node" >&2; exit 1; }
exec node "$script_parent_dir/pr-lib/process-group-runner.mjs" "$script_parent_dir/.." "$script_self" "$@"
fi
fi
# shellcheck disable=SC1091
source "$script_parent_dir/lib/plain-gh.sh"
@@ -29,6 +55,7 @@ usage() {
Usage:
scripts/pr ls
scripts/pr gc [--dry-run]
scripts/pr lock-recover <PR> <OWNER_OID> --confirmed-no-running-tools
scripts/pr review-init <PR>
scripts/pr review-checkout-main <PR>
scripts/pr review-checkout-pr <PR>
@@ -75,6 +102,8 @@ gh() {
# shellcheck disable=SC1091
source "$script_parent_dir/pr-lib/worktree.sh"
# shellcheck disable=SC1091
source "$script_parent_dir/pr-lib/operation-lock.sh"
# shellcheck disable=SC1091
source "$script_parent_dir/pr-lib/common.sh"
# shellcheck disable=SC1091
source "$script_parent_dir/pr-lib/changelog.sh"
@@ -95,18 +124,56 @@ main() {
exit 2
fi
require_cmds
local cmd="${1-}"
shift || true
if [ "$cmd" = "lock-recover" ]; then
local pr="${1-}"
local owner_oid="${2-}"
local confirmation="${3-}"
[ -n "$pr" ] && [ -n "$owner_oid" ] && [ "$#" -eq 3 ] || { usage; exit 2; }
recover_pr_operation_lock "$pr" "$owner_oid" "$confirmation"
return
fi
case "$cmd" in
ls) ;;
gc)
[ "$#" -eq 0 ] || { [ "$#" -eq 1 ] && [ "$1" = "--dry-run" ]; } || {
usage
exit 2
}
;;
review-tests)
[ "$#" -ge 2 ] || { usage; exit 2; }
;;
review-init | review-checkout-main | review-checkout-pr | review-claim | review-guard | review-artifacts-init | review-validate-artifacts | prepare-init | prepare-validate-commit | prepare-gates | prepare-push | prepare-sync-head | prepare-run | merge-verify | merge-run)
[ "$#" -ge 1 ] || { usage; exit 2; }
;;
*)
usage
exit 2
;;
esac
require_cmds
if is_locked_pr_command "$cmd"; then
local locked_pr="${1-}"
acquire_pr_operation_lock "$locked_pr"
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 131' QUIT
trap 'exit 143' TERM
fi
case "$cmd" in
ls)
list_pr_worktrees
;;
gc)
local dry_run=false
if [ "${1-}" = "--dry-run" ]; then
if [ "$#" -eq 1 ]; then
dry_run=true
fi
gc_pr_worktrees "$dry_run"
+31 -32
View File
@@ -185,38 +185,30 @@ common_repo_root() {
worktree_path_for_branch() {
local branch="$1"
local ref="refs/heads/$branch"
git worktree list --porcelain | awk -v ref="$ref" '
/^worktree / {
worktree=$2
next
}
/^branch / {
if ($2 == ref) {
print worktree
found=1
}
}
END {
if (!found) {
exit 1
}
}
'
local field worktree=""
while IFS= read -r -d '' field; do
case "$field" in
worktree\ *) worktree="${field#worktree }" ;;
"branch $ref")
[ -n "$worktree" ] || return 1
printf '%s\n' "$worktree"
return 0
;;
"") worktree="" ;;
esac
done < <(git worktree list --porcelain -z)
return 1
}
worktree_is_registered() {
local path="$1"
git worktree list --porcelain | awk -v target="$path" '
/^worktree / {
if ($2 == target) {
found=1
}
}
END {
exit found ? 0 : 1
}
'
local field
while IFS= read -r -d '' field; do
case "$field" in
worktree\ *) [ "${field#worktree }" = "$path" ] && return 0 ;;
esac
done < <(git worktree list --porcelain -z)
return 1
}
resolve_existing_dir_path() {
@@ -263,20 +255,27 @@ remove_worktree_if_present() {
return 0
fi
if worktree_is_registered "$path"; then
git worktree remove "$path" --force >/dev/null 2>&1 || true
if [ -L "$path" ] || ! is_repo_pr_worktree_dir "$path"; then
echo "Warning: refusing to remove non-canonical PR-worktree path $path"
return 0
fi
local registered_path
registered_path="$(resolve_existing_dir_path "$(dirname "$path")")/$(basename "$path")"
if [ -n "$registered_path" ] && worktree_is_registered "$registered_path"; then
git worktree remove "$registered_path" --force >/dev/null 2>&1 || true
fi
if [ ! -e "$path" ]; then
return 0
fi
if worktree_is_registered "$path"; then
if [ -n "$registered_path" ] && worktree_is_registered "$registered_path"; then
echo "Warning: failed to remove registered worktree $path"
return 0
fi
if ! is_repo_pr_worktree_dir "$path"; then
if [ -L "$path" ] || ! is_repo_pr_worktree_dir "$path"; then
echo "Warning: refusing to trash non-PR-worktree path $path"
return 0
fi
+388
View File
@@ -0,0 +1,388 @@
# Per-PR process lock shared by review, prepare, merge, and worktree GC.
PR_OPERATION_LOCK_REF=""
PR_OPERATION_LOCK_OWNER_OID=""
PR_OPERATION_LOCK_CANDIDATE_PR=""
PR_OPERATION_LOCK_CANDIDATE_OID=""
PR_OPERATION_LOCK_BLOCKED_OID=""
PR_OPERATION_LOCK_BLOCKED_REASON=""
is_canonical_pr_number() {
local pr="$1"
case "$pr" in ''|0|0*|*[!0-9]*) return 1 ;; esac
}
pr_operation_lock_ref() {
local pr="$1"
is_canonical_pr_number "$pr" || return 1
printf 'refs/openclaw/pr-operation-locks/%s\n' "$pr"
}
pr_operation_lock_zero_oid() {
local object_format
object_format=$(git -C "$(repo_root)" rev-parse --show-object-format 2>/dev/null) || return 1
case "$object_format" in
sha1) printf '%040d\n' 0 ;;
sha256) printf '%064d\n' 0 ;;
*) return 1 ;;
esac
}
pr_operation_lock_process_identity() {
local pid="$1"
case "$pid" in ''|0|1|*[!0-9]*) return 1 ;; esac
TZ=UTC0 LC_ALL=C ps -o state= -o lstart= -p "$pid" 2>/dev/null | awk '
NF {
state = $1
$1 = ""
sub(/^[[:space:]]+/, "")
printf "%s\t%s\n", state, $0
found = 1
}
END { exit found ? 0 : 1 }
'
}
pr_operation_lock_process_birth() {
local identity state birth
identity=$(pr_operation_lock_process_identity "$1") || return 1
IFS=$'\t' read -r state birth <<<"$identity"
case "$state" in Z*) return 1 ;; esac
[ -n "$birth" ] || return 1
printf '%s\n' "$birth"
}
pr_operation_lock_process_group_status() {
local pgid="$1"
case "$pgid" in ''|0|1|*[!0-9]*) return 1 ;; esac
node -e '
const pgid = Number(process.argv[1]);
if (!Number.isSafeInteger(pgid) || pgid <= 1 || pgid > 0x7fffffff) {
process.stdout.write("indeterminate\n");
process.exit(0);
}
try {
process.kill(-pgid, 0);
process.stdout.write("live\n");
} catch (error) {
process.stdout.write(error?.code === "ESRCH" ? "dead\n" : "indeterminate\n");
}
' "$pgid"
}
read_pr_operation_lock_owner() {
local owner_oid="$1"
local object_type payload parsed
object_type=$(git -C "$(repo_root)" cat-file -t "$owner_oid" 2>/dev/null) || return 1
[ "$object_type" = "blob" ] || return 1
payload=$(git -C "$(repo_root)" cat-file blob "$owner_oid" 2>/dev/null) || return 1
parsed=$(printf '%s\n' "$payload" | awk -F= '
NR == 1 && $0 == "version=3" { next }
NR == 2 && $0 == "state=active" { next }
NR == 3 && NF == 2 && $1 == "pgid" && $2 ~ /^[1-9][0-9]*$/ && $2 > 1 && $2 <= 2147483647 {
pgid = $2
next
}
NR == 4 && NF == 2 && $1 == "supervisor_pid" && $2 ~ /^[1-9][0-9]*$/ && $2 > 1 && $2 <= 2147483647 {
supervisor_pid = $2
next
}
NR == 5 && NF == 2 && $1 == "supervisor_birth" && length($2) > 0 && index($2, "\t") == 0 {
supervisor_birth = substr($0, length($1) + 2)
next
}
NR == 6 && NF == 2 && $1 == "token" && length($2) > 0 && index($2, "\t") == 0 {
token = $2
next
}
{ invalid = 1 }
END {
if (invalid || NR != 6 || pgid == "" || supervisor_pid == "" || supervisor_birth == "" || token == "") {
exit 1
}
printf "%s\t%s\t%s\t%s\n", pgid, supervisor_pid, supervisor_birth, token
}
') || return 1
local owner_pgid supervisor_pid supervisor_birth owner_token
IFS=$'\t' read -r owner_pgid supervisor_pid supervisor_birth owner_token <<<"$parsed"
case "$owner_pgid" in ''|0|1|*[!0-9]*) return 1 ;; esac
case "$supervisor_pid" in ''|0|1|*[!0-9]*) return 1 ;; esac
[ -n "$supervisor_birth" ] || return 1
if [[ ! "$owner_token" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]]; then
return 1
fi
printf '%s\t%s\t%s\t%s\n' "$owner_pgid" "$supervisor_pid" "$supervisor_birth" "$owner_token"
}
clear_pr_operation_lock_state() {
PR_OPERATION_LOCK_REF=""
PR_OPERATION_LOCK_OWNER_OID=""
PR_OPERATION_LOCK_CANDIDATE_PR=""
PR_OPERATION_LOCK_CANDIDATE_OID=""
PR_OPERATION_LOCK_BLOCKED_OID=""
PR_OPERATION_LOCK_BLOCKED_REASON=""
}
pr_operation_lock_owner_is_current() {
local root="$1"
local lock_ref="$2"
local expected_oid="$3"
local current_oid ref_status=0
if git -C "$root" symbolic-ref -q "$lock_ref" >/dev/null 2>&1; then
return 2
fi
if current_oid=$(git -C "$root" rev-parse --verify "$lock_ref" 2>/dev/null); then
[ "$current_oid" = "$expected_oid" ] && return 0
return 1
fi
git -C "$root" show-ref --verify --quiet "$lock_ref" 2>/dev/null || ref_status=$?
[ "$ref_status" -eq 1 ] && return 1
return 2
}
release_pr_operation_lock() {
if [ -z "${PR_OPERATION_LOCK_REF:-}" ] || [ -z "${PR_OPERATION_LOCK_OWNER_OID:-}" ]; then
return 0
fi
if [ -n "${OPENCLAW_PR_LOCK_NOTIFY_FD:-}" ]; then
# The outer supervisor releases only after a clean group drain. A failed,
# interrupted, or controller-lost operation leaves this exact ref sticky.
clear_pr_operation_lock_state
return 0
fi
local root lock_ref owner_oid observed_oid ref_status
root=$(repo_root) || return 1
lock_ref="$PR_OPERATION_LOCK_REF"
owner_oid="$PR_OPERATION_LOCK_OWNER_OID"
local attempts=0
while true; do
# The expected old object makes release a compare-and-swap: a delayed
# owner can never delete a successor's lock.
if git -C "$root" update-ref --no-deref -d "$lock_ref" "$owner_oid" 2>/dev/null; then
clear_pr_operation_lock_state
return 0
fi
if observed_oid=$(git -C "$root" rev-parse --verify "$lock_ref" 2>/dev/null); then
if [ "$observed_oid" != "$owner_oid" ]; then
clear_pr_operation_lock_state
return 0
fi
else
ref_status=0
git -C "$root" show-ref --verify --quiet "$lock_ref" 2>/dev/null || ref_status=$?
if [ "$ref_status" -eq 1 ]; then
clear_pr_operation_lock_state
return 0
fi
if [ "$ref_status" -ne 0 ]; then
break
fi
fi
attempts=$((attempts + 1))
[ "$attempts" -lt 20 ] || break
sleep 0.05
done
echo "Unable to release the operation lock for ${lock_ref##*/}; the owner ref is unchanged." >&2
return 1
}
notify_pr_operation_lock_supervisor() {
if [ -z "${OPENCLAW_PR_LOCK_NOTIFY_FD:-}" ]; then
return 0
fi
case "$OPENCLAW_PR_LOCK_NOTIFY_FD" in ''|*[!0-9]*) return 1 ;; esac
printf '%s\t%s\n' "$PR_OPERATION_LOCK_REF" "$PR_OPERATION_LOCK_OWNER_OID" >&"$OPENCLAW_PR_LOCK_NOTIFY_FD"
}
recover_pr_operation_lock() {
local pr="$1"
local expected_oid="$2"
local confirmation="${3-}"
is_canonical_pr_number "$pr" || { echo "Invalid PR number: $pr" >&2; return 2; }
[[ "$expected_oid" =~ ^([0-9a-f]{40}|[0-9a-f]{64})$ ]] || {
echo "Invalid operation-lock owner OID: $expected_oid" >&2
return 2
}
if [ "$confirmation" != "--confirmed-no-running-tools" ]; then
echo "Recovery requires --confirmed-no-running-tools after checking for detached PR tools." >&2
return 2
fi
local root lock_ref observed_oid
root=$(repo_root) || return 1
lock_ref=$(pr_operation_lock_ref "$pr") || return 1
observed_oid=$(git -C "$root" rev-parse --verify "$lock_ref" 2>/dev/null) || {
echo "PR #$pr has no operation lock to recover." >&2
return 1
}
if [ "$observed_oid" != "$expected_oid" ]; then
echo "PR #$pr operation-lock owner changed; refusing to delete $observed_oid." >&2
return 1
fi
# PGID liveness cannot exclude a detached child or unrelated PGID reuse.
# Recovery authority is the explicit confirmation plus this exact-OID CAS.
if ! git -C "$root" update-ref --no-deref -d "$lock_ref" "$expected_oid" 2>/dev/null; then
echo "PR #$pr operation-lock owner changed during recovery; nothing was deleted." >&2
return 1
fi
echo "Recovered the stale operation lock for PR #$pr."
}
prepare_pr_operation_lock_candidate() {
local pr="$1"
if [ "${PR_OPERATION_LOCK_CANDIDATE_PR:-}" = "$pr" ] && [ -n "${PR_OPERATION_LOCK_CANDIDATE_OID:-}" ]; then
return 0
fi
local root token group_status supervisor_pid supervisor_birth owner_oid
root=$(repo_root) || return 1
token=$(node -e 'process.stdout.write(require("node:crypto").randomUUID())') || return 1
group_status=$(pr_operation_lock_process_group_status "$$") || return 1
[ "$group_status" = "live" ] || return 1
supervisor_pid="${OPENCLAW_PR_LOCK_SUPERVISOR_PID:-$$}"
case "$supervisor_pid" in ''|0|1|*[!0-9]*) return 1 ;; esac
supervisor_birth=$(pr_operation_lock_process_birth "$supervisor_pid") || return 1
owner_oid=$(printf 'version=3\nstate=active\npgid=%s\nsupervisor_pid=%s\nsupervisor_birth=%s\ntoken=%s\n' \
"$$" "$supervisor_pid" "$supervisor_birth" "$token" |
git -C "$root" hash-object -w --stdin) || return 1
PR_OPERATION_LOCK_CANDIDATE_PR="$pr"
PR_OPERATION_LOCK_CANDIDATE_OID="$owner_oid"
}
try_acquire_pr_operation_lock() {
local pr="$1"
is_canonical_pr_number "$pr" || return 2
PR_OPERATION_LOCK_BLOCKED_OID=""
PR_OPERATION_LOCK_BLOCKED_REASON=""
local root lock_ref zero_oid owner_oid
root=$(repo_root) || return 2
lock_ref=$(pr_operation_lock_ref "$pr") || return 2
zero_oid=$(pr_operation_lock_zero_oid) || return 2
prepare_pr_operation_lock_candidate "$pr" || return 2
owner_oid="$PR_OPERATION_LOCK_CANDIDATE_OID"
local unreadable_ref_attempts=0
while true; do
if git -C "$root" update-ref --no-deref "$lock_ref" "$owner_oid" "$zero_oid" 2>/dev/null; then
PR_OPERATION_LOCK_REF="$lock_ref"
PR_OPERATION_LOCK_OWNER_OID="$owner_oid"
if ! notify_pr_operation_lock_supervisor; then
PR_OPERATION_LOCK_BLOCKED_OID="$owner_oid"
PR_OPERATION_LOCK_BLOCKED_REASON="not reported to its supervisor"
return 2
fi
return 0
fi
local observed_oid owner_data owner_pgid supervisor_pid supervisor_birth owner_token group_status
if git -C "$root" symbolic-ref -q "$lock_ref" >/dev/null 2>&1; then
return 2
fi
if ! observed_oid=$(git -C "$root" rev-parse --verify "$lock_ref" 2>/dev/null); then
# The supervisor may have released between our failed create-CAS and
# this read. A newly installed successor can also appear immediately,
# so one read miss is always a normal retry.
unreadable_ref_attempts=$((unreadable_ref_attempts + 1))
if [ "$unreadable_ref_attempts" -le 20 ]; then
# A concurrent exact release can leave a short delete-to-create window,
# including Git's transient ref lock. Bound the wait so a persistently
# unreadable ref still fails closed.
sleep 0.05
continue
fi
return 2
fi
unreadable_ref_attempts=0
if ! owner_data=$(read_pr_operation_lock_owner "$observed_oid"); then
local owner_status=0
pr_operation_lock_owner_is_current "$root" "$lock_ref" "$observed_oid" || owner_status=$?
[ "$owner_status" -eq 1 ] && continue
[ "$owner_status" -eq 0 ] || return 2
PR_OPERATION_LOCK_BLOCKED_OID="$observed_oid"
PR_OPERATION_LOCK_BLOCKED_REASON="unreadable"
return 2
fi
IFS=$'\t' read -r owner_pgid supervisor_pid supervisor_birth owner_token <<<"$owner_data"
local supervisor_identity supervisor_state current_supervisor_birth
supervisor_identity=$(pr_operation_lock_process_identity "$supervisor_pid" 2>/dev/null || true)
supervisor_state=""
current_supervisor_birth=""
if [ -n "$supervisor_identity" ]; then
IFS=$'\t' read -r supervisor_state current_supervisor_birth <<<"$supervisor_identity"
fi
# A group is active only while the exact supervisor incarnation still owns
# it. A reused PGID or orphaned descendant must surface explicit recovery.
group_status=$(pr_operation_lock_process_group_status "$owner_pgid") || return 2
case "$group_status" in
live | dead)
if [[ "$supervisor_state" != Z* ]] &&
[ -n "$current_supervisor_birth" ] &&
[ "$current_supervisor_birth" = "$supervisor_birth" ]
then
# A dead group can precede its controller's final drain and exact
# release. Waiting also covers the ordinary live-operation case.
return 1
fi
local owner_status=0
pr_operation_lock_owner_is_current "$root" "$lock_ref" "$observed_oid" || owner_status=$?
[ "$owner_status" -eq 1 ] && continue
[ "$owner_status" -eq 0 ] || return 2
# A missing controller cannot disprove a nested detached tool, even
# when its old group is dead. Only exact-OID recovery may clear it.
PR_OPERATION_LOCK_BLOCKED_OID="$observed_oid"
PR_OPERATION_LOCK_BLOCKED_REASON="orphaned"
return 2
;;
*)
local owner_status=0
pr_operation_lock_owner_is_current "$root" "$lock_ref" "$observed_oid" || owner_status=$?
[ "$owner_status" -eq 1 ] && continue
[ "$owner_status" -eq 0 ] || return 2
PR_OPERATION_LOCK_BLOCKED_OID="$observed_oid"
PR_OPERATION_LOCK_BLOCKED_REASON="indeterminate"
return 2
;;
esac
done
}
acquire_pr_operation_lock() {
local pr="$1"
local announced=false
local lock_status=0
while true; do
try_acquire_pr_operation_lock "$pr" || lock_status=$?
if [ "$lock_status" -eq 0 ]; then
return 0
fi
if [ "$lock_status" -ne 1 ]; then
if [ -n "$PR_OPERATION_LOCK_BLOCKED_OID" ]; then
echo "The prior PR #$pr operation lock is $PR_OPERATION_LOCK_BLOCKED_REASON; detached child tools cannot be ruled out." >&2
print_pr_operation_lock_recovery_guidance "$pr"
fi
echo "Unable to acquire the operation lock for PR #$pr." >&2
return "$lock_status"
fi
if [ "$announced" = "false" ]; then
echo "Waiting for the active scripts/pr operation on PR #$pr to finish..." >&2
announced=true
fi
lock_status=0
sleep 0.2
done
}
print_pr_operation_lock_recovery_guidance() {
local pr="$1"
[ -n "${PR_OPERATION_LOCK_BLOCKED_OID:-}" ] || return 1
echo "After verifying that no PR #$pr tools remain, recover the exact owner with:" >&2
echo " scripts/pr lock-recover $pr $PR_OPERATION_LOCK_BLOCKED_OID --confirmed-no-running-tools" >&2
}
+377
View File
@@ -0,0 +1,377 @@
import { spawn, spawnSync } from "node:child_process";
import { constants } from "node:os";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
const SIGNAL_GRACE_MS = 5000;
const KILL_DRAIN_MS = 5000;
const POLL_MS = 25;
const MAX_NOTIFICATION_LINE_BYTES = 4096;
const FORWARDED_SIGNALS = ["SIGHUP", "SIGINT", "SIGQUIT", "SIGTERM"];
const [repoRootArg, script, ...args] = process.argv.slice(2);
if (!repoRootArg || !script) {
console.error("process-group-runner requires a repository root and script path");
process.exit(2);
}
if (process.platform === "win32") {
console.error("scripts/pr operation locking requires a POSIX process group (use WSL on Windows)");
process.exit(1);
}
const repoRoot = resolve(repoRootArg);
const lockScript = fileURLToPath(new URL("./operation-lock.sh", import.meta.url));
const locks = new Map();
let notificationBuffer = "";
let discardingOversizedNotificationLine = false;
let notificationEnded = false;
/** @type {Error | undefined} */
let notificationFailure;
let receivedSignal;
let escalationTimer;
let killDeadline;
const operationGroup = { pid: undefined };
let operationGroupGone = false;
let hadLingeringGroup = false;
function delay(ms) {
return new Promise((resolveDelay) => {
setTimeout(resolveDelay, ms);
});
}
function toError(value, fallbackMessage) {
return value instanceof Error ? value : new Error(fallbackMessage);
}
function exitCodeForSignal(signal) {
const signalNumber = constants.signals[signal];
return typeof signalNumber === "number" ? 128 + signalNumber : 1;
}
function processGroupStatus(pgid) {
if (operationGroupGone) {
return "dead";
}
if (!Number.isSafeInteger(pgid) || pgid <= 1 || pgid > 0x7fffffff) {
return "indeterminate";
}
try {
process.kill(-pgid, 0);
return "live";
} catch (error) {
if (error?.code === "ESRCH") {
// Once absent, this operation group is gone forever. Never let later
// PGID reuse redirect a delayed signal or liveness probe.
operationGroupGone = true;
return "dead";
}
return "indeterminate";
}
}
function signalProcessGroup(signal) {
const childPid = operationGroup.pid;
if (!childPid || operationGroupGone) {
return;
}
try {
process.kill(-childPid, signal);
} catch (error) {
if (error?.code === "ESRCH") {
operationGroupGone = true;
} else {
notificationFailure ??= new Error(
`Unable to signal scripts/pr process group with ${signal}: ${String(error)}`,
);
}
}
}
function escalateSignal() {
if (killDeadline) {
return;
}
killDeadline = Date.now() + KILL_DRAIN_MS;
signalProcessGroup("SIGKILL");
}
const signalHandlers = new Map();
for (const signal of FORWARDED_SIGNALS) {
const handler = () => {
if (receivedSignal) {
escalateSignal();
return;
}
receivedSignal = signal;
signalProcessGroup(signal);
escalationTimer = setTimeout(escalateSignal, SIGNAL_GRACE_MS);
};
signalHandlers.set(signal, handler);
process.on(signal, handler);
}
const child = spawn(script, args, {
cwd: process.cwd(),
detached: true,
env: {
...process.env,
OPENCLAW_PR_DEDICATED_PROCESS_GROUP: "1",
OPENCLAW_PR_LOCK_NOTIFY_FD: "3",
OPENCLAW_PR_LOCK_SUPERVISOR_PID: String(process.pid),
},
stdio: ["inherit", "inherit", "inherit", "pipe"],
});
operationGroup.pid = child.pid;
if (killDeadline) {
signalProcessGroup("SIGKILL");
} else if (receivedSignal) {
signalProcessGroup(receivedSignal);
}
function consumeNotificationLine(line) {
const [lockRef, ownerOid, extra] = line.split("\t");
if (
extra !== undefined ||
!/^refs\/openclaw\/pr-operation-locks\/[1-9][0-9]*$/u.test(lockRef ?? "") ||
!/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u.test(ownerOid ?? "")
) {
notificationFailure ??= new Error("scripts/pr emitted malformed operation-lock metadata");
return;
}
const owner = spawnSync("git", ["-C", repoRoot, "cat-file", "blob", ownerOid], {
encoding: "utf8",
stdio: ["ignore", "pipe", "ignore"],
});
const ownerMatch =
owner.status === 0
? /^version=3\nstate=active\npgid=([1-9][0-9]*)\nsupervisor_pid=([1-9][0-9]*)\nsupervisor_birth=[^\t\n]+\ntoken=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\n?$/u.exec(
owner.stdout,
)
: undefined;
const ownerPgid = ownerMatch ? Number(ownerMatch[1]) : undefined;
const supervisorPid = ownerMatch ? Number(ownerMatch[2]) : undefined;
if (
ownerPgid === undefined ||
supervisorPid === undefined ||
!Number.isSafeInteger(ownerPgid) ||
!Number.isSafeInteger(supervisorPid) ||
ownerPgid <= 1 ||
supervisorPid <= 1 ||
ownerPgid > 0x7fffffff ||
supervisorPid > 0x7fffffff ||
ownerPgid !== child?.pid ||
supervisorPid !== process.pid
) {
notificationFailure ??= new Error(
"scripts/pr emitted an operation lock owned by another process group",
);
return;
}
locks.set(`${lockRef}\0${ownerOid}`, { lockRef, ownerOid });
}
function finishNotifications() {
if (notificationEnded) {
return;
}
if (!discardingOversizedNotificationLine && notificationBuffer.length > 0) {
consumeNotificationLine(notificationBuffer);
}
notificationBuffer = "";
notificationEnded = true;
}
function consumeNotificationChunk(chunk) {
notificationBuffer += chunk;
while (true) {
const newline = notificationBuffer.indexOf("\n");
if (discardingOversizedNotificationLine) {
if (newline === -1) {
notificationBuffer = "";
return;
}
notificationBuffer = notificationBuffer.slice(newline + 1);
discardingOversizedNotificationLine = false;
continue;
}
if (newline === -1) {
if (Buffer.byteLength(notificationBuffer) > MAX_NOTIFICATION_LINE_BYTES) {
notificationFailure ??= new Error("scripts/pr operation-lock metadata line is too large");
notificationBuffer = "";
discardingOversizedNotificationLine = true;
}
return;
}
const line = notificationBuffer.slice(0, newline);
notificationBuffer = notificationBuffer.slice(newline + 1);
if (Buffer.byteLength(line) > MAX_NOTIFICATION_LINE_BYTES) {
notificationFailure ??= new Error("scripts/pr operation-lock metadata line is too large");
continue;
}
consumeNotificationLine(line);
}
}
const notificationStream = child.stdio[3];
notificationStream.setEncoding("utf8");
notificationStream.on("data", consumeNotificationChunk);
notificationStream.once("error", (error) => {
notificationFailure ??= toError(error, "scripts/pr operation-lock notification stream failed");
});
notificationStream.once("end", finishNotifications);
notificationStream.once("close", finishNotifications);
const childResult = await new Promise((resolveResult) => {
let settled = false;
const settle = (result) => {
if (settled) {
return;
}
settled = true;
resolveResult(result);
};
child.once("error", (error) => {
notificationFailure ??= toError(error, "Unable to launch scripts/pr");
settle({ code: 1, signal: null });
});
child.once("exit", (code, signal) => settle({ code, signal }));
});
const postExitGroupStatus = child.pid ? processGroupStatus(child.pid) : "dead";
if (postExitGroupStatus === "indeterminate") {
notificationFailure ??= new Error("scripts/pr process-group state became indeterminate");
} else if (postExitGroupStatus === "live") {
// A wrapper exit does not end same-group background work. Bound and drain
// forgotten jobs, but keep the lock because their terminal state is unknown.
hadLingeringGroup = true;
notificationFailure ??= new Error("scripts/pr process group remained active after wrapper exit");
signalProcessGroup("SIGTERM");
escalationTimer ??= setTimeout(escalateSignal, SIGNAL_GRACE_MS);
} else if (!notificationEnded) {
// A detached descendant may be the last writer. It cannot be signalled by
// this group supervisor, so bound the wait and retain the lock on timeout.
killDeadline ??= Date.now() + KILL_DRAIN_MS;
}
async function waitForOperationDrain() {
while (true) {
const groupStatus = child.pid ? processGroupStatus(child.pid) : "dead";
if (groupStatus === "indeterminate") {
throw new Error("scripts/pr process-group state became indeterminate");
}
if (groupStatus === "dead" && notificationEnded) {
return;
}
if (killDeadline && Date.now() >= killDeadline) {
throw new Error(
`scripts/pr operation lifetime did not drain (group=${groupStatus}, pipe=${notificationEnded ? "closed" : "open"})`,
);
}
await delay(POLL_MS);
}
}
function releaseLock({ lockRef, ownerOid }) {
const env = { ...process.env };
delete env.OPENCLAW_PR_LOCK_NOTIFY_FD;
const result = spawnSync(
"bash",
[
"-c",
[
"set -euo pipefail",
'source "$1"',
'SUPERVISOR_REPO_ROOT="$2"',
"repo_root() { printf '%s\\n' \"$SUPERVISOR_REPO_ROOT\"; }",
'PR_OPERATION_LOCK_REF="$3"',
'PR_OPERATION_LOCK_OWNER_OID="$4"',
"release_pr_operation_lock",
].join("\n"),
"operation-lock-release",
lockScript,
repoRoot,
lockRef,
ownerOid,
],
{ encoding: "utf8", env, stdio: ["ignore", "pipe", "pipe"] },
);
if (result.status !== 0) {
throw new Error(
result.stderr.trim() ||
`Unable to release the operation lock for ${lockRef.split("/").at(-1)}`,
);
}
}
function reportRetainedLock({ lockRef, ownerOid }) {
const pr = lockRef.split("/").at(-1);
console.error(
`Retaining the operation lock for PR #${pr}; detached child tools cannot be ruled out.`,
);
console.error(`After verifying that no PR #${pr} tools remain, recover the exact owner with:`);
console.error(` scripts/pr lock-recover ${pr} ${ownerOid} --confirmed-no-running-tools`);
}
let drained = false;
try {
await waitForOperationDrain();
drained = true;
} catch (error) {
notificationFailure ??= toError(error, "scripts/pr operation drain failed");
// An out-of-group descendant can inherit the write end indefinitely. Once
// the bounded drain fails, close our read end so that sentinel cannot keep
// the controller alive; the exact lock remains sticky for manual recovery.
finishNotifications();
notificationStream.destroy();
}
if (escalationTimer) {
clearTimeout(escalationTimer);
}
for (const [signal, handler] of signalHandlers) {
process.off(signal, handler);
}
// PR commands must join all state-mutating children before returning. A clean
// exit is that trusted completion signal; abnormal exits retain the lock because
// an escaped child can outlive both the recorded group and notification pipe.
const completedCleanly =
childResult.code === 0 &&
!receivedSignal &&
!childResult.signal &&
!notificationFailure &&
!hadLingeringGroup;
const retainedLocks = [];
if (drained && completedCleanly) {
for (const lock of locks.values()) {
try {
releaseLock(lock);
} catch (error) {
notificationFailure ??= toError(error, "Unable to release a scripts/pr operation lock");
retainedLocks.push(lock);
}
}
} else {
retainedLocks.push(...locks.values());
}
for (const lock of retainedLocks) {
reportRetainedLock(lock);
}
if (notificationFailure) {
console.error(notificationFailure.message);
}
if (receivedSignal) {
process.exitCode = exitCodeForSignal(receivedSignal);
} else if (childResult.code !== null) {
process.exitCode = childResult.code;
} else {
process.exitCode = childResult.signal ? exitCodeForSignal(childResult.signal) : 1;
}
if (notificationFailure && process.exitCode === 0) {
process.exitCode = 1;
}
+30 -10
View File
@@ -139,22 +139,45 @@ gc_pr_worktrees() {
echo "skipping $dir (could not parse PR number)"
continue
fi
local lock_status=0
try_acquire_pr_operation_lock "$pr" || lock_status=$?
if [ "$lock_status" -ne 0 ]; then
if [ "$lock_status" -eq 1 ]; then
echo "skipping $dir (PR #$pr has an active scripts/pr operation)"
elif [ -n "$PR_OPERATION_LOCK_BLOCKED_OID" ]; then
echo "skipping $dir (PR #$pr operation lock is $PR_OPERATION_LOCK_BLOCKED_REASON)"
print_pr_operation_lock_recovery_guidance "$pr"
else
echo "skipping $dir (PR #$pr operation lock state is indeterminate)"
fi
continue
fi
local state
state=$(gh pr view "$pr" --json state --jq .state 2>/dev/null || printf 'UNKNOWN')
case "$state" in
MERGED|CLOSED)
if [ "$dry_run" = "true" ]; then
echo "would remove $dir (PR #$pr state=$state)"
removed=$((removed + 1))
else
remove_worktree_if_present "$dir"
delete_local_branch_if_safe "temp/pr-$pr"
delete_local_branch_if_safe "pr-$pr"
delete_local_branch_if_safe "pr-$pr-prep"
echo "removed $dir (PR #$pr state=$state)"
if [ ! -e "$dir" ] &&
! git show-ref --verify --quiet "refs/heads/temp/pr-$pr" &&
! git show-ref --verify --quiet "refs/heads/pr-$pr" &&
! git show-ref --verify --quiet "refs/heads/pr-$pr-prep"
then
echo "removed $dir (PR #$pr state=$state)"
removed=$((removed + 1))
else
echo "skipping $dir (cleanup incomplete)"
fi
fi
removed=$((removed + 1))
;;
esac
release_pr_operation_lock
done
if [ "$removed" -eq 0 ]; then
@@ -168,12 +191,9 @@ gc_pr_worktrees() {
pr_number_from_worktree_dir() {
local dir="$1"
local token
token="${dir##*/pr-}"
token="${token%%[^0-9]*}"
if [ -n "$token" ]; then
printf '%s\n' "$token"
return 0
fi
return 1
local basename=${dir##*/}
local token=${basename#pr-}
[ "$basename" != "$token" ] || return 1
is_canonical_pr_number "$token" || return 1
printf '%s\n' "$token"
}
+4 -1
View File
@@ -1402,7 +1402,9 @@ const TOOLING_SOURCE_TEST_TARGETS = new Map([
["scripts/mobile-reauth.sh", ["test/scripts/auth-monitor.test.ts"]],
["scripts/committer", ["test/scripts/committer.test.ts"]],
["scripts/gh-read", ["test/scripts/gh-read.test.ts"]],
["scripts/pr", ["test/scripts/pr-wrappers.test.ts"]],
["scripts/pr", ["test/scripts/pr-operation-lock.test.ts", "test/scripts/pr-wrappers.test.ts"]],
["scripts/pr-lib/operation-lock.sh", ["test/scripts/pr-operation-lock.test.ts"]],
["scripts/pr-lib/process-group-runner.mjs", ["test/scripts/pr-operation-lock.test.ts"]],
["scripts/pr-merge", ["test/scripts/pr-wrappers.test.ts"]],
["scripts/pr-prepare", ["test/scripts/pr-wrappers.test.ts"]],
["scripts/pr-review", ["test/scripts/pr-wrappers.test.ts"]],
@@ -2062,6 +2064,7 @@ const TOOLING_TEST_TARGETS = new Map([
"test/scripts/plugin-prerelease-test-plan.test.ts",
["test/scripts/plugin-prerelease-test-plan.test.ts"],
],
["test/scripts/pr-operation-lock.test.ts", ["test/scripts/pr-operation-lock.test.ts"]],
["test/scripts/pr-wrappers.test.ts", ["test/scripts/pr-wrappers.test.ts"]],
["test/scripts/test-projects.test.ts", ["test/scripts/test-projects.test.ts"]],
[
File diff suppressed because it is too large Load Diff
+13 -1
View File
@@ -492,6 +492,13 @@ describe("scripts/test-projects changed-target routing", () => {
}
});
it("keeps shared PR worktree helper edits on the full tooling owner suite", () => {
expect(resolveChangedTestTargetPlan(["scripts/pr-lib/worktree.sh"])).toEqual({
mode: "targets",
targets: ["test/vitest/vitest.tooling.config.ts"],
});
});
it("routes nested e2e shell helpers through their sourced owner tests", () => {
const expectedTargets = new Map([
[
@@ -1625,7 +1632,12 @@ describe("scripts/test-projects changed-target routing", () => {
const expectedTargets = new Map([
["scripts/committer", ["test/scripts/committer.test.ts"]],
["scripts/gh-read", ["test/scripts/gh-read.test.ts"]],
["scripts/pr", ["test/scripts/pr-wrappers.test.ts"]],
[
"scripts/pr",
["test/scripts/pr-operation-lock.test.ts", "test/scripts/pr-wrappers.test.ts"],
],
["scripts/pr-lib/operation-lock.sh", ["test/scripts/pr-operation-lock.test.ts"]],
["scripts/pr-lib/process-group-runner.mjs", ["test/scripts/pr-operation-lock.test.ts"]],
["scripts/pr-merge", ["test/scripts/pr-wrappers.test.ts"]],
["scripts/pr-prepare", ["test/scripts/pr-wrappers.test.ts"]],
["scripts/pr-review", ["test/scripts/pr-wrappers.test.ts"]],