diff --git a/.github/codex/prompts/mantis-telegram-desktop-proof.md b/.github/codex/prompts/mantis-telegram-desktop-proof.md index bdc242fca250..7facc2d26d10 100644 --- a/.github/codex/prompts/mantis-telegram-desktop-proof.md +++ b/.github/codex/prompts/mantis-telegram-desktop-proof.md @@ -8,7 +8,7 @@ continuous event recording, capture, and cleanup. - No PR mutations, commits, pushes, labels, reviews, or merges. - Do not read prepared worktrees. Pass their exact paths only to the lane helper. -- Write only under `MANTIS_OUTPUT_DIR`. +- Write only under `MANTIS_OUTPUT_DIR` and the fixture staging directory described below. - Never invent a pass, hide an attempt, edit trusted facts/media, or use old chat history. - A visible defect is a failure. An unproven comparison is `block`, not a pass. @@ -48,6 +48,16 @@ mock OpenAI endpoint; the QA user is the gateway owner, so owner commands such a `/send off` work without a patch. Optional field: `mockResponseChunkDelayMs`. +For scenarios that need an agent-authored plugin, write a complete plugin package +under `MANTIS_FIXTURE_PLUGINS_DIR/baseline` and/or +`MANTIS_FIXTURE_PLUGINS_DIR/candidate` before `start`. The harness copies the +selected lane directory into that lane's isolated SUT; fixture code never runs on +the runner host. Add the fixture id through `configPatch.plugins.allow` while +retaining `telegram` and `openai`, then enable it through its entry or owning slot. +Do not set `plugins.load.paths`; the harness owns that path. Use the same fixture +package in both lane directories for a fair comparison unless different fixtures +are an explicit part of the scenario. + ## Primitive CLI Use `$OPENCLAW_TELEGRAM_MANTIS_LANE_CMD` with `--lane baseline|candidate`: diff --git a/.github/workflows/mantis-telegram-desktop-proof.yml b/.github/workflows/mantis-telegram-desktop-proof.yml index 2b6a87261c2e..7abe59443feb 100644 --- a/.github/workflows/mantis-telegram-desktop-proof.yml +++ b/.github/workflows/mantis-telegram-desktop-proof.yml @@ -712,6 +712,16 @@ jobs: echo "Shared QA Telegram account is busy; retrying in 15s." >&2 sleep 15 done + keepalive_pid_file="$credential_dir/lease-keepalive.pid" + lease_lost_marker="$credential_dir/lease.json.lost" + keepalive_log="$credential_dir/lease-keepalive.log" + rm -f "$keepalive_pid_file" "$lease_lost_marker" "$keepalive_log" + /usr/bin/setsid /usr/local/lib/mantis-toolchain/node --import tsx \ + scripts/e2e/telegram-user-credential.ts heartbeat-loop \ + --lease-file "$credential_dir/lease.json" \ + --credential-role ci \ + --interval-ms 30000 "$keepalive_log" 2>&1 & + printf '%s\n' "$!" >"$keepalive_pid_file" chmod 0700 "$credential_dir" "$credential_dir/user-driver" sut_credential_dir="/tmp/openclaw-mantis-sut-credential-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" session_root="/tmp/openclaw-mantis-proof-sessions-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" @@ -729,6 +739,8 @@ jobs: echo "state_dir=$credential_dir/user-driver" echo "sut_credential_dir=$sut_credential_dir" echo "session_root=$session_root" + echo "lease_keepalive_pid_file=$keepalive_pid_file" + echo "lease_lost_marker=$lease_lost_marker" } >> "$GITHUB_OUTPUT" - name: Ensure agent key exists @@ -750,7 +762,7 @@ jobs: { printf '%s\n' 'Defaults env_keep += "CODEX_HOME CODEX_INTERNAL_ORIGINATOR_OVERRIDE"' printf '%s\n' 'Defaults env_keep += "BASELINE_REF BASELINE_SHA CANDIDATE_REF CANDIDATE_SHA"' - printf '%s\n' 'Defaults env_keep += "GITHUB_WORKSPACE MANTIS_BASELINE_ROOT MANTIS_CANDIDATE_ROOT MANTIS_INSTRUCTIONS MANTIS_OUTPUT_DIR MANTIS_PR_CONTEXT"' + printf '%s\n' 'Defaults env_keep += "GITHUB_WORKSPACE MANTIS_BASELINE_ROOT MANTIS_CANDIDATE_ROOT MANTIS_FIXTURE_PLUGINS_DIR MANTIS_INSTRUCTIONS MANTIS_OUTPUT_DIR MANTIS_PR_CONTEXT"' printf '%s\n' 'Defaults env_keep += "MANTIS_NODE_BIN MANTIS_PNPM_BIN"' printf '%s\n' 'Defaults env_keep += "OPENCLAW_TELEGRAM_MANTIS_LANE_CMD"' printf '%s\n' 'codex ALL=(mantis-sut) NOPASSWD: /usr/local/lib/mantis-toolchain/telegram-mantis-lane' @@ -763,6 +775,16 @@ jobs: sudo install -d -m 0770 -o codex -g codex "$codex_home" sudo setfacl -m u:runner:rwx,u:codex:rwx "$codex_home" sudo setfacl -d -m u:runner:rwx,u:codex:rwx "$codex_home" + session_root="/tmp/openclaw-mantis-proof-sessions-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + fixture_plugins_root="$session_root/fixture-plugins" + sudo setfacl -m u:codex:--x "$session_root" + sudo install -d -m 0710 -o root -g mantis-proof "$fixture_plugins_root" + sudo setfacl -m u:codex:--x "$fixture_plugins_root" + for lane in baseline candidate; do + sudo install -d -m 2770 -o codex -g mantis-proof "$fixture_plugins_root/$lane" + sudo setfacl -m u:mantis-sut:rwx "$fixture_plugins_root/$lane" + sudo setfacl -d -m u:codex:rwx,u:mantis-sut:rwx "$fixture_plugins_root/$lane" + done workspace_parent="$(dirname "$GITHUB_WORKSPACE")" while [ "$workspace_parent" != "/" ]; do sudo setfacl -m u:codex:--x,u:mantis-sut:--x "$workspace_parent" @@ -785,34 +807,60 @@ jobs: sudo find "$proof_worktree_root" -xdev ! -type l -perm /222 -exec chmod a-w {} + sudo chmod 0700 "$proof_worktree_root" - - name: Run Codex Mantis Telegram agent - # Pin audited 2026-08: runCodexExec rejects a missing codex-user, then - # launches the CLI as `sudo -u -- codex exec`. + - name: Prepare Codex action runtime uses: openai/codex-action@52fe01ec70a42f454c9d2ebd47598f9fd6893d56 + with: + openai-api-key: ${{ secrets.OPENCLAW_MANTIS_AGENT_OPENAI_API_KEY || secrets.OPENAI_API_KEY }} + codex-home: /tmp/mantis-codex-home-${{ github.run_id }} + safety-strategy: unprivileged-user + codex-user: codex + allow-bot-users: github-actions[bot] + + - name: Run Codex Mantis Telegram agent + # Pin audited 2026-08: preserve codex-action's unprivileged launch shape while + # making the broker's terminal lease-loss marker fence the complete process group. env: BASELINE_REF: ${{ needs.resolve_request.outputs.baseline_ref }} BASELINE_SHA: ${{ needs.resolve_request.outputs.baseline_revision }} CANDIDATE_REF: ${{ needs.resolve_request.outputs.candidate_ref }} CANDIDATE_SHA: ${{ needs.resolve_request.outputs.candidate_revision }} + CODEX_HOME: /tmp/mantis-codex-home-${{ github.run_id }} + CODEX_INTERNAL_ORIGINATOR_OVERRIDE: codex_github_action + CODEX_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_MODEL_BARE }} + FORCE_COLOR: "1" MANTIS_BASELINE_ROOT: /tmp/openclaw-mantis-proof-worktrees-${{ github.run_id }}-${{ github.run_attempt }}/baseline MANTIS_CANDIDATE_ROOT: /tmp/openclaw-mantis-proof-worktrees-${{ github.run_id }}-${{ github.run_attempt }}/candidate + MANTIS_FIXTURE_PLUGINS_DIR: /tmp/openclaw-mantis-proof-sessions-${{ github.run_id }}-${{ github.run_attempt }}/fixture-plugins MANTIS_INSTRUCTIONS: ${{ needs.resolve_request.outputs.instructions }} MANTIS_NODE_BIN: /usr/local/lib/mantis-toolchain/node MANTIS_OUTPUT_DIR: ${{ env.MANTIS_OUTPUT_DIR }} MANTIS_PR_CONTEXT: ${{ needs.resolve_request.outputs.pr_context }} MANTIS_PNPM_BIN: /usr/local/lib/mantis-toolchain/pnpm OPENCLAW_TELEGRAM_MANTIS_LANE_CMD: /usr/local/bin/openclaw-telegram-mantis-lane - with: - openai-api-key: ${{ secrets.OPENCLAW_MANTIS_AGENT_OPENAI_API_KEY || secrets.OPENAI_API_KEY }} - prompt-file: .github/codex/prompts/mantis-telegram-desktop-proof.md - model: ${{ vars.OPENCLAW_CI_OPENAI_MODEL_BARE }} - effort: high - sandbox: danger-full-access - codex-args: '["-c","service_tier=\"fast\""]' - codex-home: /tmp/mantis-codex-home-${{ github.run_id }} - safety-strategy: unprivileged-user - codex-user: codex - allow-bot-users: github-actions[bot] + shell: bash + run: | + set -euo pipefail + lease_lost_marker="${{ steps.telegram_credential.outputs.lease_lost_marker }}" + codex_bin="$(command -v codex)" + output_file="$CODEX_HOME/mantis-final-message-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt" + trap 'rm -f "$output_file"' EXIT + codex_args=( + exec + --skip-git-repo-check + --cd "$GITHUB_WORKSPACE" + --output-last-message "$output_file" + ) + if [[ -n "$CODEX_MODEL" ]]; then + codex_args+=(--model "$CODEX_MODEL") + fi + codex_args+=( + --config 'model_reasoning_effort="high"' + -c 'service_tier="fast"' + --sandbox danger-full-access + ) + scripts/mantis/run-with-lease-fence.sh "$lease_lost_marker" -- \ + sudo -u codex -- "$codex_bin" "${codex_args[@]}" \ + < .github/codex/prompts/mantis-telegram-desktop-proof.md - name: Clean up abandoned Mantis sessions id: abandoned_cleanup @@ -821,6 +869,10 @@ jobs: run: | set -euo pipefail result=0 + scripts/mantis/stop-lease-keepalive.sh \ + "${{ steps.telegram_credential.outputs.lease_keepalive_pid_file }}" \ + "${{ steps.telegram_credential.outputs.lease_file }}" \ + "$GITHUB_WORKSPACE" active_codex_pids() { sudo ps -u codex -o pid=,stat= 2>/dev/null | awk '$2 !~ /^Z/ {print $1}' || true } @@ -1162,9 +1214,18 @@ jobs: run: | set -euo pipefail lease_file="${{ steps.telegram_credential.outputs.lease_file }}" + scripts/mantis/stop-lease-keepalive.sh \ + "${{ steps.telegram_credential.outputs.lease_keepalive_pid_file }}" \ + "$lease_file" \ + "$GITHUB_WORKSPACE" if [[ -z "$lease_file" ]] || ! sudo test -f "$lease_file"; then exit 0 fi + lease_lost_marker="${{ steps.telegram_credential.outputs.lease_lost_marker }}" + if [[ -n "$lease_lost_marker" ]] && sudo test -f "$lease_lost_marker"; then + echo "::warning::lease lost mid-run; nothing to release" + exit 0 + fi sudo env \ OPENCLAW_QA_CONVEX_SECRET_CI="$OPENCLAW_QA_CONVEX_SECRET_CI" \ OPENCLAW_QA_CONVEX_SITE_URL="$OPENCLAW_QA_CONVEX_SITE_URL" \ diff --git a/scripts/e2e/telegram-mantis-lane.ts b/scripts/e2e/telegram-mantis-lane.ts index fcb5190cff00..135e3297f113 100644 --- a/scripts/e2e/telegram-mantis-lane.ts +++ b/scripts/e2e/telegram-mantis-lane.ts @@ -839,6 +839,7 @@ async function startLane(values: Map, roots: Roots): Promise; + fixturePluginsDir?: string; gatewayPort: number; groupId: string; mcpAppFixture?: boolean; @@ -192,6 +193,13 @@ export function writeSutConfig(params: { const workspace = path.join(tempRoot, "workspace"); fs.mkdirSync(stateDir, { recursive: true }); fs.mkdirSync(workspace, { recursive: true }); + let fixturePluginsRoot: string | undefined; + if (params.fixturePluginsDir) { + fixturePluginsRoot = path.join(tempRoot, "fixture-plugins"); + // Fixture code crosses into the same isolated runtime as candidate code. Copying keeps + // agent staging immutable from the SUT while adding no authority outside the container. + fs.cpSync(params.fixturePluginsDir, fixturePluginsRoot, { recursive: true }); + } const configPath = path.join(tempRoot, "openclaw.json"); const baseConfig = { agents: { @@ -287,6 +295,7 @@ export function writeSutConfig(params: { allow: ["telegram", "openai"], enabled: true, entries: { openai: { enabled: true }, telegram: { enabled: true } }, + ...(fixturePluginsRoot ? { load: { paths: [fixturePluginsRoot] } } : {}), }, }; const config = mergeConfig(baseConfig, params.configPatch ?? {}); @@ -545,6 +554,7 @@ function cleanupFailureMessage(message: string, cleanupErrors: unknown[]): strin export async function startMantisSut(params: { configPatch?: Record; + fixturePluginsDir?: string; gatewayPort: number; groupId: string; mockPort: number; diff --git a/scripts/e2e/telegram-user-credential.ts b/scripts/e2e/telegram-user-credential.ts index ae087f3377ba..01e284261b77 100644 --- a/scripts/e2e/telegram-user-credential.ts +++ b/scripts/e2e/telegram-user-credential.ts @@ -44,28 +44,24 @@ const CHUNKED_PAYLOAD_MAX_CHUNKS = optionalPositiveInteger( "OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_CHUNKS", ); +function usageText() { + return [ + "Usage:", + " node --import tsx scripts/e2e/telegram-user-credential.ts export (--desktop-tdata-dir | --desktop-tdata-archive ) --output ", + " node --import tsx scripts/e2e/telegram-user-credential.ts restore --payload-file --user-driver-dir --desktop-workdir ", + " node --import tsx scripts/e2e/telegram-user-credential.ts lease-restore --user-driver-dir --desktop-workdir --lease-file [--payload-output ] [--env-file ] [--credential-role maintainer|ci]", + " node --import tsx scripts/e2e/telegram-user-credential.ts heartbeat --lease-file [--env-file ] [--credential-role maintainer|ci]", + " node --import tsx scripts/e2e/telegram-user-credential.ts heartbeat-loop --lease-file [--interval-ms ] [--env-file ] [--credential-role maintainer|ci]", + " node --import tsx scripts/e2e/telegram-user-credential.ts release --lease-file [--env-file ] [--credential-role maintainer|ci]", + ].join("\n"); +} + function usage(): never { - throw new Error( - [ - "Usage:", - " node --import tsx scripts/e2e/telegram-user-credential.ts export (--desktop-tdata-dir | --desktop-tdata-archive ) --output ", - " node --import tsx scripts/e2e/telegram-user-credential.ts restore --payload-file --user-driver-dir --desktop-workdir ", - " node --import tsx scripts/e2e/telegram-user-credential.ts lease-restore --user-driver-dir --desktop-workdir --lease-file [--payload-output ] [--env-file ] [--credential-role maintainer|ci]", - " node --import tsx scripts/e2e/telegram-user-credential.ts release --lease-file [--env-file ] [--credential-role maintainer|ci]", - ].join("\n"), - ); + throw new Error(usageText()); } function printUsage() { - console.log( - [ - "Usage:", - " node --import tsx scripts/e2e/telegram-user-credential.ts export (--desktop-tdata-dir | --desktop-tdata-archive ) --output ", - " node --import tsx scripts/e2e/telegram-user-credential.ts restore --payload-file --user-driver-dir --desktop-workdir ", - " node --import tsx scripts/e2e/telegram-user-credential.ts lease-restore --user-driver-dir --desktop-workdir --lease-file [--payload-output ] [--env-file ] [--credential-role maintainer|ci]", - " node --import tsx scripts/e2e/telegram-user-credential.ts release --lease-file [--env-file ] [--credential-role maintainer|ci]", - ].join("\n"), - ); + console.log(usageText()); } function parseArgs(argv: string[]) { @@ -272,10 +268,22 @@ function joinBrokerEndpoint(siteUrl: string, endpoint: string) { return `${normalized}/qa-credentials/v1/${endpoint}`; } +class QaCredentialBrokerError extends Error { + readonly code: string; + + constructor(code: string, message: string) { + super(message); + this.code = code; + this.name = "QaCredentialBrokerError"; + } +} + function assertBrokerSuccess(payload: JsonObject, action: string) { if (payload.status === "error") { - throw new Error( - `${action} failed: ${requireString(payload, "code")} ${readOptionalCredentialString(payload, "message") || ""}`.trim(), + const code = requireString(payload, "code"); + throw new QaCredentialBrokerError( + code, + `${action} failed: ${code} ${readOptionalCredentialString(payload, "message") || ""}`.trim(), ); } if (payload.status !== "ok") { @@ -711,7 +719,7 @@ async function releaseTelegramUserLeaseBody(params: { }); } -async function releaseTelegramUserLease(opts: Map) { +async function resolveTelegramUserLeaseAction(opts: Map) { const leaseFile = opts.get("lease-file"); if (!leaseFile) { usage(); @@ -719,6 +727,89 @@ async function releaseTelegramUserLease(opts: Map) { const lease = await readJson(leaseFile); const leaseRole = resolveTelegramUserCredentialRole(requireString(lease, "actorRole")); const config = await resolveConvexLeaseConfig(opts, leaseRole); + return { config, lease, leaseFile }; +} + +async function heartbeatTelegramUserLeaseBody(params: { + lease: JsonObject; + leaseTtlMs: number; + siteUrl: string; + token: string; +}) { + return postBroker({ + action: "heartbeat", + siteUrl: params.siteUrl, + token: params.token, + body: { + kind: requireString(params.lease, "kind"), + ownerId: requireString(params.lease, "ownerId"), + actorRole: requireString(params.lease, "actorRole"), + credentialId: requireString(params.lease, "credentialId"), + leaseToken: requireString(params.lease, "leaseToken"), + leaseTtlMs: params.leaseTtlMs, + }, + }); +} + +async function heartbeatTelegramUserLease(opts: Map) { + const { config, lease } = await resolveTelegramUserLeaseAction(opts); + await heartbeatTelegramUserLeaseBody({ + lease, + leaseTtlMs: config.leaseTtlMs, + siteUrl: config.siteUrl, + token: config.token, + }); + console.log( + JSON.stringify({ status: "ok", credentialId: requireString(lease, "credentialId") }, null, 2), + ); +} + +async function heartbeatTelegramUserLeaseLoop(opts: Map) { + const { config, lease, leaseFile } = await resolveTelegramUserLeaseAction(opts); + const intervalMs = optionalPositiveInteger( + opts.get("interval-ms"), + config.heartbeatIntervalMs, + "interval-ms", + ); + const lostMarker = `${expandHome(leaseFile)}.lost`; + await unlink(lostMarker).catch((error: unknown) => { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + return; + } + throw error; + }); + + while (true) { + try { + await heartbeatTelegramUserLeaseBody({ + lease, + leaseTtlMs: config.leaseTtlMs, + siteUrl: config.siteUrl, + token: config.token, + }); + } catch (error) { + if ( + error instanceof QaCredentialBrokerError && + (error.code === "LEASE_NOT_OWNER" || error.code === "LEASE_EXPIRED") + ) { + await writePrivateJson(lostMarker, { code: error.code }); + return; + } + // Heartbeats are idempotent, so all non-terminal failures are safe to retry. A lapsed + // lease returns LEASE_EXPIRED next time, making loss observable through the marker. + const code = error instanceof Error && "code" in error ? error.code : undefined; + const discriminator = + typeof code === "string" ? code : error instanceof Error ? error.name : "unknown"; + console.error(`Credential lease heartbeat failed (${discriminator}); retrying.`); + } + await new Promise((resolve) => { + setTimeout(resolve, intervalMs); + }); + } +} + +async function releaseTelegramUserLease(opts: Map) { + const { config, lease, leaseFile } = await resolveTelegramUserLeaseAction(opts); await releaseTelegramUserLeaseBody({ siteUrl: config.siteUrl, token: config.token, @@ -743,6 +834,10 @@ async function main(argv = process.argv) { await restoreTelegramUserPayloadFromFile(opts); } else if (command === "lease-restore") { await leaseAndRestoreTelegramUser(opts); + } else if (command === "heartbeat") { + await heartbeatTelegramUserLease(opts); + } else if (command === "heartbeat-loop") { + await heartbeatTelegramUserLeaseLoop(opts); } else if (command === "release") { await releaseTelegramUserLease(opts); } else { diff --git a/scripts/mantis/run-with-lease-fence.sh b/scripts/mantis/run-with-lease-fence.sh new file mode 100755 index 000000000000..033873057c68 --- /dev/null +++ b/scripts/mantis/run-with-lease-fence.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +if (( $# < 3 )) || [[ "$2" != "--" ]]; then + echo "Usage: run-with-lease-fence.sh -- " >&2 + exit 64 +fi + +lost_marker_path="$1" +shift 2 + +# The marker records broker-confirmed loss of the shared-account lease; continuing +# Telegram I/O would violate broker serialization. The <=10s poll window is accepted +# against the 20-minute lease TTL. +setsid "$@" & +command_pid=$! + +command_exit() { + local status + set +e + wait "$command_pid" + status=$? + set -e + exit "$status" +} + +while true; do + if ! kill -0 "$command_pid" 2>/dev/null; then + command_exit + fi + + if [[ -f "$lost_marker_path" ]]; then + if ! kill -0 "$command_pid" 2>/dev/null; then + command_exit + fi + echo "::error::Telegram QA lease lost mid-run; fencing proof" >&2 + kill -TERM -- "-$command_pid" 2>/dev/null || true + deadline=$((SECONDS + 15)) + while kill -0 -- "-$command_pid" 2>/dev/null && (( SECONDS < deadline )); do + sleep 1 + done + if kill -0 -- "-$command_pid" 2>/dev/null; then + kill -KILL -- "-$command_pid" 2>/dev/null || true + fi + wait "$command_pid" 2>/dev/null || true + exit 97 + fi + + for _ in {1..10}; do + sleep 1 + if ! kill -0 "$command_pid" 2>/dev/null; then + command_exit + fi + done +done diff --git a/scripts/mantis/stop-lease-keepalive.sh b/scripts/mantis/stop-lease-keepalive.sh new file mode 100755 index 000000000000..a2431d3625fb --- /dev/null +++ b/scripts/mantis/stop-lease-keepalive.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +stop_lease_keepalive() { + local keepalive_pid_file="$1" + local lease_file="$2" + local expected_cwd="$3" + [[ -n "$keepalive_pid_file" && -f "$keepalive_pid_file" ]] || return 0 + local keepalive_pid + keepalive_pid="$(<"$keepalive_pid_file")" + [[ "$keepalive_pid" =~ ^[1-9][0-9]*$ ]] + if [[ -d "/proc/$keepalive_pid" ]]; then + local keepalive_uid keepalive_pgid keepalive_exe keepalive_cwd keepalive_args + keepalive_uid="$(stat -c %u "/proc/$keepalive_pid")" || return 0 + keepalive_pgid="$(ps -o pgid= -p "$keepalive_pid" | tr -d ' ')" || return 0 + keepalive_exe="$(readlink -f "/proc/$keepalive_pid/exe")" || return 0 + keepalive_cwd="$(readlink -f "/proc/$keepalive_pid/cwd")" || return 0 + keepalive_args="$(tr '\0' '\n' <"/proc/$keepalive_pid/cmdline")" || return 0 + [[ "$keepalive_uid" == "$(id -u)" ]] + [[ "$keepalive_pgid" == "$keepalive_pid" ]] + [[ "$keepalive_exe" == /usr/local/lib/mantis-toolchain/node ]] + [[ "$keepalive_cwd" == "$expected_cwd" ]] + grep -Fxq "scripts/e2e/telegram-user-credential.ts" <<<"$keepalive_args" + grep -Fxq "heartbeat-loop" <<<"$keepalive_args" + grep -Fxq "$lease_file" <<<"$keepalive_args" + kill -TERM "$keepalive_pid" 2>/dev/null || true + local deadline=$((SECONDS + 10)) + while kill -0 "$keepalive_pid" 2>/dev/null && ((SECONDS < deadline)); do + sleep 1 + done + if kill -0 "$keepalive_pid" 2>/dev/null; then + kill -KILL "$keepalive_pid" 2>/dev/null || true + fi + deadline=$((SECONDS + 5)) + while kill -0 "$keepalive_pid" 2>/dev/null && ((SECONDS < deadline)); do + sleep 1 + done + ! kill -0 "$keepalive_pid" 2>/dev/null + fi + rm -f "$keepalive_pid_file" +} + +stop_lease_keepalive "$@" diff --git a/test/scripts/mantis-telegram-desktop-proof-workflow.test.ts b/test/scripts/mantis-telegram-desktop-proof-workflow.test.ts index 1ab4d7d2f743..730a73dadd98 100644 --- a/test/scripts/mantis-telegram-desktop-proof-workflow.test.ts +++ b/test/scripts/mantis-telegram-desktop-proof-workflow.test.ts @@ -8,6 +8,8 @@ const MANTIS_SUT_SCRIPT = "scripts/e2e/telegram-mantis-sut.ts"; const MANTIS_LANE_SCRIPT = "scripts/e2e/telegram-mantis-lane.ts"; const DESKTOP_CRABBOX_SCRIPT = "scripts/e2e/telegram-desktop-crabbox.ts"; const SUT_CONTAINER_WRAPPER = "scripts/mantis/mantis-sut-container.sh"; +const STOP_LEASE_KEEPALIVE_SCRIPT = "scripts/mantis/stop-lease-keepalive.sh"; +const RUN_WITH_LEASE_FENCE_SCRIPT = "scripts/mantis/run-with-lease-fence.sh"; const CREDENTIAL_SCRIPT = "scripts/e2e/telegram-user-credential.ts"; const USER_DRIVER = "scripts/e2e/telegram-user-driver.py"; const QA_LAB_RUNTIME_API = "extensions/qa-lab/runtime-api.ts"; @@ -154,6 +156,77 @@ describe("Mantis Telegram Desktop proof workflow", () => { ); }); + it("keeps the shared Telegram lease alive through proof cleanup", () => { + const acquire = workflowStep("Install TDLib and restore Telegram QA user").run ?? ""; + const abandoned = workflowStep("Clean up abandoned Mantis sessions").run ?? ""; + const release = workflowStep("Release Telegram QA user lease").run ?? ""; + const credentialScript = readFileSync(CREDENTIAL_SCRIPT, "utf8"); + const stopKeepaliveScript = readFileSync(STOP_LEASE_KEEPALIVE_SCRIPT, "utf8"); + + expect(credentialScript).toContain('command === "heartbeat"'); + expect(credentialScript).toContain('command === "heartbeat-loop"'); + expect(credentialScript).toContain('action: "heartbeat"'); + expect(credentialScript).toContain('error.code === "LEASE_NOT_OWNER"'); + expect(credentialScript).toContain('error.code === "LEASE_EXPIRED"'); + expect(acquire.indexOf("telegram-user-credential.ts lease-restore")).toBeLessThan( + acquire.indexOf("telegram-user-credential.ts heartbeat-loop"), + ); + expect(acquire).toContain("/usr/bin/setsid /usr/local/lib/mantis-toolchain/node --import tsx"); + expect(acquire).toContain('--interval-ms 30000 "$keepalive_log" 2>&1 &'); + expect(acquire).toContain('echo "lease_keepalive_pid_file=$keepalive_pid_file"'); + expect(acquire).toContain('echo "lease_lost_marker=$lease_lost_marker"'); + + for (const cleanup of [abandoned, release]) { + expect(cleanup).toContain(STOP_LEASE_KEEPALIVE_SCRIPT); + expect(cleanup).toContain("steps.telegram_credential.outputs.lease_keepalive_pid_file"); + expect(cleanup).toContain("steps.telegram_credential.outputs.lease_file"); + expect(cleanup).toContain('"$GITHUB_WORKSPACE"'); + expect(cleanup).not.toContain("stop_lease_keepalive() {"); + } + expect(stopKeepaliveScript).toContain('[[ "$keepalive_pid" =~ ^[1-9][0-9]*$ ]]'); + expect(stopKeepaliveScript).toContain('[[ "$keepalive_uid" == "$(id -u)" ]]'); + expect(stopKeepaliveScript).toContain('[[ "$keepalive_pgid" == "$keepalive_pid" ]]'); + expect(stopKeepaliveScript).toContain( + '[[ "$keepalive_exe" == /usr/local/lib/mantis-toolchain/node ]]', + ); + expect(stopKeepaliveScript).toContain('[[ "$keepalive_cwd" == "$expected_cwd" ]]'); + expect(stopKeepaliveScript).toContain( + 'grep -Fxq "scripts/e2e/telegram-user-credential.ts" <<<"$keepalive_args"', + ); + expect(stopKeepaliveScript).toContain('grep -Fxq "heartbeat-loop" <<<"$keepalive_args"'); + expect(stopKeepaliveScript).toContain('grep -Fxq "$lease_file" <<<"$keepalive_args"'); + expect(stopKeepaliveScript).toContain('kill -TERM "$keepalive_pid"'); + expect(stopKeepaliveScript).toContain('kill -KILL "$keepalive_pid"'); + expect(release.indexOf(STOP_LEASE_KEEPALIVE_SCRIPT)).toBeLessThan( + release.indexOf('if [[ -z "$lease_file" ]]'), + ); + expect(release.indexOf('if [[ -z "$lease_file" ]]')).toBeLessThan( + release.indexOf('sudo test -f "$lease_lost_marker"'), + ); + expect(release.indexOf('sudo test -f "$lease_lost_marker"')).toBeLessThan( + release.indexOf('telegram-user-credential.ts" release'), + ); + expect(release).toContain("lease lost mid-run; nothing to release"); + expect(release).toContain("steps.telegram_credential.outputs.lease_lost_marker"); + expect(release).not.toMatch(/telegram-user-credential\.ts[^\n]*release[^\n]*\|\| true/u); + }); + + it("fences the active agent proof when the Telegram lease is lost", () => { + const agent = workflowStep("Run Codex Mantis Telegram agent"); + const setup = workflowStep("Prepare Codex action runtime"); + const fenceScript = readFileSync(RUN_WITH_LEASE_FENCE_SCRIPT, "utf8"); + const run = agent.run ?? ""; + + expect(setup.uses).toContain("openai/codex-action@"); + expect(run).toContain('scripts/mantis/run-with-lease-fence.sh "$lease_lost_marker" --'); + expect(run).toContain("steps.telegram_credential.outputs.lease_lost_marker"); + expect(run).toContain('sudo -u codex -- "$codex_bin" "${codex_args[@]}"'); + expect(fenceScript.indexOf('kill -TERM -- "-$command_pid"')).toBeLessThan( + fenceScript.indexOf('kill -KILL -- "-$command_pid"'), + ); + expect(fenceScript).toContain("exit 97"); + }); + it("reports an honest blocked proof without failing the workflow", () => { const trusted = workflowStep("Restore and validate trusted lane evidence").run ?? ""; const inspect = workflowStep("Inspect Mantis evidence manifest").run ?? ""; @@ -709,6 +782,36 @@ describe("Mantis Telegram Desktop proof workflow", () => { expect(prompt).not.toContain("OPENCLAW_TELEGRAM_USER_PROOF_CMD"); }); + it("stages agent-authored fixture plugins inside either isolated SUT lane", () => { + const agent = workflowStep("Run Codex Mantis Telegram agent"); + const prepare = workflowStep("Prepare Codex user").run ?? ""; + const laneScript = readFileSync(MANTIS_LANE_SCRIPT, "utf8"); + const sutScript = readFileSync(MANTIS_SUT_SCRIPT, "utf8"); + const wrapper = readFileSync(SUT_CONTAINER_WRAPPER, "utf8"); + const prompt = readFileSync(PROMPT, "utf8"); + + expect(agent.env).toHaveProperty("MANTIS_FIXTURE_PLUGINS_DIR"); + const fixturePluginsDir = agent.env?.MANTIS_FIXTURE_PLUGINS_DIR ?? ""; + expect(fixturePluginsDir).toContain("/tmp/openclaw-mantis-proof-sessions-"); + expect(fixturePluginsDir).toContain("/fixture-plugins"); + expect(prepare).toContain('fixture_plugins_root="$session_root/fixture-plugins"'); + expect(prepare).toContain("for lane in baseline candidate"); + expect(prepare).toContain('"$fixture_plugins_root/$lane"'); + expect(laneScript).toContain( + 'fixturePluginsDir: path.join(roots.sessionRoot, "fixture-plugins", lane)', + ); + expect(sutScript).toContain('path.join(tempRoot, "fixture-plugins")'); + expect(sutScript).toContain("fs.cpSync(params.fixturePluginsDir"); + expect(sutScript).toContain("load: { paths: [fixturePluginsRoot] }"); + expect(wrapper).toContain('/bin/cp -a --no-dereference "$quarantine/." "$safe_runtime/"'); + expect(wrapper).not.toContain("type=bind,src=$fixture_plugins"); + expect(wrapper).not.toContain("cp -al"); + expect(prompt).toContain("MANTIS_FIXTURE_PLUGINS_DIR"); + expect(prompt).toContain("before `start`"); + expect(prompt).toMatch(/same fixture\s+package in both lane directories/u); + expect(prompt).toContain("configPatch.plugins.allow"); + }); + it("incrementally refreshes stale baseline builds while preparing both proof lanes in parallel", () => { const workflow = parse(readFileSync(WORKFLOW, "utf8")) as Workflow; const steps = workflow.jobs?.run_telegram_desktop_proof?.steps ?? []; @@ -844,10 +947,12 @@ describe("Mantis Telegram Desktop proof workflow", () => { it("runs the Mantis Codex agent in fast high-effort mode", () => { const agent = workflowStep("Run Codex Mantis Telegram agent"); + const setup = workflowStep("Prepare Codex action runtime"); + const run = agent.run ?? ""; - expect(agent.uses).toContain("openai/codex-action@"); - expect(agent.with?.effort).toBe("high"); - expect(agent.with?.["codex-args"]).toBe('["-c","service_tier=\\"fast\\""]'); + expect(setup.uses).toContain("openai/codex-action@"); + expect(run).toContain("--config 'model_reasoning_effort=\"high\"'"); + expect(run).toContain("-c 'service_tier=\"fast\"'"); }); it("derives refs from the PR instead of parsing comment prose", () => { diff --git a/test/scripts/run-with-lease-fence.test.ts b/test/scripts/run-with-lease-fence.test.ts new file mode 100644 index 000000000000..a144fd780326 --- /dev/null +++ b/test/scripts/run-with-lease-fence.test.ts @@ -0,0 +1,75 @@ +import { spawn, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { waitForChildClose, waitForFile, waitForPidFile } from "../helpers/process-wait.js"; +import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js"; + +const SCRIPT = "scripts/mantis/run-with-lease-fence.sh"; +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +const posixIt = process.platform === "win32" ? it.skip : it; + +describe("run-with-lease-fence", () => { + posixIt( + "stops the active process group after terminal lease loss", + async () => { + const root = tempDirs.make("openclaw-lease-fence-"); + const lostMarker = path.join(root, "lease.lost"); + const commandPidFile = path.join(root, "command.pid"); + const ticksFile = path.join(root, "ticks.log"); + let stderr = ""; + const child = spawn( + SCRIPT, + [ + lostMarker, + "--", + "/bin/bash", + "-c", + 'trap "exit 0" TERM; printf "%s\\n" "$$" >"$1"; while :; do printf "tick\\n" >>"$2"; sleep 1; done', + "lease-fence-command", + commandPidFile, + ticksFile, + ], + { stdio: ["ignore", "ignore", "pipe"] }, + ); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + + const commandPid = await waitForPidFile(commandPidFile, 2_000); + await waitForFile(ticksFile, 2_000); + fs.writeFileSync(lostMarker, "lost\n"); + + try { + await expect(waitForChildClose(child, 12_000)).resolves.toEqual({ + code: 97, + signal: null, + }); + const ticksAfterFence = fs.readFileSync(ticksFile, "utf8"); + await new Promise((resolve) => setTimeout(resolve, 1_200)); + expect(fs.readFileSync(ticksFile, "utf8")).toBe(ticksAfterFence); + expect(stderr).toContain("::error::Telegram QA lease lost mid-run; fencing proof"); + } finally { + if (child.exitCode === null) { + child.kill("SIGKILL"); + } + try { + process.kill(-commandPid, "SIGKILL"); + } catch {} + } + }, + 20_000, + ); + + posixIt("propagates a clean command exit", () => { + const root = tempDirs.make("openclaw-lease-fence-clean-"); + const result = spawnSync(SCRIPT, [path.join(root, "lease.lost"), "--", "/bin/true"], { + encoding: "utf8", + }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).toBe(""); + }); +}); diff --git a/test/scripts/telegram-user-credential.test.ts b/test/scripts/telegram-user-credential.test.ts index 9f65dc48cb8d..5317bda23900 100644 --- a/test/scripts/telegram-user-credential.test.ts +++ b/test/scripts/telegram-user-credential.test.ts @@ -2,6 +2,8 @@ import { spawn } from "node:child_process"; import { existsSync, readFileSync, writeFileSync } from "node:fs"; import { readFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; import path, { win32 } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { fetchJsonWithTimeout, runCommand } from "../../scripts/e2e/telegram-user-credential-io.ts"; @@ -61,6 +63,19 @@ async function waitForDead(pid: number, timeoutMs: number): Promise { throw new Error(`process still alive: ${pid}`); } +async function waitForText(readText: () => string, expected: string, timeoutMs: number) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (readText().includes(expected)) { + return; + } + await new Promise((resolve) => { + setTimeout(resolve, 5); + }); + } + throw new Error(`timeout waiting for output: ${expected}`); +} + async function waitForExit( child: ReturnType, timeoutMs: number, @@ -79,6 +94,36 @@ async function waitForExit( }); } +function spawnCredentialCli(args: string[]) { + const child = spawn( + process.execPath, + ["--import", "tsx", "scripts/e2e/telegram-user-credential.ts", ...args], + { + env: { + ...process.env, + OPENCLAW_QA_CONVEX_SECRET_CI: undefined, + OPENCLAW_QA_CONVEX_SITE_URL: undefined, + }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + let stdout = ""; + let stderr = ""; + child.stdout?.setEncoding("utf8").on("data", (chunk: string) => { + stdout += chunk; + }); + child.stderr?.setEncoding("utf8").on("data", (chunk: string) => { + stderr += chunk; + }); + return { child, output: () => ({ stderr, stdout }) }; +} + +async function runCredentialCli(args: string[]) { + const { child, output } = spawnCredentialCli(args); + const exit = await waitForExit(child, PROCESS_WAIT_TIMEOUT_MS); + return { ...exit, ...output() }; +} + afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); @@ -288,6 +333,174 @@ describe("telegram user credential IO", () => { ).toThrow("Usage:"); }); + it("heartbeats leases and marks ownership loss without logging secrets", async () => { + const dir = tempDirs.make("openclaw-telegram-credential-heartbeat-"); + const leaseFile = path.join(dir, "lease.json"); + const envFile = path.join(dir, "broker.env"); + const leaseToken = "test-lease-token"; + const brokerSecret = "test-broker-secret"; + const requests: Array<{ body: Record; path: string | undefined }> = []; + let brokerResponse: Record = { status: "ok" }; + const server = createServer(async (request, response) => { + const chunks: Buffer[] = []; + for await (const chunk of request) { + chunks.push(Buffer.from(chunk)); + } + requests.push({ + body: JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record, + path: request.url, + }); + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify(brokerResponse)); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const siteUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + writeFileSync( + leaseFile, + JSON.stringify({ + actorRole: "ci", + credentialId: "credential-1", + kind: "telegram-user", + leaseToken, + ownerId: "owner-1", + siteUrl, + }), + ); + writeFileSync( + envFile, + `OPENCLAW_QA_CONVEX_SITE_URL=${siteUrl}\nOPENCLAW_QA_CONVEX_SECRET_CI=${brokerSecret}\n`, + ); + + try { + const heartbeat = await runCredentialCli([ + "heartbeat", + "--lease-file", + leaseFile, + "--env-file", + envFile, + "--credential-role", + "ci", + ]); + expect(heartbeat).toMatchObject({ code: 0, signal: null }); + expect(requests).toEqual([ + { + body: { + actorRole: "ci", + credentialId: "credential-1", + kind: "telegram-user", + leaseToken, + leaseTtlMs: 1_200_000, + ownerId: "owner-1", + }, + path: "/qa-credentials/v1/heartbeat", + }, + ]); + expect(`${heartbeat.stdout}${heartbeat.stderr}`).not.toContain(leaseToken); + expect(`${heartbeat.stdout}${heartbeat.stderr}`).not.toContain(brokerSecret); + + brokerResponse = { status: "error", code: "LEASE_NOT_OWNER" }; + const lost = await runCredentialCli([ + "heartbeat-loop", + "--lease-file", + leaseFile, + "--env-file", + envFile, + "--credential-role", + "ci", + "--interval-ms", + "1", + ]); + expect(lost).toMatchObject({ code: 0, signal: null }); + expect(JSON.parse(readFileSync(`${leaseFile}.lost`, "utf8"))).toEqual({ + code: "LEASE_NOT_OWNER", + }); + expect(`${lost.stdout}${lost.stderr}`).not.toContain(leaseToken); + expect(`${lost.stdout}${lost.stderr}`).not.toContain(brokerSecret); + } finally { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + } + }); + + it("retries non-terminal heartbeat failures without logging secrets", async () => { + const dir = tempDirs.make("openclaw-telegram-credential-heartbeat-retry-"); + const leaseFile = path.join(dir, "lease.json"); + const envFile = path.join(dir, "broker.env"); + const leaseToken = "retry-lease-token"; + const brokerSecret = "retry-broker-secret"; + let heartbeatCount = 0; + let resolveSecondHeartbeat!: () => void; + const secondHeartbeat = new Promise((resolve) => { + resolveSecondHeartbeat = resolve; + }); + const server = createServer(async (request, response) => { + for await (const _chunk of request) { + // Drain the request before responding so the real CLI can reuse the connection. + } + heartbeatCount += 1; + if (heartbeatCount === 1) { + response.writeHead(500, { "content-type": "application/json" }); + response.end(JSON.stringify({ status: "error", code: "BROKER_UNAVAILABLE" })); + return; + } + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ status: "ok" })); + resolveSecondHeartbeat(); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const siteUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + writeFileSync( + leaseFile, + JSON.stringify({ + actorRole: "ci", + credentialId: "credential-retry", + kind: "telegram-user", + leaseToken, + ownerId: "owner-retry", + siteUrl, + }), + ); + writeFileSync( + envFile, + `OPENCLAW_QA_CONVEX_SITE_URL=${siteUrl}\nOPENCLAW_QA_CONVEX_SECRET_CI=${brokerSecret}\n`, + ); + + const { child, output } = spawnCredentialCli([ + "heartbeat-loop", + "--lease-file", + leaseFile, + "--env-file", + envFile, + "--credential-role", + "ci", + "--interval-ms", + "1", + ]); + try { + const outcome = await Promise.race([ + secondHeartbeat.then(() => "heartbeat" as const), + waitForExit(child, PROCESS_WAIT_TIMEOUT_MS).then(() => "exit" as const), + ]); + expect(outcome).toBe("heartbeat"); + expect(heartbeatCount).toBeGreaterThanOrEqual(2); + expect(existsSync(`${leaseFile}.lost`)).toBe(false); + const retryMessage = "Credential lease heartbeat failed (BROKER_UNAVAILABLE); retrying."; + await waitForText(() => output().stderr, retryMessage, PROCESS_WAIT_TIMEOUT_MS); + expect(output().stderr).toContain(retryMessage); + expect(`${output().stdout}${output().stderr}`).not.toContain(leaseToken); + expect(`${output().stdout}${output().stderr}`).not.toContain(brokerSecret); + } finally { + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGTERM"); + } + await waitForExit(child, PROCESS_WAIT_TIMEOUT_MS); + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + } + }); + it("fails hung child processes instead of waiting for the outer proof timeout", async () => { await expect( runCommand(process.execPath, ["-e", "setInterval(() => {}, 1000)"], undefined, {