From 45bad50e26902052ea2eb340b027f3518d3db7b4 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 27 Jul 2026 04:33:06 -0400 Subject: [PATCH] fix(qa): harden Mantis Telegram proof harness (#114429) * fix(qa): harden Mantis Telegram proof harness * fix(qa): require explicit Mantis dispatch --- .../telegram-crabbox-e2e-proof/SKILL.md | 17 ++ .../prompts/mantis-telegram-desktop-proof.md | 20 +- .../mantis-telegram-desktop-proof.yml | 260 +++++++----------- scripts/e2e/mock-openai-server.mjs | 51 +++- scripts/e2e/telegram-user-crabbox-proof.ts | 77 +++++- test/scripts/e2e-mock-config-limits.test.ts | 24 ++ ...is-telegram-desktop-proof-workflow.test.ts | 68 +++-- .../telegram-user-crabbox-proof.test.ts | 57 ++++ 8 files changed, 374 insertions(+), 200 deletions(-) diff --git a/.agents/skills/telegram-crabbox-e2e-proof/SKILL.md b/.agents/skills/telegram-crabbox-e2e-proof/SKILL.md index b7e6ef188cd0..cfdbf7f9cec9 100644 --- a/.agents/skills/telegram-crabbox-e2e-proof/SKILL.md +++ b/.agents/skills/telegram-crabbox-e2e-proof/SKILL.md @@ -47,6 +47,23 @@ proof_cmd="${OPENCLAW_TELEGRAM_USER_PROOF_CMD:-openclaw-telegram-user-crabbox-pr --output-dir .artifacts/qa-e2e/telegram-user-crabbox/pr-review ``` +When the scenario needs Telegram link previews explicitly enabled or disabled, +set the SUT config before Gateway startup: + +```bash +"$proof_cmd" start \ + --tdlib-url http://artifacts.openclaw.ai/tdlib-v1.8.0-linux-x64.tgz \ + --link-preview false \ + --mock-response-chunk-delay-ms 1200 \ + --mock-response-file .artifacts/qa-e2e/telegram-user-crabbox/reply.txt \ + --output-dir .artifacts/qa-e2e/telegram-user-crabbox/pr-review +``` + +Do not edit the generated config or restart the held-session Gateway to apply +this setting; that bypasses the runner's lifecycle and cleanup ownership. +The chunk delay splits the configured response into two streaming deltas so a +recording can show the initial preview and a later in-place edit. + The runner defaults to `--class standard`, `--record-fps 24`, `--preview-fps 24`, and `--preview-width 1920`. Keep those defaults unless the proof needs something else. diff --git a/.github/codex/prompts/mantis-telegram-desktop-proof.md b/.github/codex/prompts/mantis-telegram-desktop-proof.md index 704ab2b71dc7..2109cf9a0e4a 100644 --- a/.github/codex/prompts/mantis-telegram-desktop-proof.md +++ b/.github/codex/prompts/mantis-telegram-desktop-proof.md @@ -111,14 +111,16 @@ than Telegram-visible behavior`. Use this manifest shape and do not create Reopen that same Telegram button after its ticket expires and capture the expired state. Do not substitute Control UI, transcript, curl, or a newly minted button for any part of that path. -5. Create detached worktrees under +5. Use the workflow-prepared detached worktrees under `.artifacts/qa-e2e/mantis/telegram-desktop-proof-worktrees/baseline` and - `.artifacts/qa-e2e/mantis/telegram-desktop-proof-worktrees/candidate`, then - install and build each worktree with the repo's normal `pnpm` commands. + `.artifacts/qa-e2e/mantis/telegram-desktop-proof-worktrees/candidate`. + Verify their `HEAD`s match `BASELINE_SHA` and `CANDIDATE_SHA`; do not create, + install, rebuild, or replace them. The workflow prepared both with a pinned + Node/pnpm toolchain before agent secrets were available. If `MANTIS_CANDIDATE_TRUST` is `fork-pr-head`, treat the candidate worktree as untrusted fork code: do not pass GitHub, OpenAI, - Crabbox, Convex, or other workflow secrets into candidate install, build, or - runtime commands. The candidate SUT may receive only the proof runner's + Crabbox, Convex, or other workflow secrets into candidate runtime commands. + The candidate SUT may receive only the proof runner's short-lived Telegram bot token, generated local config/state paths, and mock model key needed for this isolated proof. 6. In each worktree, run the real-user Telegram Crabbox proof flow from the @@ -133,6 +135,14 @@ than Telegram-visible behavior`. Use this manifest shape and do not create own; do not kill it while Crabbox is still waiting for bootstrap. Use a long command timeout for `start`, `send`, `view`, and `finish`. You may iterate and rerun if the visual result is not convincing. + When the requested scenario needs `channels.telegram.linkPreview: false`, + pass `--link-preview false` to `start`. The runner injects that setting into + the isolated SUT config before Gateway startup. Do not edit the generated + config or restart the Gateway to apply it. + When the proof must show an in-place streamed edit, also pass + `--mock-response-chunk-delay-ms 1200` and use a mock response long enough + for the first chunk to clear the preview debounce. Capture both the initial + partial reply and the later edit before finishing. 7. Open Telegram Desktop directly to the newest relevant message with the runner `view` command before finishing each recording. Keep the chat scrolled to the bottom so new proof messages appear in-frame. diff --git a/.github/workflows/mantis-telegram-desktop-proof.yml b/.github/workflows/mantis-telegram-desktop-proof.yml index b889fa884b7b..459713bceec2 100644 --- a/.github/workflows/mantis-telegram-desktop-proof.yml +++ b/.github/workflows/mantis-telegram-desktop-proof.yml @@ -1,10 +1,6 @@ name: Mantis Telegram Desktop Proof on: - issue_comment: - types: [created] - pull_request_target: # zizmor: ignore[dangerous-triggers] maintainer-owned Mantis label trigger; trusted base workflow validates refs before checkout/use - types: [labeled] workflow_dispatch: inputs: pr_number: @@ -53,63 +49,8 @@ env: MANTIS_OUTPUT_DIR: .artifacts/qa-e2e/mantis/telegram-desktop-proof jobs: - authorize_actor: - name: Authorize workflow actor - if: >- - ${{ - github.event_name == 'workflow_dispatch' || - ( - github.event_name == 'pull_request_target' && - github.event.action == 'labeled' && - github.event.label.name == 'mantis: telegram-visible-proof' - ) || - ( - github.event_name == 'issue_comment' && - github.event.issue.pull_request && - contains(github.event.issue.labels.*.name, 'mantis: telegram-visible-proof') && - ( - contains(github.event.comment.body, '@openclaw-mantis') || - contains(github.event.comment.body, '/openclaw-mantis') - ) - ) - }} - runs-on: ubuntu-24.04 - outputs: - authorized: ${{ steps.permission.outputs.authorized }} - steps: - - name: Require maintainer-level repository access - id: permission - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 - with: - script: | - if (context.eventName === "pull_request_target") { - core.info(`Accepted Mantis label trigger from ${context.actor}.`); - core.setOutput("authorized", "true"); - return; - } - - const allowed = new Set(["admin", "maintain", "write"]); - const { owner, repo } = context.repo; - const { data } = await github.rest.repos.getCollaboratorPermissionLevel({ - owner, - repo, - username: context.actor, - }); - const permission = data.permission; - core.info(`Actor ${context.actor} permission: ${permission}`); - if (!allowed.has(permission)) { - core.notice( - `Workflow requires write/maintain/admin access. Actor "${context.actor}" has "${permission}".`, - ); - core.setOutput("authorized", "false"); - return; - } - core.setOutput("authorized", "true"); - resolve_request: name: Resolve Mantis request - needs: authorize_actor - if: needs.authorize_actor.outputs.authorized == 'true' runs-on: ubuntu-24.04 outputs: baseline_ref: ${{ steps.resolve.outputs.baseline_ref }} @@ -128,57 +69,19 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 with: script: | - const eventName = context.eventName; - function setOutput(name, value) { core.setOutput(name, value ?? ""); core.info(`${name}=${value ?? ""}`); } const inputs = context.payload.inputs ?? {}; - const prNumber = - eventName === "workflow_dispatch" - ? inputs.pr_number - : eventName === "pull_request_target" - ? String(context.payload.pull_request?.number ?? "") - : String(context.payload.issue?.number ?? ""); + const prNumber = inputs.pr_number; if (!prNumber) { core.setFailed("Mantis Telegram desktop proof requires a pull request."); return; } - const body = - eventName === "workflow_dispatch" - ? inputs.instructions || "" - : eventName === "issue_comment" - ? context.payload.comment?.body || "" - : ""; - if (eventName === "issue_comment") { - const normalized = body.toLowerCase(); - const requestedDesktopProof = - (normalized.includes("@openclaw-mantis") || normalized.includes("/openclaw-mantis")) && - (normalized.includes("desktop proof") || - normalized.includes("desktop-proof") || - normalized.includes("telegram desktop") || - normalized.includes("native telegram") || - normalized.includes("visible proof") || - normalized.includes("visible-proof") || - normalized.includes("telegram-visible-proof")); - if (!requestedDesktopProof) { - core.notice("Comment mentioned Mantis but did not request Telegram desktop proof."); - setOutput("should_run", "false"); - setOutput("baseline_ref", ""); - setOutput("candidate_ref", ""); - setOutput("pr_number", ""); - setOutput("instructions", ""); - setOutput("crabbox_provider", ""); - setOutput("lease_id", ""); - setOutput("publish_artifact_name", ""); - setOutput("publish_run_id", ""); - setOutput("request_source", "unsupported_issue_comment"); - return; - } - } + const body = inputs.instructions || ""; const { owner, repo } = context.repo; const { data: pr } = await github.rest.pulls.get({ @@ -201,16 +104,7 @@ jobs: setOutput("lease_id", inputs.crabbox_lease_id || ""); setOutput("publish_artifact_name", inputs.publish_artifact_name || ""); setOutput("publish_run_id", inputs.publish_run_id || ""); - setOutput("request_source", eventName); - - if (eventName === "issue_comment") { - await github.rest.reactions.createForIssueComment({ - owner, - repo, - comment_id: context.payload.comment.id, - content: "eyes", - }).catch((error) => core.warning(`Could not add eyes reaction: ${error.message}`)); - } + setOutput("request_source", "workflow_dispatch"); validate_refs: name: Validate selected refs @@ -387,13 +281,32 @@ jobs: run: | set -euo pipefail test -f scripts/e2e/telegram-user-driver.py - cat >"${RUNNER_TEMP}/openclaw-telegram-user-crabbox-proof" <<'EOF' + node_bin="$(command -v node)" + corepack_bin="$(command -v corepack)" + "$node_bin" "$corepack_bin" pnpm --version >/dev/null + cat >"${RUNNER_TEMP}/mantis-node" <"${RUNNER_TEMP}/mantis-pnpm" <"${RUNNER_TEMP}/openclaw-telegram-user-crabbox-proof" </dev/null media_tools="${RUNNER_TEMP}/mantis-media-tools" install -d "$media_tools" @@ -408,6 +321,76 @@ jobs: ffmpeg -version >/dev/null ffprobe -version >/dev/null + - name: Prepare proof worktrees with pinned toolchain + env: + BASELINE_SHA: ${{ needs.validate_refs.outputs.baseline_revision }} + CANDIDATE_SHA: ${{ needs.validate_refs.outputs.candidate_revision }} + MANTIS_PR_NUMBER: ${{ needs.resolve_request.outputs.pr_number }} + shell: bash + run: | + set -euo pipefail + worktree_root="$GITHUB_WORKSPACE/.artifacts/qa-e2e/mantis/telegram-desktop-proof-worktrees" + baseline_root="$worktree_root/baseline" + candidate_root="$worktree_root/candidate" + toolchain_dir=/usr/local/lib/mantis-toolchain + corepack_home="${RUNNER_TEMP}/mantis-corepack" + mkdir -p "$worktree_root" "$corepack_home" + git cat-file -e "${BASELINE_SHA}^{commit}" + if ! git cat-file -e "${CANDIDATE_SHA}^{commit}"; then + git fetch --no-tags origin "pull/${MANTIS_PR_NUMBER}/head" + fi + git cat-file -e "${CANDIDATE_SHA}^{commit}" + git worktree add --detach "$baseline_root" "$BASELINE_SHA" + git worktree add --detach "$candidate_root" "$CANDIDATE_SHA" + candidate_git_link="$(cat "$candidate_root/.git")" + + prepare_worktree() { + local repo_root="$1" + local safe_home="$2" + mkdir -p "$safe_home" + ( + cd "$repo_root" + env -i \ + CI=1 \ + COREPACK_HOME="$corepack_home" \ + HOME="$safe_home" \ + OPENCLAW_BUILD_PRIVATE_QA=1 \ + OPENCLAW_ENABLE_PRIVATE_QA_CLI=1 \ + PATH="$toolchain_dir:/usr/bin:/bin" \ + "$toolchain_dir/pnpm" install --frozen-lockfile + env -i \ + CI=1 \ + COREPACK_HOME="$corepack_home" \ + HOME="$safe_home" \ + OPENCLAW_BUILD_PRIVATE_QA=1 \ + OPENCLAW_ENABLE_PRIVATE_QA_CLI=1 \ + PATH="$toolchain_dir:/usr/bin:/bin" \ + "$toolchain_dir/pnpm" build + ) + } + + prepare_worktree "$baseline_root" "${RUNNER_TEMP}/mantis-baseline-home" + sudo useradd --system --no-create-home --shell /usr/sbin/nologin mantis-builder + candidate_home="${RUNNER_TEMP}/mantis-candidate-home" + candidate_corepack_home="${RUNNER_TEMP}/mantis-candidate-corepack" + sudo install -d -m 0700 -o mantis-builder -g mantis-builder "$candidate_home" + sudo install -d -m 0700 -o mantis-builder -g mantis-builder "$candidate_corepack_home" + sudo chown -R mantis-builder:mantis-builder "$candidate_root" + sudo -u mantis-builder env -i \ + CI=1 \ + COREPACK_HOME="$candidate_corepack_home" \ + HOME="$candidate_home" \ + OPENCLAW_BUILD_PRIVATE_QA=1 \ + OPENCLAW_ENABLE_PRIVATE_QA_CLI=1 \ + PATH="$toolchain_dir:/usr/bin:/bin" \ + /bin/bash -c 'cd "$1" && "$2" install --frozen-lockfile && "$2" build' \ + bash "$candidate_root" "$toolchain_dir/pnpm" + test "$(cat "$candidate_root/.git")" = "$candidate_git_link" + git -C "$candidate_root" diff --exit-code + git -C "$candidate_root" diff --cached --exit-code + test "$(git -C "$baseline_root" rev-parse HEAD)" = "$BASELINE_SHA" + test "$(git -C "$candidate_root" rev-parse HEAD)" = "$CANDIDATE_SHA" + - name: Ensure agent key exists env: OPENAI_API_KEY: ${{ secrets.OPENCLAW_MANTIS_AGENT_OPENAI_API_KEY || secrets.OPENAI_API_KEY }} @@ -428,6 +411,7 @@ jobs: printf '%s\n' 'Defaults env_keep += "BASELINE_REF BASELINE_SHA CANDIDATE_REF CANDIDATE_SHA"' printf '%s\n' 'Defaults env_keep += "CRABBOX_ACCESS_CLIENT_ID CRABBOX_ACCESS_CLIENT_SECRET CRABBOX_COORDINATOR CRABBOX_COORDINATOR_TOKEN CRABBOX_AWS_REGION CRABBOX_CAPACITY_REGIONS CRABBOX_LEASE_ID CRABBOX_PROVIDER"' printf '%s\n' 'Defaults env_keep += "GH_TOKEN MANTIS_CANDIDATE_TRUST MANTIS_INSTRUCTIONS MANTIS_OUTPUT_DIR MANTIS_PR_NUMBER"' + printf '%s\n' 'Defaults env_keep += "MANTIS_NODE_BIN MANTIS_PNPM_BIN"' printf '%s\n' 'Defaults env_keep += "OPENCLAW_BUILD_PRIVATE_QA OPENCLAW_ENABLE_PRIVATE_QA_CLI OPENCLAW_QA_CONVEX_SECRET_CI OPENCLAW_QA_CONVEX_SITE_URL OPENCLAW_QA_CREDENTIAL_OWNER_ID OPENCLAW_QA_MANTIS_CRABBOX_COORDINATOR OPENCLAW_QA_MANTIS_CRABBOX_COORDINATOR_TOKEN"' printf '%s\n' 'Defaults env_keep += "OPENCLAW_TELEGRAM_USER_CRABBOX_BIN OPENCLAW_TELEGRAM_USER_CRABBOX_PROVIDER OPENCLAW_TELEGRAM_USER_DRIVER_SCRIPT OPENCLAW_TELEGRAM_USER_PROOF_CMD"' } | sudo tee /etc/sudoers.d/mantis-codex-env >/dev/null @@ -462,7 +446,9 @@ jobs: GH_TOKEN: ${{ github.token }} MANTIS_CANDIDATE_TRUST: ${{ needs.validate_refs.outputs.candidate_trust }} MANTIS_INSTRUCTIONS: ${{ needs.resolve_request.outputs.instructions }} + MANTIS_NODE_BIN: /usr/local/lib/mantis-toolchain/node MANTIS_OUTPUT_DIR: ${{ env.MANTIS_OUTPUT_DIR }} + MANTIS_PNPM_BIN: /usr/local/lib/mantis-toolchain/pnpm MANTIS_PR_NUMBER: ${{ needs.resolve_request.outputs.pr_number }} OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }} @@ -483,12 +469,13 @@ jobs: codex-home: /tmp/mantis-codex-home-${{ github.run_id }} safety-strategy: unprivileged-user codex-user: codex - allow-bot-users: clawsweeper[bot] + allow-bot-users: github-actions[bot] - name: Release leaked Telegram proof leases if: ${{ always() }} env: CRABBOX_PROVIDER: ${{ needs.resolve_request.outputs.crabbox_provider }} + MANTIS_NODE_BIN: /usr/local/lib/mantis-toolchain/node OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }} shell: bash @@ -500,7 +487,7 @@ jobs: status=0 mapfile -d '' session_files < <(sudo find .artifacts/qa-e2e -name session.json -type f -print0) for session_file in "${session_files[@]}"; do - if ! sudo -u codex node -e 'const fs = require("fs"); const session = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.exit(session.command === "telegram-user-crabbox-session" ? 0 : 1);' "$session_file"; then + if ! sudo -u codex "$MANTIS_NODE_BIN" -e 'const fs = require("fs"); const session = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.exit(session.command === "telegram-user-crabbox-session" ? 0 : 1);' "$session_file"; then continue fi lease_file="${session_file%/session.json}/.session/lease.json" @@ -512,20 +499,20 @@ jobs: OPENCLAW_QA_CONVEX_SITE_URL="$OPENCLAW_QA_CONVEX_SITE_URL" \ OPENCLAW_TELEGRAM_USER_CRABBOX_BIN=/usr/local/bin/crabbox \ OPENCLAW_TELEGRAM_USER_CRABBOX_PROVIDER="$CRABBOX_PROVIDER" \ - node --import tsx "$GITHUB_WORKSPACE/scripts/e2e/telegram-user-crabbox-proof.ts" \ + "$MANTIS_NODE_BIN" --import tsx "$GITHUB_WORKSPACE/scripts/e2e/telegram-user-crabbox-proof.ts" \ finish --session "$session_file" --preview-crop telegram-window; then status=1 fi done mapfile -d '' lease_files < <(sudo find .artifacts/qa-e2e -path '*/.session/lease.json' -type f -print0) for lease_file in "${lease_files[@]}"; do - if ! sudo -u codex node -e 'const fs = require("fs"); const lease = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.exit(lease.kind === "telegram-user" ? 0 : 1);' "$lease_file"; then + if ! sudo -u codex "$MANTIS_NODE_BIN" -e 'const fs = require("fs"); const lease = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.exit(lease.kind === "telegram-user" ? 0 : 1);' "$lease_file"; then continue fi if ! sudo -u codex env \ OPENCLAW_QA_CONVEX_SECRET_CI="$OPENCLAW_QA_CONVEX_SECRET_CI" \ OPENCLAW_QA_CONVEX_SITE_URL="$OPENCLAW_QA_CONVEX_SITE_URL" \ - node --import tsx "$GITHUB_WORKSPACE/scripts/e2e/telegram-user-credential.ts" \ + "$MANTIS_NODE_BIN" --import tsx "$GITHUB_WORKSPACE/scripts/e2e/telegram-user-credential.ts" \ release --lease-file "$lease_file"; then status=1 fi @@ -699,44 +686,3 @@ jobs: --artifact-url "$PUBLISH_ARTIFACT_URL" \ --run-url "https://github.com/${GITHUB_REPOSITORY}/actions/runs/${PUBLISH_RUN_ID}" \ --request-source "$REQUEST_SOURCE" - - clear_issue_comment_reaction: - name: Clear Mantis command reaction - needs: [resolve_request, validate_refs, run_telegram_desktop_proof] - if: ${{ always() && github.event_name == 'issue_comment' && needs.resolve_request.outputs.request_source == 'issue_comment' }} - runs-on: ubuntu-24.04 - permissions: - issues: write - steps: - - name: Remove workflow eyes reaction - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 - with: - script: | - const { owner, repo } = context.repo; - const commentId = context.payload.comment?.id; - if (!commentId) { - core.info("No issue comment id found; skipping reaction cleanup."); - return; - } - - const reactions = await github.paginate(github.rest.reactions.listForIssueComment, { - owner, - repo, - comment_id: commentId, - per_page: 100, - }); - const eyes = reactions.filter( - (reaction) => reaction.content === "eyes" && reaction.user?.login === "github-actions[bot]", - ); - for (const reaction of eyes) { - await github.rest.reactions.deleteForIssueComment({ - owner, - repo, - comment_id: commentId, - reaction_id: reaction.id, - }); - core.info(`Removed eyes reaction ${reaction.id} from comment ${commentId}.`); - } - if (eyes.length === 0) { - core.info(`No workflow eyes reaction found on comment ${commentId}.`); - } diff --git a/scripts/e2e/mock-openai-server.mjs b/scripts/e2e/mock-openai-server.mjs index 82dc881e313e..41f831c5f24f 100644 --- a/scripts/e2e/mock-openai-server.mjs +++ b/scripts/e2e/mock-openai-server.mjs @@ -1,8 +1,9 @@ // Mock OpenAI-compatible server for broader E2E scenarios. import { createHash } from "node:crypto"; import http from "node:http"; +import { setTimeout as delay } from "node:timers/promises"; import { escapeRegExp } from "../lib/regexp.mjs"; -import { readTcpPortEnv } from "./lib/env-limits.mjs"; +import { readPositiveIntEnv, readTcpPortEnv } from "./lib/env-limits.mjs"; import { boundedRequestLogBody, isRequestBodyTooLargeError, @@ -18,8 +19,21 @@ const port = : readTcpPortEnv("OPENCLAW_MOCK_OPENAI_PORT"); const successMarker = process.env.SUCCESS_MARKER ?? "OPENCLAW_E2E_OK"; const requestLog = process.env.MOCK_REQUEST_LOG; +const responseChunkDelayMs = process.env.MOCK_RESPONSE_CHUNK_DELAY_MS + ? readPositiveIntEnv("MOCK_RESPONSE_CHUNK_DELAY_MS", undefined) + : 0; -function responseEvents(text) { +function splitResponseText(text) { + if (text.length < 2) { + return [text]; + } + const midpoint = Math.floor(text.length / 2); + const whitespace = text.lastIndexOf(" ", midpoint); + const splitAt = whitespace > 0 ? whitespace : Math.max(1, midpoint); + return [text.slice(0, splitAt), text.slice(splitAt)]; +} + +function responseEvents(text, deltas = [text]) { const itemId = "msg_e2e_1"; return [ { @@ -32,13 +46,13 @@ function responseEvents(text) { status: "in_progress", }, }, - { + ...deltas.map((delta) => ({ type: "response.output_text.delta", item_id: itemId, output_index: 0, content_index: 0, - delta: text, - }, + delta, + })), { type: "response.output_text.done", item_id: itemId, @@ -81,6 +95,31 @@ function responseEvents(text) { ]; } +async function writeDefaultResponseEvents(res, text) { + if (responseChunkDelayMs === 0) { + writeSse(res, responseEvents(text)); + return; + } + const events = responseEvents(text, splitResponseText(text)); + res.writeHead(200, { + "content-type": "text/event-stream", + "cache-control": "no-store", + connection: "keep-alive", + }); + let deltaCount = 0; + for (const event of events) { + if (event.type === "response.output_text.delta" && deltaCount > 0) { + await delay(responseChunkDelayMs); + } + res.write(`data: ${JSON.stringify(event)}\n\n`); + if (event.type === "response.output_text.delta") { + deltaCount += 1; + } + } + res.write("data: [DONE]\n\n"); + res.end(); +} + function buildMockFunctionCall(name, args) { const serialized = JSON.stringify(args); const suffix = createHash("sha256") @@ -380,7 +419,7 @@ const server = http.createServer((req, res) => { }); return; } - writeSse(res, responseEvents(responseText)); + await writeDefaultResponseEvents(res, responseText); return; } diff --git a/scripts/e2e/telegram-user-crabbox-proof.ts b/scripts/e2e/telegram-user-crabbox-proof.ts index 576052f4a9b8..4e08bf138a8e 100644 --- a/scripts/e2e/telegram-user-crabbox-proof.ts +++ b/scripts/e2e/telegram-user-crabbox-proof.ts @@ -66,7 +66,9 @@ type Options = { idleTimeout: string; keepBox: boolean; leaseId?: string; + linkPreview?: boolean; mcpAppFixture: boolean; + mockResponseChunkDelayMs?: number; mockResponseText: string; mockPort: number; outputDir: string; @@ -92,6 +94,8 @@ type Options = { timeoutMs: number; ttl: string; userDriverScript: string; + nodeBin: string; + pnpmBin?: string; }; type FunnelBridge = { @@ -208,7 +212,9 @@ function usageText() { " --desktop-chat-title Telegram Desktop chat to select before recording.", " --id Reuse an existing Crabbox desktop lease.", " --keep-box Leave the Crabbox lease running for VNC debugging.", + " --link-preview Set channels.telegram.linkPreview before Gateway startup.", " --mock-response-file Text returned by the mock model.", + " --mock-response-chunk-delay-ms Split the mock reply across two delayed deltas.", " --mcp-app-fixture Configure the pinned MCP App fixture through a Crabbox Funnel.", " --output-dir Artifact directory under the repo.", " --message-id Telegram message id for proof-view deep link.", @@ -283,6 +289,16 @@ function parseTcpPort(value: string, label: string) { return parsed; } +function parseBoolean(value: string, label: string) { + if (value === "true") { + return true; + } + if (value === "false") { + return false; + } + throw new Error(`${label} must be true or false.`); +} + function createTelegramProofRunId() { return `${new Date().toISOString().replace(/[:.]/gu, "-")}-${randomUUID().slice(0, 8)}`; } @@ -332,6 +348,8 @@ export function parseArgs(argvInput: string[]): Options { ttl: "120m", userDriverScript: trimToValue(process.env.OPENCLAW_TELEGRAM_USER_DRIVER_SCRIPT) ?? DEFAULT_USER_DRIVER, + nodeBin: trimToValue(process.env.MANTIS_NODE_BIN) ?? process.execPath, + pnpmBin: trimToValue(process.env.MANTIS_PNPM_BIN), }; const commandSeparator = argv.indexOf("--"); if (command === "run" && commandSeparator >= 0) { @@ -383,10 +401,17 @@ export function parseArgs(argvInput: string[]): Options { opts.idleTimeout = readValue(); } else if (arg === "--keep-box") { opts.keepBox = true; + } else if (arg === "--link-preview") { + opts.linkPreview = parseBoolean(readValue(), "--link-preview"); } else if (arg === "--mock-port") { opts.mockPort = parseTcpPort(readValue(), "--mock-port"); } else if (arg === "--mock-response-file") { opts.mockResponseText = fs.readFileSync(resolveRepoPath(process.cwd(), readValue()), "utf8"); + } else if (arg === "--mock-response-chunk-delay-ms") { + opts.mockResponseChunkDelayMs = parsePositiveTimerMs( + readValue(), + "--mock-response-chunk-delay-ms", + ); } else if (arg === "--mcp-app-fixture") { opts.mcpAppFixture = true; } else if (arg === "--message-id") { @@ -541,12 +566,20 @@ function childProcessBaseEnv() { return env; } -function mockServerEnv(params: { mockPort: number; mockResponseText: string; requestLog: string }) { +function mockServerEnv(params: { + mockPort: number; + mockResponseChunkDelayMs?: number; + mockResponseText: string; + requestLog: string; +}) { return { ...childProcessBaseEnv(), MOCK_PORT: String(params.mockPort), MOCK_REQUEST_LOG: params.requestLog, SUCCESS_MARKER: params.mockResponseText, + ...(params.mockResponseChunkDelayMs === undefined + ? {} + : { MOCK_RESPONSE_CHUNK_DELAY_MS: String(params.mockResponseChunkDelayMs) }), }; } @@ -577,8 +610,16 @@ export function createOpenClawGatewaySpawnSpec(params: { comSpec?: string; nodeExecPath?: string; npmExecPath?: string; + pnpmExecPath?: string; platform?: NodeJS.Platform; }): GatewaySpawnSpec { + if (params.pnpmExecPath) { + return { + args: ["openclaw", "gateway", "--port", String(params.gatewayPort)], + command: params.pnpmExecPath, + options: { cwd: params.repoRoot, env: params.env, shell: false }, + }; + } const spec = createPnpmRunnerSpawnSpec({ comSpec: params.comSpec, cwd: params.repoRoot, @@ -1137,6 +1178,7 @@ function telegramResultObject(value: unknown, label: string): JsonObject { export function writeSutConfig(params: { gatewayPort: number; groupId: string; + linkPreview?: boolean; mcpAppFixture?: boolean; mockPort: number; outputDir: string; @@ -1185,6 +1227,7 @@ export function writeSutConfig(params: { requireMention: false, }, }, + ...(params.linkPreview === undefined ? {} : { linkPreview: params.linkPreview }), replyToMode: "first", }, }, @@ -1264,10 +1307,14 @@ export async function startLocalSut( groupId: string; mockResponseText: string; mockPort: number; + linkPreview?: boolean; + mockResponseChunkDelayMs?: number; outputDir: string; sutToken: string; testerId: string; repoRoot: string; + nodeBin?: string; + pnpmBin?: string; }, deps: StartLocalSutDeps = {}, ) { @@ -1282,10 +1329,14 @@ export async function startLocalSut( const drained = await drainUpdates(params.sutToken); const config = writeConfig(params); const requestLog = path.join(params.outputDir, "mock-openai-requests.ndjson"); - mock = spawnLoggedCommand("node", ["scripts/e2e/mock-openai-server.mjs"], { - cwd: params.repoRoot, - env: mockServerEnv({ ...params, requestLog }), - }); + mock = spawnLoggedCommand( + params.nodeBin ?? process.execPath, + ["scripts/e2e/mock-openai-server.mjs"], + { + cwd: params.repoRoot, + env: mockServerEnv({ ...params, requestLog }), + }, + ); const runningMock = mock; await waitForOutputReady( runningMock.child, @@ -1297,6 +1348,7 @@ export async function startLocalSut( const gatewaySpec = createGatewaySpawnSpec({ env: gatewayEnv({ ...config, sutToken: params.sutToken }), gatewayPort: params.gatewayPort, + pnpmExecPath: params.pnpmBin, repoRoot: params.repoRoot, }); gateway = spawnLoggedCommand(gatewaySpec.command, gatewaySpec.args, gatewaySpec.options); @@ -1376,11 +1428,15 @@ async function startLocalSutDaemon(params: { groupId: string; mockResponseText: string; mockPort: number; + linkPreview?: boolean; mcpAppFixture?: boolean; + mockResponseChunkDelayMs?: number; outputDir: string; sutToken: string; testerId: string; repoRoot: string; + nodeBin?: string; + pnpmBin?: string; }) { const drained = await drainSutUpdates(params.sutToken); const config = writeSutConfig(params); @@ -1392,7 +1448,7 @@ async function startLocalSutDaemon(params: { let gatewayPid: number | undefined; try { mockPid = spawnDaemon({ - command: "node", + command: params.nodeBin ?? process.execPath, args: ["scripts/e2e/mock-openai-server.mjs"], cwd: params.repoRoot, env: mockServerEnv({ ...params, requestLog }), @@ -1414,6 +1470,7 @@ async function startLocalSutDaemon(params: { const gatewaySpec = createOpenClawGatewaySpawnSpec({ env: gatewayEnvVars, gatewayPort: params.gatewayPort, + pnpmExecPath: params.pnpmBin, repoRoot: params.repoRoot, }); gatewayPid = spawnDaemon({ @@ -2414,10 +2471,14 @@ async function startSession(root: string, opts: Options, outputDir: string) { funnelBridge, gatewayPort: opts.gatewayPort, groupId: credential.groupId, + linkPreview: opts.linkPreview, mockResponseText: opts.mockResponseText, + mockResponseChunkDelayMs: opts.mockResponseChunkDelayMs, mockPort: opts.mockPort, mcpAppFixture: opts.mcpAppFixture, outputDir, + nodeBin: opts.nodeBin, + pnpmBin: opts.pnpmBin, repoRoot: root, sutToken: credential.sutToken, testerId: credential.testerUserId, @@ -2978,9 +3039,13 @@ async function main() { const sutRuntime = await startLocalSut({ gatewayPort: opts.gatewayPort, groupId: credential.groupId, + linkPreview: opts.linkPreview, mockResponseText: opts.mockResponseText, + mockResponseChunkDelayMs: opts.mockResponseChunkDelayMs, mockPort: opts.mockPort, outputDir, + nodeBin: opts.nodeBin, + pnpmBin: opts.pnpmBin, repoRoot: root, sutToken: credential.sutToken, testerId: credential.testerUserId, diff --git a/test/scripts/e2e-mock-config-limits.test.ts b/test/scripts/e2e-mock-config-limits.test.ts index fda8685dd6c3..9ca563b18931 100644 --- a/test/scripts/e2e-mock-config-limits.test.ts +++ b/test/scripts/e2e-mock-config-limits.test.ts @@ -19,6 +19,7 @@ const scrubbedEnvKeys = [ "FIXTURE_PORT", "MOCK_PORT", "MOCK_REQUEST_LOG", + "MOCK_RESPONSE_CHUNK_DELAY_MS", "MOCK_TLS_CERT", "MOCK_TLS_KEY", "OPENCLAW_CONFIG_RELOAD_LOG_MAX_READ_BYTES", @@ -159,6 +160,29 @@ describe("mock OpenAI response markers", () => { }); }); + it("can split a deterministic response across delayed streaming deltas", async () => { + await withMockServer( + mockOpenAiPath, + { + MOCK_RESPONSE_CHUNK_DELAY_MS: "80", + SUCCESS_MARKER: "First streamed preview remains visible before the follow-up edit arrives.", + }, + async (baseUrl) => { + const startedAt = Date.now(); + const response = await fetch(`${baseUrl}/v1/responses`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ input: "return the configured marker", stream: true }), + }); + const body = await response.text(); + + expect(response.status).toBe(200); + expect(body.match(/response\.output_text\.delta/gu)).toHaveLength(2); + expect(Date.now() - startedAt).toBeGreaterThanOrEqual(60); + }, + ); + }); + it("drives the MCP App fixture tool before returning the visible marker", async () => { await withMockServer(mockOpenAiPath, {}, async (baseUrl) => { const first = await fetch(`${baseUrl}/v1/responses`, { diff --git a/test/scripts/mantis-telegram-desktop-proof-workflow.test.ts b/test/scripts/mantis-telegram-desktop-proof-workflow.test.ts index f0212dd60313..dc59c67eae45 100644 --- a/test/scripts/mantis-telegram-desktop-proof-workflow.test.ts +++ b/test/scripts/mantis-telegram-desktop-proof-workflow.test.ts @@ -34,9 +34,6 @@ type Workflow = { env?: Record; jobs?: Record; on?: { - pull_request_target?: { - types?: string[]; - }; workflow_dispatch?: { inputs?: Record< string, @@ -172,6 +169,8 @@ describe("Mantis Telegram Desktop proof workflow", () => { expect(cleanupStep.run).toContain("sudo -u codex env"); expect(cleanupStep.run).not.toContain("*/telegram-user-crabbox/*/session.json"); expect(cleanupStep.run).not.toContain("*/telegram-user-crabbox/*/.session/lease.json"); + expect(cleanupStep.run).toContain('sudo -u codex "$MANTIS_NODE_BIN"'); + expect(cleanupStep.run).not.toContain("sudo -u codex node"); }); it("cleans partially started proof daemons when local SUT startup fails", () => { @@ -184,31 +183,17 @@ describe("Mantis Telegram Desktop proof workflow", () => { expect(proofScript).toContain("throw error;"); }); - it("uses the OpenClaw Mantis mention as the comment trigger", () => { - const workflow = readFileSync(WORKFLOW, "utf8"); - const liveWorkflow = readFileSync(LIVE_WORKFLOW, "utf8"); - expect(workflow).toContain("@openclaw-mantis"); - expect(workflow).toContain("/openclaw-mantis"); - expect(workflow).toContain("mantis: telegram-visible-proof"); - expect(workflow).toContain('setOutput("should_run", "false")'); - expect(workflow).toContain('normalized.includes("telegram desktop")'); - expect(liveWorkflow).toContain('normalized.includes("telegram desktop")'); - expect(liveWorkflow).toContain("!requestedDesktopProof"); - expect(workflow).not.toContain("@Mantis"); - expect(workflow).not.toContain("@mantis"); - expect(workflow).not.toContain('"/mantis"'); - }); - - it("runs when ClawSweeper applies the Telegram proof label", () => { + it("requires explicit maintainer dispatch before executing a PR worktree", () => { const workflow = parse(readFileSync(WORKFLOW, "utf8")) as Workflow; const workflowText = readFileSync(WORKFLOW, "utf8"); - expect(workflow.on?.pull_request_target?.types).toContain("labeled"); - expect(workflowText).toContain("github.event.label.name == 'mantis: telegram-visible-proof'"); - expect(workflowText).toContain('eventName === "pull_request_target"'); - expect(workflowText).toContain("context.payload.pull_request?.number"); - expect(workflowText).toContain("Accepted Mantis label trigger"); - expect(workflowText).toContain("allow-bot-users: clawsweeper[bot]"); + expect(workflow.on?.workflow_dispatch).toBeDefined(); + expect(workflowText).not.toContain("issue_comment:"); + expect(workflowText).not.toContain("pull_request_target:"); + expect(workflowText).not.toContain("clear_issue_comment_reaction:"); + expect(workflowText).toContain("allow-bot-users: github-actions[bot]"); + expect(workflowText).not.toContain("allow-bot-users: clawsweeper[bot]"); + expect(workflowText).toContain('setOutput("request_source", "workflow_dispatch")'); }); it("can publish an existing proof artifact without recapturing", () => { @@ -298,9 +283,13 @@ describe("Mantis Telegram Desktop proof workflow", () => { it("installs local proof tools before the Codex agent runs", () => { const install = workflowStep("Install local proof tools"); expect(install.run).toContain("test -f scripts/e2e/telegram-user-driver.py"); + expect(install.run).toContain('node_bin="$(command -v node)"'); + expect(install.run).toContain('corepack_bin="$(command -v corepack)"'); + expect(install.run).toContain("/usr/local/lib/mantis-toolchain/node"); + expect(install.run).toContain("/usr/local/lib/mantis-toolchain/pnpm"); expect(install.run).toContain("/usr/local/bin/openclaw-telegram-user-crabbox-proof"); expect(install.run).toContain( - 'exec node --import tsx "${GITHUB_WORKSPACE}/scripts/e2e/telegram-user-crabbox-proof.ts" "$@"', + 'exec /usr/local/lib/mantis-toolchain/node --import tsx "${GITHUB_WORKSPACE}/scripts/e2e/telegram-user-crabbox-proof.ts" "\\$@"', ); expect(install.run).toContain("BtbN/FFmpeg-Builds"); expect(install.run).toContain("ffmpeg-master-latest-linux64-gpl.tar.xz"); @@ -316,6 +305,8 @@ describe("Mantis Telegram Desktop proof workflow", () => { "/usr/local/bin/openclaw-telegram-user-crabbox-proof", ); expect(agent.env?.OPENCLAW_TELEGRAM_USER_CRABBOX_BIN).toBe("/usr/local/bin/crabbox"); + expect(agent.env?.MANTIS_NODE_BIN).toBe("/usr/local/lib/mantis-toolchain/node"); + expect(agent.env?.MANTIS_PNPM_BIN).toBe("/usr/local/lib/mantis-toolchain/pnpm"); expect(agent.env?.CRABBOX_COORDINATOR).toContain( "secrets.CRABBOX_COORDINATOR || secrets.OPENCLAW_QA_MANTIS_CRABBOX_COORDINATOR", ); @@ -328,9 +319,13 @@ describe("Mantis Telegram Desktop proof workflow", () => { "OPENCLAW_TELEGRAM_USER_CRABBOX_BIN OPENCLAW_TELEGRAM_USER_CRABBOX_PROVIDER OPENCLAW_TELEGRAM_USER_DRIVER_SCRIPT OPENCLAW_TELEGRAM_USER_PROOF_CMD", ); expect(prepare.run).toContain("MANTIS_CANDIDATE_TRUST"); + expect(prepare.run).toContain("MANTIS_NODE_BIN MANTIS_PNPM_BIN"); const prompt = readFileSync(PROMPT, "utf8"); expect(prompt).toContain("$OPENCLAW_TELEGRAM_USER_PROOF_CMD"); + expect(prompt).toContain("`--link-preview false`"); + expect(prompt).toContain("Do not edit the generated\n config or restart the Gateway"); + expect(prompt).toContain("`--mock-response-chunk-delay-ms 1200`"); expect(prompt).toContain("do not run\n `pnpm qa:telegram-user:crabbox` directly"); expect(prompt).toContain("Let `start` return or fail on its\n own"); expect(prompt).toContain("`--mcp-app-fixture` option"); @@ -342,6 +337,27 @@ describe("Mantis Telegram Desktop proof workflow", () => { ); }); + it("prepares exact proof worktrees before agent secrets are available", () => { + const step = workflowStep("Prepare proof worktrees with pinned toolchain"); + const run = step.run ?? ""; + + expect(run).toContain('git cat-file -e "${BASELINE_SHA}^{commit}"'); + expect(run).toContain('git fetch --no-tags origin "pull/${MANTIS_PR_NUMBER}/head"'); + expect(run).toContain('git worktree add --detach "$baseline_root" "$BASELINE_SHA"'); + expect(run).toContain('git worktree add --detach "$candidate_root" "$CANDIDATE_SHA"'); + expect(run.match(/env -i/gu)).toHaveLength(3); + expect(run).toContain('"$toolchain_dir/pnpm" install --frozen-lockfile'); + expect(run).toContain('"$toolchain_dir/pnpm" build'); + expect(run).toContain("sudo -u mantis-builder env -i"); + expect(run).toContain('sudo chown -R mantis-builder:mantis-builder "$candidate_root"'); + expect(run).toContain('test "$(cat "$candidate_root/.git")" = "$candidate_git_link"'); + expect(run).toContain('git -C "$candidate_root" diff --exit-code'); + expect(run).not.toContain("GH_TOKEN"); + expect(run).not.toContain("OPENAI_API_KEY"); + expect(run).not.toContain("CRABBOX_"); + expect(run).not.toContain("OPENCLAW_QA_"); + }); + it("pins AWS Crabbox proof runs to the working region", () => { const workflow = parse(readFileSync(WORKFLOW, "utf8")) as Workflow; const liveWorkflow = parse(readFileSync(LIVE_WORKFLOW, "utf8")) as Workflow; diff --git a/test/scripts/telegram-user-crabbox-proof.test.ts b/test/scripts/telegram-user-crabbox-proof.test.ts index dc74f4a0f593..1f0a5c5e0ae9 100644 --- a/test/scripts/telegram-user-crabbox-proof.test.ts +++ b/test/scripts/telegram-user-crabbox-proof.test.ts @@ -113,6 +113,20 @@ describe("telegram user Crabbox proof log polling", () => { expect(spec.options.shell).toBe(false); }); + it("uses an explicitly pinned pnpm executable for a worktree gateway", () => { + const spec = createOpenClawGatewaySpawnSpec({ + env: { PATH: "/definitely-missing" }, + gatewayPort: 19042, + pnpmExecPath: "/opt/mantis-toolchain/pnpm", + repoRoot: "/repo", + }); + + expect(spec.command).toBe("/opt/mantis-toolchain/pnpm"); + expect(spec.args).toEqual(["openclaw", "gateway", "--port", "19042"]); + expect(spec.options.cwd).toBe("/repo"); + expect(spec.options.shell).toBe(false); + }); + it("allows cold remote setup to outlive ordinary command timeouts", () => { expect(REMOTE_SETUP_COMMAND_TIMEOUT_MS).toBeGreaterThan(COMMAND_TIMEOUT_MS); expect(REMOTE_SETUP_COMMAND_TIMEOUT_MS).toBeGreaterThanOrEqual(90 * 60 * 1000); @@ -166,6 +180,24 @@ describe("telegram user Crabbox proof log polling", () => { expect(parseArgs(["--text", "-ping"]).text).toBe("-ping"); }); + it("accepts an explicit Telegram link-preview setting", () => { + expect(parseArgs(["start", "--link-preview", "false"]).linkPreview).toBe(false); + expect(parseArgs(["start", "--link-preview", "true"]).linkPreview).toBe(true); + expect(parseArgs(["start"]).linkPreview).toBeUndefined(); + expect(() => parseArgs(["start", "--link-preview", "disabled"])).toThrow( + "--link-preview must be true or false.", + ); + }); + + it("accepts a positive mock response chunk delay", () => { + expect( + parseArgs(["start", "--mock-response-chunk-delay-ms", "1200"]).mockResponseChunkDelayMs, + ).toBe(1200); + expect(() => parseArgs(["start", "--mock-response-chunk-delay-ms", "0"])).toThrow( + "--mock-response-chunk-delay-ms must be a positive integer.", + ); + }); + it("rejects duplicate single-value proof controls while keeping repeated expectations", () => { expect(() => parseArgs(["--output-dir", ".artifacts/one", "--output-dir", ".artifacts/two"]), @@ -241,6 +273,31 @@ describe("telegram user Crabbox proof log polling", () => { expect(JSON.stringify(config)).not.toContain("resource-ok"); }); + it("injects the requested Telegram link-preview setting before startup", () => { + const disabledConfigRoot = writeSutConfig({ + gatewayPort: 19042, + groupId: "group", + linkPreview: false, + mockPort: 19043, + outputDir: makeTempDir(), + testerId: "tester", + }); + const defaultConfigRoot = writeSutConfig({ + gatewayPort: 19044, + groupId: "group", + mockPort: 19045, + outputDir: makeTempDir(), + testerId: "tester", + }); + tempDirs.push(disabledConfigRoot.tempRoot, defaultConfigRoot.tempRoot); + + const disabledConfig = JSON.parse(fs.readFileSync(disabledConfigRoot.configPath, "utf8")); + const defaultConfig = JSON.parse(fs.readFileSync(defaultConfigRoot.configPath, "utf8")); + + expect(disabledConfig.channels.telegram.linkPreview).toBe(false); + expect(defaultConfig.channels.telegram).not.toHaveProperty("linkPreview"); + }); + it("pins the browser fixture SDK and exposes only the required app capabilities", () => { const fixture = fs.readFileSync("scripts/e2e/mcp-app-conformance-server.mjs", "utf8"); const uiPackage = JSON.parse(fs.readFileSync("ui/package.json", "utf8"));