fix(qa): harden Mantis Telegram proof harness (#114429)

* fix(qa): harden Mantis Telegram proof harness

* fix(qa): require explicit Mantis dispatch
This commit is contained in:
Peter Steinberger
2026-07-27 04:33:06 -04:00
committed by GitHub
parent 4aa2ab2b25
commit 45bad50e26
8 changed files with 374 additions and 200 deletions
@@ -47,6 +47,23 @@ proof_cmd="${OPENCLAW_TELEGRAM_USER_PROOF_CMD:-openclaw-telegram-user-crabbox-pr
--output-dir .artifacts/qa-e2e/telegram-user-crabbox/pr-review
```
When the scenario needs Telegram link previews explicitly enabled or disabled,
set the SUT config before Gateway startup:
```bash
"$proof_cmd" start \
--tdlib-url http://artifacts.openclaw.ai/tdlib-v1.8.0-linux-x64.tgz \
--link-preview false \
--mock-response-chunk-delay-ms 1200 \
--mock-response-file .artifacts/qa-e2e/telegram-user-crabbox/reply.txt \
--output-dir .artifacts/qa-e2e/telegram-user-crabbox/pr-review
```
Do not edit the generated config or restart the held-session Gateway to apply
this setting; that bypasses the runner's lifecycle and cleanup ownership.
The chunk delay splits the configured response into two streaming deltas so a
recording can show the initial preview and a later in-place edit.
The runner defaults to `--class standard`, `--record-fps 24`,
`--preview-fps 24`, and `--preview-width 1920`. Keep those defaults unless the
proof needs something else.
@@ -111,14 +111,16 @@ than Telegram-visible behavior`. Use this manifest shape and do not create
Reopen that same Telegram button after its ticket expires and capture the
expired state. Do not substitute Control UI, transcript, curl, or a newly
minted button for any part of that path.
5. Create detached worktrees under
5. Use the workflow-prepared detached worktrees under
`.artifacts/qa-e2e/mantis/telegram-desktop-proof-worktrees/baseline` and
`.artifacts/qa-e2e/mantis/telegram-desktop-proof-worktrees/candidate`, then
install and build each worktree with the repo's normal `pnpm` commands.
`.artifacts/qa-e2e/mantis/telegram-desktop-proof-worktrees/candidate`.
Verify their `HEAD`s match `BASELINE_SHA` and `CANDIDATE_SHA`; do not create,
install, rebuild, or replace them. The workflow prepared both with a pinned
Node/pnpm toolchain before agent secrets were available.
If `MANTIS_CANDIDATE_TRUST` is `fork-pr-head`, treat the
candidate worktree as untrusted fork code: do not pass GitHub, OpenAI,
Crabbox, Convex, or other workflow secrets into candidate install, build, or
runtime commands. The candidate SUT may receive only the proof runner's
Crabbox, Convex, or other workflow secrets into candidate runtime commands.
The candidate SUT may receive only the proof runner's
short-lived Telegram bot token, generated local config/state paths, and mock
model key needed for this isolated proof.
6. In each worktree, run the real-user Telegram Crabbox proof flow from the
@@ -133,6 +135,14 @@ than Telegram-visible behavior`. Use this manifest shape and do not create
own; do not kill it while Crabbox is still waiting for bootstrap. Use a long
command timeout for `start`, `send`, `view`, and `finish`. You may iterate
and rerun if the visual result is not convincing.
When the requested scenario needs `channels.telegram.linkPreview: false`,
pass `--link-preview false` to `start`. The runner injects that setting into
the isolated SUT config before Gateway startup. Do not edit the generated
config or restart the Gateway to apply it.
When the proof must show an in-place streamed edit, also pass
`--mock-response-chunk-delay-ms 1200` and use a mock response long enough
for the first chunk to clear the preview debounce. Capture both the initial
partial reply and the later edit before finishing.
7. Open Telegram Desktop directly to the newest relevant message with the
runner `view` command before finishing each recording. Keep the chat scrolled
to the bottom so new proof messages appear in-frame.
@@ -1,10 +1,6 @@
name: Mantis Telegram Desktop Proof
on:
issue_comment:
types: [created]
pull_request_target: # zizmor: ignore[dangerous-triggers] maintainer-owned Mantis label trigger; trusted base workflow validates refs before checkout/use
types: [labeled]
workflow_dispatch:
inputs:
pr_number:
@@ -53,63 +49,8 @@ env:
MANTIS_OUTPUT_DIR: .artifacts/qa-e2e/mantis/telegram-desktop-proof
jobs:
authorize_actor:
name: Authorize workflow actor
if: >-
${{
github.event_name == 'workflow_dispatch' ||
(
github.event_name == 'pull_request_target' &&
github.event.action == 'labeled' &&
github.event.label.name == 'mantis: telegram-visible-proof'
) ||
(
github.event_name == 'issue_comment' &&
github.event.issue.pull_request &&
contains(github.event.issue.labels.*.name, 'mantis: telegram-visible-proof') &&
(
contains(github.event.comment.body, '@openclaw-mantis') ||
contains(github.event.comment.body, '/openclaw-mantis')
)
)
}}
runs-on: ubuntu-24.04
outputs:
authorized: ${{ steps.permission.outputs.authorized }}
steps:
- name: Require maintainer-level repository access
id: permission
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
with:
script: |
if (context.eventName === "pull_request_target") {
core.info(`Accepted Mantis label trigger from ${context.actor}.`);
core.setOutput("authorized", "true");
return;
}
const allowed = new Set(["admin", "maintain", "write"]);
const { owner, repo } = context.repo;
const { data } = await github.rest.repos.getCollaboratorPermissionLevel({
owner,
repo,
username: context.actor,
});
const permission = data.permission;
core.info(`Actor ${context.actor} permission: ${permission}`);
if (!allowed.has(permission)) {
core.notice(
`Workflow requires write/maintain/admin access. Actor "${context.actor}" has "${permission}".`,
);
core.setOutput("authorized", "false");
return;
}
core.setOutput("authorized", "true");
resolve_request:
name: Resolve Mantis request
needs: authorize_actor
if: needs.authorize_actor.outputs.authorized == 'true'
runs-on: ubuntu-24.04
outputs:
baseline_ref: ${{ steps.resolve.outputs.baseline_ref }}
@@ -128,57 +69,19 @@ jobs:
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
with:
script: |
const eventName = context.eventName;
function setOutput(name, value) {
core.setOutput(name, value ?? "");
core.info(`${name}=${value ?? ""}`);
}
const inputs = context.payload.inputs ?? {};
const prNumber =
eventName === "workflow_dispatch"
? inputs.pr_number
: eventName === "pull_request_target"
? String(context.payload.pull_request?.number ?? "")
: String(context.payload.issue?.number ?? "");
const prNumber = inputs.pr_number;
if (!prNumber) {
core.setFailed("Mantis Telegram desktop proof requires a pull request.");
return;
}
const body =
eventName === "workflow_dispatch"
? inputs.instructions || ""
: eventName === "issue_comment"
? context.payload.comment?.body || ""
: "";
if (eventName === "issue_comment") {
const normalized = body.toLowerCase();
const requestedDesktopProof =
(normalized.includes("@openclaw-mantis") || normalized.includes("/openclaw-mantis")) &&
(normalized.includes("desktop proof") ||
normalized.includes("desktop-proof") ||
normalized.includes("telegram desktop") ||
normalized.includes("native telegram") ||
normalized.includes("visible proof") ||
normalized.includes("visible-proof") ||
normalized.includes("telegram-visible-proof"));
if (!requestedDesktopProof) {
core.notice("Comment mentioned Mantis but did not request Telegram desktop proof.");
setOutput("should_run", "false");
setOutput("baseline_ref", "");
setOutput("candidate_ref", "");
setOutput("pr_number", "");
setOutput("instructions", "");
setOutput("crabbox_provider", "");
setOutput("lease_id", "");
setOutput("publish_artifact_name", "");
setOutput("publish_run_id", "");
setOutput("request_source", "unsupported_issue_comment");
return;
}
}
const body = inputs.instructions || "";
const { owner, repo } = context.repo;
const { data: pr } = await github.rest.pulls.get({
@@ -201,16 +104,7 @@ jobs:
setOutput("lease_id", inputs.crabbox_lease_id || "");
setOutput("publish_artifact_name", inputs.publish_artifact_name || "");
setOutput("publish_run_id", inputs.publish_run_id || "");
setOutput("request_source", eventName);
if (eventName === "issue_comment") {
await github.rest.reactions.createForIssueComment({
owner,
repo,
comment_id: context.payload.comment.id,
content: "eyes",
}).catch((error) => core.warning(`Could not add eyes reaction: ${error.message}`));
}
setOutput("request_source", "workflow_dispatch");
validate_refs:
name: Validate selected refs
@@ -387,13 +281,32 @@ jobs:
run: |
set -euo pipefail
test -f scripts/e2e/telegram-user-driver.py
cat >"${RUNNER_TEMP}/openclaw-telegram-user-crabbox-proof" <<'EOF'
node_bin="$(command -v node)"
corepack_bin="$(command -v corepack)"
"$node_bin" "$corepack_bin" pnpm --version >/dev/null
cat >"${RUNNER_TEMP}/mantis-node" <<EOF
#!/usr/bin/env bash
set -euo pipefail
exec node --import tsx "${GITHUB_WORKSPACE}/scripts/e2e/telegram-user-crabbox-proof.ts" "$@"
exec "$node_bin" "\$@"
EOF
cat >"${RUNNER_TEMP}/mantis-pnpm" <<EOF
#!/usr/bin/env bash
set -euo pipefail
exec "$node_bin" "$corepack_bin" pnpm "\$@"
EOF
cat >"${RUNNER_TEMP}/openclaw-telegram-user-crabbox-proof" <<EOF
#!/usr/bin/env bash
set -euo pipefail
exec /usr/local/lib/mantis-toolchain/node --import tsx "${GITHUB_WORKSPACE}/scripts/e2e/telegram-user-crabbox-proof.ts" "\$@"
EOF
chmod 0755 "${RUNNER_TEMP}/mantis-node" "${RUNNER_TEMP}/mantis-pnpm"
chmod 0755 "${RUNNER_TEMP}/openclaw-telegram-user-crabbox-proof"
sudo install -d -m 0755 /usr/local/lib/mantis-toolchain
sudo install -m 0755 "${RUNNER_TEMP}/mantis-node" /usr/local/lib/mantis-toolchain/node
sudo install -m 0755 "${RUNNER_TEMP}/mantis-pnpm" /usr/local/lib/mantis-toolchain/pnpm
sudo install -m 0755 "${RUNNER_TEMP}/openclaw-telegram-user-crabbox-proof" /usr/local/bin/openclaw-telegram-user-crabbox-proof
/usr/local/lib/mantis-toolchain/node --version
/usr/local/lib/mantis-toolchain/pnpm --version
/usr/local/bin/openclaw-telegram-user-crabbox-proof --help >/dev/null
media_tools="${RUNNER_TEMP}/mantis-media-tools"
install -d "$media_tools"
@@ -408,6 +321,76 @@ jobs:
ffmpeg -version >/dev/null
ffprobe -version >/dev/null
- name: Prepare proof worktrees with pinned toolchain
env:
BASELINE_SHA: ${{ needs.validate_refs.outputs.baseline_revision }}
CANDIDATE_SHA: ${{ needs.validate_refs.outputs.candidate_revision }}
MANTIS_PR_NUMBER: ${{ needs.resolve_request.outputs.pr_number }}
shell: bash
run: |
set -euo pipefail
worktree_root="$GITHUB_WORKSPACE/.artifacts/qa-e2e/mantis/telegram-desktop-proof-worktrees"
baseline_root="$worktree_root/baseline"
candidate_root="$worktree_root/candidate"
toolchain_dir=/usr/local/lib/mantis-toolchain
corepack_home="${RUNNER_TEMP}/mantis-corepack"
mkdir -p "$worktree_root" "$corepack_home"
git cat-file -e "${BASELINE_SHA}^{commit}"
if ! git cat-file -e "${CANDIDATE_SHA}^{commit}"; then
git fetch --no-tags origin "pull/${MANTIS_PR_NUMBER}/head"
fi
git cat-file -e "${CANDIDATE_SHA}^{commit}"
git worktree add --detach "$baseline_root" "$BASELINE_SHA"
git worktree add --detach "$candidate_root" "$CANDIDATE_SHA"
candidate_git_link="$(cat "$candidate_root/.git")"
prepare_worktree() {
local repo_root="$1"
local safe_home="$2"
mkdir -p "$safe_home"
(
cd "$repo_root"
env -i \
CI=1 \
COREPACK_HOME="$corepack_home" \
HOME="$safe_home" \
OPENCLAW_BUILD_PRIVATE_QA=1 \
OPENCLAW_ENABLE_PRIVATE_QA_CLI=1 \
PATH="$toolchain_dir:/usr/bin:/bin" \
"$toolchain_dir/pnpm" install --frozen-lockfile
env -i \
CI=1 \
COREPACK_HOME="$corepack_home" \
HOME="$safe_home" \
OPENCLAW_BUILD_PRIVATE_QA=1 \
OPENCLAW_ENABLE_PRIVATE_QA_CLI=1 \
PATH="$toolchain_dir:/usr/bin:/bin" \
"$toolchain_dir/pnpm" build
)
}
prepare_worktree "$baseline_root" "${RUNNER_TEMP}/mantis-baseline-home"
sudo useradd --system --no-create-home --shell /usr/sbin/nologin mantis-builder
candidate_home="${RUNNER_TEMP}/mantis-candidate-home"
candidate_corepack_home="${RUNNER_TEMP}/mantis-candidate-corepack"
sudo install -d -m 0700 -o mantis-builder -g mantis-builder "$candidate_home"
sudo install -d -m 0700 -o mantis-builder -g mantis-builder "$candidate_corepack_home"
sudo chown -R mantis-builder:mantis-builder "$candidate_root"
sudo -u mantis-builder env -i \
CI=1 \
COREPACK_HOME="$candidate_corepack_home" \
HOME="$candidate_home" \
OPENCLAW_BUILD_PRIVATE_QA=1 \
OPENCLAW_ENABLE_PRIVATE_QA_CLI=1 \
PATH="$toolchain_dir:/usr/bin:/bin" \
/bin/bash -c 'cd "$1" && "$2" install --frozen-lockfile && "$2" build' \
bash "$candidate_root" "$toolchain_dir/pnpm"
test "$(cat "$candidate_root/.git")" = "$candidate_git_link"
git -C "$candidate_root" diff --exit-code
git -C "$candidate_root" diff --cached --exit-code
test "$(git -C "$baseline_root" rev-parse HEAD)" = "$BASELINE_SHA"
test "$(git -C "$candidate_root" rev-parse HEAD)" = "$CANDIDATE_SHA"
- name: Ensure agent key exists
env:
OPENAI_API_KEY: ${{ secrets.OPENCLAW_MANTIS_AGENT_OPENAI_API_KEY || secrets.OPENAI_API_KEY }}
@@ -428,6 +411,7 @@ jobs:
printf '%s\n' 'Defaults env_keep += "BASELINE_REF BASELINE_SHA CANDIDATE_REF CANDIDATE_SHA"'
printf '%s\n' 'Defaults env_keep += "CRABBOX_ACCESS_CLIENT_ID CRABBOX_ACCESS_CLIENT_SECRET CRABBOX_COORDINATOR CRABBOX_COORDINATOR_TOKEN CRABBOX_AWS_REGION CRABBOX_CAPACITY_REGIONS CRABBOX_LEASE_ID CRABBOX_PROVIDER"'
printf '%s\n' 'Defaults env_keep += "GH_TOKEN MANTIS_CANDIDATE_TRUST MANTIS_INSTRUCTIONS MANTIS_OUTPUT_DIR MANTIS_PR_NUMBER"'
printf '%s\n' 'Defaults env_keep += "MANTIS_NODE_BIN MANTIS_PNPM_BIN"'
printf '%s\n' 'Defaults env_keep += "OPENCLAW_BUILD_PRIVATE_QA OPENCLAW_ENABLE_PRIVATE_QA_CLI OPENCLAW_QA_CONVEX_SECRET_CI OPENCLAW_QA_CONVEX_SITE_URL OPENCLAW_QA_CREDENTIAL_OWNER_ID OPENCLAW_QA_MANTIS_CRABBOX_COORDINATOR OPENCLAW_QA_MANTIS_CRABBOX_COORDINATOR_TOKEN"'
printf '%s\n' 'Defaults env_keep += "OPENCLAW_TELEGRAM_USER_CRABBOX_BIN OPENCLAW_TELEGRAM_USER_CRABBOX_PROVIDER OPENCLAW_TELEGRAM_USER_DRIVER_SCRIPT OPENCLAW_TELEGRAM_USER_PROOF_CMD"'
} | sudo tee /etc/sudoers.d/mantis-codex-env >/dev/null
@@ -462,7 +446,9 @@ jobs:
GH_TOKEN: ${{ github.token }}
MANTIS_CANDIDATE_TRUST: ${{ needs.validate_refs.outputs.candidate_trust }}
MANTIS_INSTRUCTIONS: ${{ needs.resolve_request.outputs.instructions }}
MANTIS_NODE_BIN: /usr/local/lib/mantis-toolchain/node
MANTIS_OUTPUT_DIR: ${{ env.MANTIS_OUTPUT_DIR }}
MANTIS_PNPM_BIN: /usr/local/lib/mantis-toolchain/pnpm
MANTIS_PR_NUMBER: ${{ needs.resolve_request.outputs.pr_number }}
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }}
@@ -483,12 +469,13 @@ jobs:
codex-home: /tmp/mantis-codex-home-${{ github.run_id }}
safety-strategy: unprivileged-user
codex-user: codex
allow-bot-users: clawsweeper[bot]
allow-bot-users: github-actions[bot]
- name: Release leaked Telegram proof leases
if: ${{ always() }}
env:
CRABBOX_PROVIDER: ${{ needs.resolve_request.outputs.crabbox_provider }}
MANTIS_NODE_BIN: /usr/local/lib/mantis-toolchain/node
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }}
shell: bash
@@ -500,7 +487,7 @@ jobs:
status=0
mapfile -d '' session_files < <(sudo find .artifacts/qa-e2e -name session.json -type f -print0)
for session_file in "${session_files[@]}"; do
if ! sudo -u codex node -e 'const fs = require("fs"); const session = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.exit(session.command === "telegram-user-crabbox-session" ? 0 : 1);' "$session_file"; then
if ! sudo -u codex "$MANTIS_NODE_BIN" -e 'const fs = require("fs"); const session = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.exit(session.command === "telegram-user-crabbox-session" ? 0 : 1);' "$session_file"; then
continue
fi
lease_file="${session_file%/session.json}/.session/lease.json"
@@ -512,20 +499,20 @@ jobs:
OPENCLAW_QA_CONVEX_SITE_URL="$OPENCLAW_QA_CONVEX_SITE_URL" \
OPENCLAW_TELEGRAM_USER_CRABBOX_BIN=/usr/local/bin/crabbox \
OPENCLAW_TELEGRAM_USER_CRABBOX_PROVIDER="$CRABBOX_PROVIDER" \
node --import tsx "$GITHUB_WORKSPACE/scripts/e2e/telegram-user-crabbox-proof.ts" \
"$MANTIS_NODE_BIN" --import tsx "$GITHUB_WORKSPACE/scripts/e2e/telegram-user-crabbox-proof.ts" \
finish --session "$session_file" --preview-crop telegram-window; then
status=1
fi
done
mapfile -d '' lease_files < <(sudo find .artifacts/qa-e2e -path '*/.session/lease.json' -type f -print0)
for lease_file in "${lease_files[@]}"; do
if ! sudo -u codex node -e 'const fs = require("fs"); const lease = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.exit(lease.kind === "telegram-user" ? 0 : 1);' "$lease_file"; then
if ! sudo -u codex "$MANTIS_NODE_BIN" -e 'const fs = require("fs"); const lease = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.exit(lease.kind === "telegram-user" ? 0 : 1);' "$lease_file"; then
continue
fi
if ! sudo -u codex env \
OPENCLAW_QA_CONVEX_SECRET_CI="$OPENCLAW_QA_CONVEX_SECRET_CI" \
OPENCLAW_QA_CONVEX_SITE_URL="$OPENCLAW_QA_CONVEX_SITE_URL" \
node --import tsx "$GITHUB_WORKSPACE/scripts/e2e/telegram-user-credential.ts" \
"$MANTIS_NODE_BIN" --import tsx "$GITHUB_WORKSPACE/scripts/e2e/telegram-user-credential.ts" \
release --lease-file "$lease_file"; then
status=1
fi
@@ -699,44 +686,3 @@ jobs:
--artifact-url "$PUBLISH_ARTIFACT_URL" \
--run-url "https://github.com/${GITHUB_REPOSITORY}/actions/runs/${PUBLISH_RUN_ID}" \
--request-source "$REQUEST_SOURCE"
clear_issue_comment_reaction:
name: Clear Mantis command reaction
needs: [resolve_request, validate_refs, run_telegram_desktop_proof]
if: ${{ always() && github.event_name == 'issue_comment' && needs.resolve_request.outputs.request_source == 'issue_comment' }}
runs-on: ubuntu-24.04
permissions:
issues: write
steps:
- name: Remove workflow eyes reaction
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
with:
script: |
const { owner, repo } = context.repo;
const commentId = context.payload.comment?.id;
if (!commentId) {
core.info("No issue comment id found; skipping reaction cleanup.");
return;
}
const reactions = await github.paginate(github.rest.reactions.listForIssueComment, {
owner,
repo,
comment_id: commentId,
per_page: 100,
});
const eyes = reactions.filter(
(reaction) => reaction.content === "eyes" && reaction.user?.login === "github-actions[bot]",
);
for (const reaction of eyes) {
await github.rest.reactions.deleteForIssueComment({
owner,
repo,
comment_id: commentId,
reaction_id: reaction.id,
});
core.info(`Removed eyes reaction ${reaction.id} from comment ${commentId}.`);
}
if (eyes.length === 0) {
core.info(`No workflow eyes reaction found on comment ${commentId}.`);
}
+45 -6
View File
@@ -1,8 +1,9 @@
// Mock OpenAI-compatible server for broader E2E scenarios.
import { createHash } from "node:crypto";
import http from "node:http";
import { setTimeout as delay } from "node:timers/promises";
import { escapeRegExp } from "../lib/regexp.mjs";
import { readTcpPortEnv } from "./lib/env-limits.mjs";
import { readPositiveIntEnv, readTcpPortEnv } from "./lib/env-limits.mjs";
import {
boundedRequestLogBody,
isRequestBodyTooLargeError,
@@ -18,8 +19,21 @@ const port =
: readTcpPortEnv("OPENCLAW_MOCK_OPENAI_PORT");
const successMarker = process.env.SUCCESS_MARKER ?? "OPENCLAW_E2E_OK";
const requestLog = process.env.MOCK_REQUEST_LOG;
const responseChunkDelayMs = process.env.MOCK_RESPONSE_CHUNK_DELAY_MS
? readPositiveIntEnv("MOCK_RESPONSE_CHUNK_DELAY_MS", undefined)
: 0;
function responseEvents(text) {
function splitResponseText(text) {
if (text.length < 2) {
return [text];
}
const midpoint = Math.floor(text.length / 2);
const whitespace = text.lastIndexOf(" ", midpoint);
const splitAt = whitespace > 0 ? whitespace : Math.max(1, midpoint);
return [text.slice(0, splitAt), text.slice(splitAt)];
}
function responseEvents(text, deltas = [text]) {
const itemId = "msg_e2e_1";
return [
{
@@ -32,13 +46,13 @@ function responseEvents(text) {
status: "in_progress",
},
},
{
...deltas.map((delta) => ({
type: "response.output_text.delta",
item_id: itemId,
output_index: 0,
content_index: 0,
delta: text,
},
delta,
})),
{
type: "response.output_text.done",
item_id: itemId,
@@ -81,6 +95,31 @@ function responseEvents(text) {
];
}
async function writeDefaultResponseEvents(res, text) {
if (responseChunkDelayMs === 0) {
writeSse(res, responseEvents(text));
return;
}
const events = responseEvents(text, splitResponseText(text));
res.writeHead(200, {
"content-type": "text/event-stream",
"cache-control": "no-store",
connection: "keep-alive",
});
let deltaCount = 0;
for (const event of events) {
if (event.type === "response.output_text.delta" && deltaCount > 0) {
await delay(responseChunkDelayMs);
}
res.write(`data: ${JSON.stringify(event)}\n\n`);
if (event.type === "response.output_text.delta") {
deltaCount += 1;
}
}
res.write("data: [DONE]\n\n");
res.end();
}
function buildMockFunctionCall(name, args) {
const serialized = JSON.stringify(args);
const suffix = createHash("sha256")
@@ -380,7 +419,7 @@ const server = http.createServer((req, res) => {
});
return;
}
writeSse(res, responseEvents(responseText));
await writeDefaultResponseEvents(res, responseText);
return;
}
+71 -6
View File
@@ -66,7 +66,9 @@ type Options = {
idleTimeout: string;
keepBox: boolean;
leaseId?: string;
linkPreview?: boolean;
mcpAppFixture: boolean;
mockResponseChunkDelayMs?: number;
mockResponseText: string;
mockPort: number;
outputDir: string;
@@ -92,6 +94,8 @@ type Options = {
timeoutMs: number;
ttl: string;
userDriverScript: string;
nodeBin: string;
pnpmBin?: string;
};
type FunnelBridge = {
@@ -208,7 +212,9 @@ function usageText() {
" --desktop-chat-title <name> Telegram Desktop chat to select before recording.",
" --id <cbx_id> Reuse an existing Crabbox desktop lease.",
" --keep-box Leave the Crabbox lease running for VNC debugging.",
" --link-preview <true|false> Set channels.telegram.linkPreview before Gateway startup.",
" --mock-response-file <path> Text returned by the mock model.",
" --mock-response-chunk-delay-ms <ms> Split the mock reply across two delayed deltas.",
" --mcp-app-fixture Configure the pinned MCP App fixture through a Crabbox Funnel.",
" --output-dir <path> Artifact directory under the repo.",
" --message-id <id> Telegram message id for proof-view deep link.",
@@ -283,6 +289,16 @@ function parseTcpPort(value: string, label: string) {
return parsed;
}
function parseBoolean(value: string, label: string) {
if (value === "true") {
return true;
}
if (value === "false") {
return false;
}
throw new Error(`${label} must be true or false.`);
}
function createTelegramProofRunId() {
return `${new Date().toISOString().replace(/[:.]/gu, "-")}-${randomUUID().slice(0, 8)}`;
}
@@ -332,6 +348,8 @@ export function parseArgs(argvInput: string[]): Options {
ttl: "120m",
userDriverScript:
trimToValue(process.env.OPENCLAW_TELEGRAM_USER_DRIVER_SCRIPT) ?? DEFAULT_USER_DRIVER,
nodeBin: trimToValue(process.env.MANTIS_NODE_BIN) ?? process.execPath,
pnpmBin: trimToValue(process.env.MANTIS_PNPM_BIN),
};
const commandSeparator = argv.indexOf("--");
if (command === "run" && commandSeparator >= 0) {
@@ -383,10 +401,17 @@ export function parseArgs(argvInput: string[]): Options {
opts.idleTimeout = readValue();
} else if (arg === "--keep-box") {
opts.keepBox = true;
} else if (arg === "--link-preview") {
opts.linkPreview = parseBoolean(readValue(), "--link-preview");
} else if (arg === "--mock-port") {
opts.mockPort = parseTcpPort(readValue(), "--mock-port");
} else if (arg === "--mock-response-file") {
opts.mockResponseText = fs.readFileSync(resolveRepoPath(process.cwd(), readValue()), "utf8");
} else if (arg === "--mock-response-chunk-delay-ms") {
opts.mockResponseChunkDelayMs = parsePositiveTimerMs(
readValue(),
"--mock-response-chunk-delay-ms",
);
} else if (arg === "--mcp-app-fixture") {
opts.mcpAppFixture = true;
} else if (arg === "--message-id") {
@@ -541,12 +566,20 @@ function childProcessBaseEnv() {
return env;
}
function mockServerEnv(params: { mockPort: number; mockResponseText: string; requestLog: string }) {
function mockServerEnv(params: {
mockPort: number;
mockResponseChunkDelayMs?: number;
mockResponseText: string;
requestLog: string;
}) {
return {
...childProcessBaseEnv(),
MOCK_PORT: String(params.mockPort),
MOCK_REQUEST_LOG: params.requestLog,
SUCCESS_MARKER: params.mockResponseText,
...(params.mockResponseChunkDelayMs === undefined
? {}
: { MOCK_RESPONSE_CHUNK_DELAY_MS: String(params.mockResponseChunkDelayMs) }),
};
}
@@ -577,8 +610,16 @@ export function createOpenClawGatewaySpawnSpec(params: {
comSpec?: string;
nodeExecPath?: string;
npmExecPath?: string;
pnpmExecPath?: string;
platform?: NodeJS.Platform;
}): GatewaySpawnSpec {
if (params.pnpmExecPath) {
return {
args: ["openclaw", "gateway", "--port", String(params.gatewayPort)],
command: params.pnpmExecPath,
options: { cwd: params.repoRoot, env: params.env, shell: false },
};
}
const spec = createPnpmRunnerSpawnSpec({
comSpec: params.comSpec,
cwd: params.repoRoot,
@@ -1137,6 +1178,7 @@ function telegramResultObject(value: unknown, label: string): JsonObject {
export function writeSutConfig(params: {
gatewayPort: number;
groupId: string;
linkPreview?: boolean;
mcpAppFixture?: boolean;
mockPort: number;
outputDir: string;
@@ -1185,6 +1227,7 @@ export function writeSutConfig(params: {
requireMention: false,
},
},
...(params.linkPreview === undefined ? {} : { linkPreview: params.linkPreview }),
replyToMode: "first",
},
},
@@ -1264,10 +1307,14 @@ export async function startLocalSut(
groupId: string;
mockResponseText: string;
mockPort: number;
linkPreview?: boolean;
mockResponseChunkDelayMs?: number;
outputDir: string;
sutToken: string;
testerId: string;
repoRoot: string;
nodeBin?: string;
pnpmBin?: string;
},
deps: StartLocalSutDeps = {},
) {
@@ -1282,10 +1329,14 @@ export async function startLocalSut(
const drained = await drainUpdates(params.sutToken);
const config = writeConfig(params);
const requestLog = path.join(params.outputDir, "mock-openai-requests.ndjson");
mock = spawnLoggedCommand("node", ["scripts/e2e/mock-openai-server.mjs"], {
cwd: params.repoRoot,
env: mockServerEnv({ ...params, requestLog }),
});
mock = spawnLoggedCommand(
params.nodeBin ?? process.execPath,
["scripts/e2e/mock-openai-server.mjs"],
{
cwd: params.repoRoot,
env: mockServerEnv({ ...params, requestLog }),
},
);
const runningMock = mock;
await waitForOutputReady(
runningMock.child,
@@ -1297,6 +1348,7 @@ export async function startLocalSut(
const gatewaySpec = createGatewaySpawnSpec({
env: gatewayEnv({ ...config, sutToken: params.sutToken }),
gatewayPort: params.gatewayPort,
pnpmExecPath: params.pnpmBin,
repoRoot: params.repoRoot,
});
gateway = spawnLoggedCommand(gatewaySpec.command, gatewaySpec.args, gatewaySpec.options);
@@ -1376,11 +1428,15 @@ async function startLocalSutDaemon(params: {
groupId: string;
mockResponseText: string;
mockPort: number;
linkPreview?: boolean;
mcpAppFixture?: boolean;
mockResponseChunkDelayMs?: number;
outputDir: string;
sutToken: string;
testerId: string;
repoRoot: string;
nodeBin?: string;
pnpmBin?: string;
}) {
const drained = await drainSutUpdates(params.sutToken);
const config = writeSutConfig(params);
@@ -1392,7 +1448,7 @@ async function startLocalSutDaemon(params: {
let gatewayPid: number | undefined;
try {
mockPid = spawnDaemon({
command: "node",
command: params.nodeBin ?? process.execPath,
args: ["scripts/e2e/mock-openai-server.mjs"],
cwd: params.repoRoot,
env: mockServerEnv({ ...params, requestLog }),
@@ -1414,6 +1470,7 @@ async function startLocalSutDaemon(params: {
const gatewaySpec = createOpenClawGatewaySpawnSpec({
env: gatewayEnvVars,
gatewayPort: params.gatewayPort,
pnpmExecPath: params.pnpmBin,
repoRoot: params.repoRoot,
});
gatewayPid = spawnDaemon({
@@ -2414,10 +2471,14 @@ async function startSession(root: string, opts: Options, outputDir: string) {
funnelBridge,
gatewayPort: opts.gatewayPort,
groupId: credential.groupId,
linkPreview: opts.linkPreview,
mockResponseText: opts.mockResponseText,
mockResponseChunkDelayMs: opts.mockResponseChunkDelayMs,
mockPort: opts.mockPort,
mcpAppFixture: opts.mcpAppFixture,
outputDir,
nodeBin: opts.nodeBin,
pnpmBin: opts.pnpmBin,
repoRoot: root,
sutToken: credential.sutToken,
testerId: credential.testerUserId,
@@ -2978,9 +3039,13 @@ async function main() {
const sutRuntime = await startLocalSut({
gatewayPort: opts.gatewayPort,
groupId: credential.groupId,
linkPreview: opts.linkPreview,
mockResponseText: opts.mockResponseText,
mockResponseChunkDelayMs: opts.mockResponseChunkDelayMs,
mockPort: opts.mockPort,
outputDir,
nodeBin: opts.nodeBin,
pnpmBin: opts.pnpmBin,
repoRoot: root,
sutToken: credential.sutToken,
testerId: credential.testerUserId,
@@ -19,6 +19,7 @@ const scrubbedEnvKeys = [
"FIXTURE_PORT",
"MOCK_PORT",
"MOCK_REQUEST_LOG",
"MOCK_RESPONSE_CHUNK_DELAY_MS",
"MOCK_TLS_CERT",
"MOCK_TLS_KEY",
"OPENCLAW_CONFIG_RELOAD_LOG_MAX_READ_BYTES",
@@ -159,6 +160,29 @@ describe("mock OpenAI response markers", () => {
});
});
it("can split a deterministic response across delayed streaming deltas", async () => {
await withMockServer(
mockOpenAiPath,
{
MOCK_RESPONSE_CHUNK_DELAY_MS: "80",
SUCCESS_MARKER: "First streamed preview remains visible before the follow-up edit arrives.",
},
async (baseUrl) => {
const startedAt = Date.now();
const response = await fetch(`${baseUrl}/v1/responses`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ input: "return the configured marker", stream: true }),
});
const body = await response.text();
expect(response.status).toBe(200);
expect(body.match(/response\.output_text\.delta/gu)).toHaveLength(2);
expect(Date.now() - startedAt).toBeGreaterThanOrEqual(60);
},
);
});
it("drives the MCP App fixture tool before returning the visible marker", async () => {
await withMockServer(mockOpenAiPath, {}, async (baseUrl) => {
const first = await fetch(`${baseUrl}/v1/responses`, {
@@ -34,9 +34,6 @@ type Workflow = {
env?: Record<string, string>;
jobs?: Record<string, WorkflowJob>;
on?: {
pull_request_target?: {
types?: string[];
};
workflow_dispatch?: {
inputs?: Record<
string,
@@ -172,6 +169,8 @@ describe("Mantis Telegram Desktop proof workflow", () => {
expect(cleanupStep.run).toContain("sudo -u codex env");
expect(cleanupStep.run).not.toContain("*/telegram-user-crabbox/*/session.json");
expect(cleanupStep.run).not.toContain("*/telegram-user-crabbox/*/.session/lease.json");
expect(cleanupStep.run).toContain('sudo -u codex "$MANTIS_NODE_BIN"');
expect(cleanupStep.run).not.toContain("sudo -u codex node");
});
it("cleans partially started proof daemons when local SUT startup fails", () => {
@@ -184,31 +183,17 @@ describe("Mantis Telegram Desktop proof workflow", () => {
expect(proofScript).toContain("throw error;");
});
it("uses the OpenClaw Mantis mention as the comment trigger", () => {
const workflow = readFileSync(WORKFLOW, "utf8");
const liveWorkflow = readFileSync(LIVE_WORKFLOW, "utf8");
expect(workflow).toContain("@openclaw-mantis");
expect(workflow).toContain("/openclaw-mantis");
expect(workflow).toContain("mantis: telegram-visible-proof");
expect(workflow).toContain('setOutput("should_run", "false")');
expect(workflow).toContain('normalized.includes("telegram desktop")');
expect(liveWorkflow).toContain('normalized.includes("telegram desktop")');
expect(liveWorkflow).toContain("!requestedDesktopProof");
expect(workflow).not.toContain("@Mantis");
expect(workflow).not.toContain("@mantis");
expect(workflow).not.toContain('"/mantis"');
});
it("runs when ClawSweeper applies the Telegram proof label", () => {
it("requires explicit maintainer dispatch before executing a PR worktree", () => {
const workflow = parse(readFileSync(WORKFLOW, "utf8")) as Workflow;
const workflowText = readFileSync(WORKFLOW, "utf8");
expect(workflow.on?.pull_request_target?.types).toContain("labeled");
expect(workflowText).toContain("github.event.label.name == 'mantis: telegram-visible-proof'");
expect(workflowText).toContain('eventName === "pull_request_target"');
expect(workflowText).toContain("context.payload.pull_request?.number");
expect(workflowText).toContain("Accepted Mantis label trigger");
expect(workflowText).toContain("allow-bot-users: clawsweeper[bot]");
expect(workflow.on?.workflow_dispatch).toBeDefined();
expect(workflowText).not.toContain("issue_comment:");
expect(workflowText).not.toContain("pull_request_target:");
expect(workflowText).not.toContain("clear_issue_comment_reaction:");
expect(workflowText).toContain("allow-bot-users: github-actions[bot]");
expect(workflowText).not.toContain("allow-bot-users: clawsweeper[bot]");
expect(workflowText).toContain('setOutput("request_source", "workflow_dispatch")');
});
it("can publish an existing proof artifact without recapturing", () => {
@@ -298,9 +283,13 @@ describe("Mantis Telegram Desktop proof workflow", () => {
it("installs local proof tools before the Codex agent runs", () => {
const install = workflowStep("Install local proof tools");
expect(install.run).toContain("test -f scripts/e2e/telegram-user-driver.py");
expect(install.run).toContain('node_bin="$(command -v node)"');
expect(install.run).toContain('corepack_bin="$(command -v corepack)"');
expect(install.run).toContain("/usr/local/lib/mantis-toolchain/node");
expect(install.run).toContain("/usr/local/lib/mantis-toolchain/pnpm");
expect(install.run).toContain("/usr/local/bin/openclaw-telegram-user-crabbox-proof");
expect(install.run).toContain(
'exec node --import tsx "${GITHUB_WORKSPACE}/scripts/e2e/telegram-user-crabbox-proof.ts" "$@"',
'exec /usr/local/lib/mantis-toolchain/node --import tsx "${GITHUB_WORKSPACE}/scripts/e2e/telegram-user-crabbox-proof.ts" "\\$@"',
);
expect(install.run).toContain("BtbN/FFmpeg-Builds");
expect(install.run).toContain("ffmpeg-master-latest-linux64-gpl.tar.xz");
@@ -316,6 +305,8 @@ describe("Mantis Telegram Desktop proof workflow", () => {
"/usr/local/bin/openclaw-telegram-user-crabbox-proof",
);
expect(agent.env?.OPENCLAW_TELEGRAM_USER_CRABBOX_BIN).toBe("/usr/local/bin/crabbox");
expect(agent.env?.MANTIS_NODE_BIN).toBe("/usr/local/lib/mantis-toolchain/node");
expect(agent.env?.MANTIS_PNPM_BIN).toBe("/usr/local/lib/mantis-toolchain/pnpm");
expect(agent.env?.CRABBOX_COORDINATOR).toContain(
"secrets.CRABBOX_COORDINATOR || secrets.OPENCLAW_QA_MANTIS_CRABBOX_COORDINATOR",
);
@@ -328,9 +319,13 @@ describe("Mantis Telegram Desktop proof workflow", () => {
"OPENCLAW_TELEGRAM_USER_CRABBOX_BIN OPENCLAW_TELEGRAM_USER_CRABBOX_PROVIDER OPENCLAW_TELEGRAM_USER_DRIVER_SCRIPT OPENCLAW_TELEGRAM_USER_PROOF_CMD",
);
expect(prepare.run).toContain("MANTIS_CANDIDATE_TRUST");
expect(prepare.run).toContain("MANTIS_NODE_BIN MANTIS_PNPM_BIN");
const prompt = readFileSync(PROMPT, "utf8");
expect(prompt).toContain("$OPENCLAW_TELEGRAM_USER_PROOF_CMD");
expect(prompt).toContain("`--link-preview false`");
expect(prompt).toContain("Do not edit the generated\n config or restart the Gateway");
expect(prompt).toContain("`--mock-response-chunk-delay-ms 1200`");
expect(prompt).toContain("do not run\n `pnpm qa:telegram-user:crabbox` directly");
expect(prompt).toContain("Let `start` return or fail on its\n own");
expect(prompt).toContain("`--mcp-app-fixture` option");
@@ -342,6 +337,27 @@ describe("Mantis Telegram Desktop proof workflow", () => {
);
});
it("prepares exact proof worktrees before agent secrets are available", () => {
const step = workflowStep("Prepare proof worktrees with pinned toolchain");
const run = step.run ?? "";
expect(run).toContain('git cat-file -e "${BASELINE_SHA}^{commit}"');
expect(run).toContain('git fetch --no-tags origin "pull/${MANTIS_PR_NUMBER}/head"');
expect(run).toContain('git worktree add --detach "$baseline_root" "$BASELINE_SHA"');
expect(run).toContain('git worktree add --detach "$candidate_root" "$CANDIDATE_SHA"');
expect(run.match(/env -i/gu)).toHaveLength(3);
expect(run).toContain('"$toolchain_dir/pnpm" install --frozen-lockfile');
expect(run).toContain('"$toolchain_dir/pnpm" build');
expect(run).toContain("sudo -u mantis-builder env -i");
expect(run).toContain('sudo chown -R mantis-builder:mantis-builder "$candidate_root"');
expect(run).toContain('test "$(cat "$candidate_root/.git")" = "$candidate_git_link"');
expect(run).toContain('git -C "$candidate_root" diff --exit-code');
expect(run).not.toContain("GH_TOKEN");
expect(run).not.toContain("OPENAI_API_KEY");
expect(run).not.toContain("CRABBOX_");
expect(run).not.toContain("OPENCLAW_QA_");
});
it("pins AWS Crabbox proof runs to the working region", () => {
const workflow = parse(readFileSync(WORKFLOW, "utf8")) as Workflow;
const liveWorkflow = parse(readFileSync(LIVE_WORKFLOW, "utf8")) as Workflow;
@@ -113,6 +113,20 @@ describe("telegram user Crabbox proof log polling", () => {
expect(spec.options.shell).toBe(false);
});
it("uses an explicitly pinned pnpm executable for a worktree gateway", () => {
const spec = createOpenClawGatewaySpawnSpec({
env: { PATH: "/definitely-missing" },
gatewayPort: 19042,
pnpmExecPath: "/opt/mantis-toolchain/pnpm",
repoRoot: "/repo",
});
expect(spec.command).toBe("/opt/mantis-toolchain/pnpm");
expect(spec.args).toEqual(["openclaw", "gateway", "--port", "19042"]);
expect(spec.options.cwd).toBe("/repo");
expect(spec.options.shell).toBe(false);
});
it("allows cold remote setup to outlive ordinary command timeouts", () => {
expect(REMOTE_SETUP_COMMAND_TIMEOUT_MS).toBeGreaterThan(COMMAND_TIMEOUT_MS);
expect(REMOTE_SETUP_COMMAND_TIMEOUT_MS).toBeGreaterThanOrEqual(90 * 60 * 1000);
@@ -166,6 +180,24 @@ describe("telegram user Crabbox proof log polling", () => {
expect(parseArgs(["--text", "-ping"]).text).toBe("-ping");
});
it("accepts an explicit Telegram link-preview setting", () => {
expect(parseArgs(["start", "--link-preview", "false"]).linkPreview).toBe(false);
expect(parseArgs(["start", "--link-preview", "true"]).linkPreview).toBe(true);
expect(parseArgs(["start"]).linkPreview).toBeUndefined();
expect(() => parseArgs(["start", "--link-preview", "disabled"])).toThrow(
"--link-preview must be true or false.",
);
});
it("accepts a positive mock response chunk delay", () => {
expect(
parseArgs(["start", "--mock-response-chunk-delay-ms", "1200"]).mockResponseChunkDelayMs,
).toBe(1200);
expect(() => parseArgs(["start", "--mock-response-chunk-delay-ms", "0"])).toThrow(
"--mock-response-chunk-delay-ms must be a positive integer.",
);
});
it("rejects duplicate single-value proof controls while keeping repeated expectations", () => {
expect(() =>
parseArgs(["--output-dir", ".artifacts/one", "--output-dir", ".artifacts/two"]),
@@ -241,6 +273,31 @@ describe("telegram user Crabbox proof log polling", () => {
expect(JSON.stringify(config)).not.toContain("resource-ok");
});
it("injects the requested Telegram link-preview setting before startup", () => {
const disabledConfigRoot = writeSutConfig({
gatewayPort: 19042,
groupId: "group",
linkPreview: false,
mockPort: 19043,
outputDir: makeTempDir(),
testerId: "tester",
});
const defaultConfigRoot = writeSutConfig({
gatewayPort: 19044,
groupId: "group",
mockPort: 19045,
outputDir: makeTempDir(),
testerId: "tester",
});
tempDirs.push(disabledConfigRoot.tempRoot, defaultConfigRoot.tempRoot);
const disabledConfig = JSON.parse(fs.readFileSync(disabledConfigRoot.configPath, "utf8"));
const defaultConfig = JSON.parse(fs.readFileSync(defaultConfigRoot.configPath, "utf8"));
expect(disabledConfig.channels.telegram.linkPreview).toBe(false);
expect(defaultConfig.channels.telegram).not.toHaveProperty("linkPreview");
});
it("pins the browser fixture SDK and exposes only the required app capabilities", () => {
const fixture = fs.readFileSync("scripts/e2e/mcp-app-conformance-server.mjs", "utf8");
const uiPackage = JSON.parse(fs.readFileSync("ui/package.json", "utf8"));