From f4cb54578397aa4cced6a0fa563774b1e71be018 Mon Sep 17 00:00:00 2001 From: Ayaan Zaidi Date: Sat, 22 Aug 2026 17:47:03 +0530 Subject: [PATCH] docs(mantis): script catalog-tool turns through Code Mode exec (#127878) * docs(mantis): script catalog-tool turns through Code Mode exec Run 32570733988 blocked both lanes with 'Tool pdf not found': the proof agent scripted a top-level pdf function_call, but the SUT agent runs Code Mode where catalog tools execute from JavaScript inside exec. Document the canonical route in the proof prompt, add a complete exec(pdf) scripted-event example plus the document-extract configPatch to the staged-media recipe, and pin the new guidance in the workflow test. * docs(mantis): wait for the exec round trip before finish ClawSweeper P2: send releases the mock response once the outbound message is visible, and finish tears the lane down, so a slow exec/pdf round trip could lose its function_call_output follow-up. Observe until the cumulative provider-request count covers the follow-up and assert the recorded function_call_output before finish; pin the contract in the workflow test. --- .../staged-media-provider-proof.md | 100 ++++++++++++++++-- .../prompts/mantis-telegram-desktop-proof.md | 3 + ...is-telegram-desktop-proof-workflow.test.ts | 12 +++ 3 files changed, 108 insertions(+), 7 deletions(-) diff --git a/.github/codex/prompts/mantis-recipes/staged-media-provider-proof.md b/.github/codex/prompts/mantis-recipes/staged-media-provider-proof.md index d74683a71414..17009b5a91f6 100644 --- a/.github/codex/prompts/mantis-recipes/staged-media-provider-proof.md +++ b/.github/codex/prompts/mantis-recipes/staged-media-provider-proof.md @@ -11,9 +11,11 @@ $lane observe --lane baseline --seconds 60 --until-provider-requests 1 requests="$($lane requests --lane baseline)" jq -e '[.requests[].contentFacts[]? | select(.type == "legacy_media")] | length > 0' \ <<<"$requests" -$lane finish --lane baseline --focus-message-id "$message_id" ``` +With no tool round trip, finish now using `message_id`. Otherwise continue below; +`finish` stops the lane. + For a reply-mention turn, first `send --media "$media"` without text, capture its `.sent.messageId`, then `send --reply-to "$message_id" --text '@{sut} inspect this document'`. A bare unmentioned upload stages the file but produces no provider turn. @@ -22,9 +24,93 @@ Repeat for `candidate` with its returned message id, selecting `type == "input_f Assert the complete selected facts: `filename`, `mimeType`, and `byteLength` when present. The structured facts are comparison evidence; never scrape `body` strings. -If baseline needs a tool round trip, take the tool argument from the recorded -`legacy_media.filename`. Build a complete response-events JSON array from -`toolCallEvents()` in `scripts/e2e/mock-openai-server.mjs` -(`response.output_item.added`, `response.function_call_arguments.delta`, -`response.output_item.done`, `response.completed`), then install it before the -next turn with `mock --lane baseline --response-events-file `. +For a PDF tool round trip, start each lane with this patch. `pdf` is already in +the Code Mode catalog; `document-extract` lets the mock OpenAI route execute it. + +```json +{ "configPatch": { "plugins": { "allow": ["telegram", "openai", "document-extract"] } } } +``` + +Replace `` below with the recorded value and save the +array as `pdf-exec-events.json` under `MANTIS_OUTPUT_DIR`: + +```json +[ + { + "type": "response.output_item.added", + "item": { + "type": "function_call", + "id": "fc_mantis_pdf_exec", + "call_id": "call_mantis_pdf_exec", + "name": "exec", + "arguments": "" + } + }, + { + "type": "response.function_call_arguments.delta", + "delta": "{\"language\":\"javascript\",\"code\":\"return await pdf({ pdf: \\\"\\\", prompt: \\\"Inspect this PDF.\\\" });\"}" + }, + { + "type": "response.output_item.done", + "item": { + "type": "function_call", + "id": "fc_mantis_pdf_exec", + "call_id": "call_mantis_pdf_exec", + "name": "exec", + "arguments": "{\"language\":\"javascript\",\"code\":\"return await pdf({ pdf: \\\"\\\", prompt: \\\"Inspect this PDF.\\\" });\"}" + } + }, + { + "type": "response.completed", + "response": { + "id": "resp_mantis_pdf_exec", + "status": "completed", + "output": [ + { + "type": "function_call", + "id": "fc_mantis_pdf_exec", + "call_id": "call_mantis_pdf_exec", + "name": "exec", + "arguments": "{\"language\":\"javascript\",\"code\":\"return await pdf({ pdf: \\\"\\\", prompt: \\\"Inspect this PDF.\\\" });\"}" + } + ], + "usage": { + "input_tokens": 64, + "output_tokens": 16, + "total_tokens": 80, + "input_tokens_details": { "cached_tokens": 0 } + } + } + } +] +``` + +Save this beside it as `pdf-exec-script.json`, install the two-response script, +then send the tool-driven turn: + +```json +{ + "responses": [ + { "eventsFile": "pdf-exec-events.json" }, + { "text": "PDF tool round trip complete." } + ] +} +``` + +```bash +script="$MANTIS_OUTPUT_DIR/pdf-exec-script.json" +sha256="$(sha256sum "$script" | cut -d ' ' -f 1)" +$lane mock --lane baseline --script "$script" "$sha256" +tool_sent="$($lane send --lane baseline --text '@{sut} inspect the staged PDF with the pdf tool')" +tool_message_id="$(jq -er '.sent.messageId' <<<"$tool_sent")" +$lane observe --lane baseline --seconds 120 --until-provider-requests 3 +requests="$($lane requests --lane baseline)" +jq -e '[.requests[] | .body.input[]? | select(.type == "function_call_output" + and .call_id == "call_mantis_pdf_exec")] | length > 0' <<<"$requests" +$lane finish --lane baseline --focus-message-id "$tool_message_id" +``` + +`finish` tears the lane down, so wait for the cumulative provider-request count +(staging turn, exec turn, follow-up) and assert the recorded +`function_call_output` before finishing; its `output` carries the serialized +exec result. Repeat the same script, turn, wait, and assertions for `candidate`. diff --git a/.github/codex/prompts/mantis-telegram-desktop-proof.md b/.github/codex/prompts/mantis-telegram-desktop-proof.md index 3c8442bf7693..bd1fdc923397 100644 --- a/.github/codex/prompts/mantis-telegram-desktop-proof.md +++ b/.github/codex/prompts/mantis-telegram-desktop-proof.md @@ -106,6 +106,9 @@ from `responseEvents` in `scripts/e2e/mock-openai-server.mjs`, and use `packages/ai/src/transports/openai-responses-stream-parity.test.ts` for reasoning event examples. These harness sources are safe to read; prepared proof worktrees remain off limits. +The SUT agent runs Code Mode. Script catalog-tool turns as an `exec` function +call whose JavaScript invokes the catalog tool, such as `pdf(...)`. See +`mantis-recipes/staged-media-provider-proof.md` for the complete event script. For normal group turns, address the current bot with `@{sut}`; the harness expands it to the live SUT username. Omit it only when an unmentioned message is intentionally part of the scenario. diff --git a/test/scripts/mantis-telegram-desktop-proof-workflow.test.ts b/test/scripts/mantis-telegram-desktop-proof-workflow.test.ts index f1dac487680a..3a71f81846c8 100644 --- a/test/scripts/mantis-telegram-desktop-proof-workflow.test.ts +++ b/test/scripts/mantis-telegram-desktop-proof-workflow.test.ts @@ -781,6 +781,18 @@ describe("Mantis Telegram Desktop proof workflow", () => { expect(prompt).toContain("not who sent it"); expect(prompt).not.toContain("trusted, tamper-protected"); expect(prompt).not.toContain("Provider request logs are diagnostic and pacing signals"); + expect(prompt).toContain("Script catalog-tool turns as an `exec` function"); + // The exec/pdf round trip outlives `send`; the recipe must wait for the + // follow-up function_call_output request before `finish` tears the lane down. + const stagedMediaRecipe = readFileSync( + ".github/codex/prompts/mantis-recipes/staged-media-provider-proof.md", + "utf8", + ); + expect(stagedMediaRecipe).toContain("--until-provider-requests 3"); + expect(stagedMediaRecipe).toContain('select(.type == "function_call_output"'); + expect(stagedMediaRecipe.indexOf("--until-provider-requests 3")).toBeLessThan( + stagedMediaRecipe.lastIndexOf("finish --lane baseline"), + ); expect(prompt).toContain("mantis-recipes/"); expect(prompt).toContain("recipe-suggestion.md"); expect(prompt).toContain("do not call `finish` and describe the block only in prose");