feat(e2e): record provider media content facts for Mantis proofs (#127830)

* feat(e2e): record provider media content facts

* docs(mantis): list staged-media recipe in recipe index

* fix(mantis): isolate mock provider evidence

* fix(mantis): state provider facts as tamper-evident, not provenance-trusted

The mock sidecar makes provider request facts tamper-evident: candidate
code cannot alter or remove a recorded request. It cannot make them
provenance-authenticated — every process in the SUT container shares the
internal network, UID, and gateway config, so nothing distinguishes the
gateway flow from any other candidate-originated request. The proof
prompt and busy-queue recipe now state exactly that property instead of
calling the facts trusted.

* fix(mantis): redact unparseable provider request bodies

Media redaction walks the parsed JSON body, so a request that fails
JSON.parse fell back to logging the raw text — leaking base64 payloads
the redactor exists to strip. Unparseable bodies now log a bounded
byte-count marker instead; regression posts a malformed body carrying a
data URL and asserts the payload never reaches the record.

* fix(mantis): expose newest provider records through a seq-stamped tail

The lane's requests surface kept the first 100 provider records, so a
session longer than the window hid exactly the newest requests a proof
asserts on. The mock server now stamps each record with a producer-owned
absolute seq ordinal, and the lane reads a bounded 128-record tail —
mirroring the sibling botApiRequests window. Regression writes 130
records and asserts the tail keeps seq 3..130; it fails pre-fix.
This commit is contained in:
Ayaan Zaidi
2026-08-22 16:30:30 +05:30
committed by GitHub
parent da0cb592dd
commit d9d44981fb
14 changed files with 507 additions and 84 deletions
+106
View File
@@ -17,6 +17,7 @@ const scrubbedEnvKeys = [
"CLICKCLACK_FIXTURE_PORT",
"CLICKCLACK_FIXTURE_REQUEST_MAX_BYTES",
"FIXTURE_PORT",
"MOCK_BIND_HOST",
"MOCK_PORT",
"MOCK_REQUEST_LOG",
"MOCK_RESPONSE_CHUNK_DELAY_MS",
@@ -370,6 +371,111 @@ describe("mock OpenAI response markers", () => {
}
});
it("records bounded media facts without provider payload bytes", async () => {
const root = await mkdtemp(join(tmpdir(), "openclaw-mock-content-facts-"));
const requestLog = join(root, "requests.ndjson");
const pdfBytes = "private-pdf-bytes";
const pdfBase64 = Buffer.from(pdfBytes).toString("base64");
try {
await writeFile(requestLog, "");
await withMockServer(mockOpenAiPath, { MOCK_REQUEST_LOG: requestLog }, async (baseUrl) => {
const send = async (input: unknown) => {
const response = await fetch(`${baseUrl}/v1/responses`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ input, stream: false }),
});
expect(response.status).toBe(200);
};
await send([
{
type: "message",
role: "user",
content: Array.from({ length: 128 }, (_, index) => ({
type: "input_text",
text: `historical turn ${index}`,
})),
},
{
type: "message",
role: "user",
content: [
{
type: "input_file",
filename: "proof.pdf",
file_data: `data:application/pdf;base64,${pdfBase64}`,
},
{ type: "input_text", text: "Summarize the staged document." },
],
},
]);
await send([
{
type: "message",
role: "user",
content: [
{
type: "input_text",
text: "[media attached: /tmp/session/proof.pdf (application/pdf)]\nSummarize it.",
},
],
},
]);
const recorded = await readFile(requestLog, "utf8");
const entries = recorded
.trim()
.split("\n")
.map((line) => JSON.parse(line));
expect(entries[0]?.contentFacts).toHaveLength(128);
expect(entries[0]?.contentFactsTruncated).toBe(true);
expect(entries[0]?.contentFacts.slice(-2)).toEqual([
{
type: "input_file",
filename: "proof.pdf",
mimeType: "application/pdf",
byteLength: Buffer.byteLength(pdfBytes),
},
{ type: "input_text" },
]);
expect(entries[1]?.contentFacts).toEqual([
{ type: "input_text" },
{
type: "legacy_media",
filename: "/tmp/session/proof.pdf",
mimeType: "application/pdf",
},
]);
expect(recorded).not.toContain(pdfBase64);
expect(entries[0]?.body).toContain("data:application/pdf;base64,[redacted:17 bytes]");
expect(entries.map((entry) => entry.seq)).toEqual([1, 2]);
// Redaction walks parsed JSON, so an unparseable body must never be
// logged as raw text — that path would leak the base64 payload.
const malformed = `{"input": "data:application/pdf;base64,${pdfBase64}"`;
const response = await fetch(`${baseUrl}/v1/responses`, {
method: "POST",
headers: { "content-type": "application/json" },
body: malformed,
});
expect(response.status).toBe(200);
const withMalformed = await readFile(requestLog, "utf8");
expect(withMalformed).not.toContain(pdfBase64);
const malformedEntry = withMalformed
.trim()
.split("\n")
.map((line) => JSON.parse(line))
.at(-1);
expect(malformedEntry?.body).toBe(
`[unparseable request body redacted: ${Buffer.byteLength(malformed)} bytes]`,
);
expect(malformedEntry?.seq).toBe(3);
});
} finally {
await rm(root, { force: true, recursive: true });
}
});
it("supports scripted connection drops", async () => {
const root = await mkdtemp(join(tmpdir(), "openclaw-mock-response-drop-"));
const control = join(root, "response.json");
@@ -5,6 +5,7 @@ import { parse } from "yaml";
const PROOF_SCRIPT = "scripts/e2e/telegram-user-crabbox-proof.ts";
const MANTIS_SUT_SCRIPT = "scripts/e2e/telegram-mantis-sut.ts";
const MOCK_OPENAI_SERVER = "scripts/e2e/mock-openai-server.mjs";
const MANTIS_LANE_SCRIPT = "scripts/e2e/telegram-mantis-lane.ts";
const DESKTOP_CRABBOX_SCRIPT = "scripts/e2e/telegram-desktop-crabbox.ts";
const SUT_CONTAINER_WRAPPER = "scripts/mantis/mantis-sut-container.sh";
@@ -773,6 +774,13 @@ describe("Mantis Telegram Desktop proof workflow", () => {
expect(prompt).toContain("`requests`");
expect(prompt).toContain("`finish [--focus-message-id ID]`");
expect(prompt).toContain("Identical pixels alone do not force `block`");
// Precise trust claim: the sidecar makes facts tamper-evident (candidate
// cannot rewrite records), but requests originate inside the untrusted SUT,
// so the prompt must not present them as provenance-authenticated.
expect(prompt).toContain("Provider request facts are tamper-evident comparison evidence");
expect(prompt).toContain("not who sent it");
expect(prompt).not.toContain("trusted, tamper-protected");
expect(prompt).not.toContain("Provider request logs are diagnostic and pacing signals");
expect(prompt).toContain("mantis-recipes/");
expect(prompt).toContain("recipe-suggestion.md");
expect(prompt).toContain("do not call `finish` and describe the block only in prose");
@@ -1157,6 +1165,7 @@ describe("Mantis Telegram Desktop proof workflow", () => {
it("does not pass the full workflow environment into the local Telegram SUT", () => {
const sutScript = readFileSync(MANTIS_SUT_SCRIPT, "utf8");
const laneScript = readFileSync(MANTIS_LANE_SCRIPT, "utf8");
const mockServer = readFileSync(MOCK_OPENAI_SERVER, "utf8");
const prompt = readFileSync(PROMPT, "utf8");
const workflow = readFileSync(WORKFLOW, "utf8");
const wrapper = readFileSync(SUT_CONTAINER_WRAPPER, "utf8");
@@ -1228,10 +1237,7 @@ describe("Mantis Telegram Desktop proof workflow", () => {
expect(workflow).toContain(
'sudo install -m 0444 "$toolchain_build/scripts/e2e/mock-openai-server.mjs"',
);
expect(wrapper).toContain("node /opt/mantis/mock-openai-server.mjs");
expect(wrapper).toContain(
'--mount "type=bind,src=$mock_server_script,dst=/opt/mantis/mock-openai-server.mjs,readonly"',
);
expect(wrapper).not.toContain('node /opt/mantis/mock-openai-server.mjs >"$MOCK_LOG"');
expect(wrapper).not.toContain("node scripts/e2e/mock-openai-server.mjs");
expect(workflow).toContain('sudo usermod -aG mantis-proof "$recorder_user"');
expect(workflow).toContain(
@@ -1284,6 +1290,25 @@ describe("Mantis Telegram Desktop proof workflow", () => {
expect(wrapper).toContain(
"--env TELEGRAM_PROXY_RECORD_FILE=/opt/mantis/proxy-control/requests.ndjson",
);
const mockContainerSpec = wrapper.slice(
wrapper.indexOf('"$docker_bin" run --detach --name "$mock_container_name"'),
wrapper.indexOf('wait_for_mock_openai "$mock_container_name"'),
);
expect(mockContainerSpec).toContain('--network "$network_name"');
expect(mockContainerSpec).toContain("--network-alias mock-openai");
expect(mockContainerSpec).not.toContain("$egress_network_name");
expect(mockContainerSpec).toContain(
'--mount "type=bind,src=$mock_server_script,dst=/opt/mantis/mock-openai-server.mjs,readonly"',
);
expect(mockContainerSpec).toContain(
'--mount "type=bind,src=$response_control_dir,dst=/opt/mantis/mock-control"',
);
expect(wrapper).toContain("--env MOCK_BIND_HOST=0.0.0.0");
expect(mockContainerSpec).toContain('--user "$(id -u mantis-sut):$(id -g mantis-sut)"');
expect(mockServer).toContain('const bindHost = process.env.MOCK_BIND_HOST ?? "127.0.0.1"');
expect(mockServer).toContain("server.listen(port, bindHost");
expect(wrapper).toContain('wait_for_mock_openai "$mock_container_name" "$mock_log"');
expect(wrapper).toContain("mock OpenAI container exited before readiness");
// Candidate code shares the mantis-sut UID with the proxy record sink, so
// the SUT container must shadow proxy-control; otherwise the lane under
// test could rewrite its own trusted Bot API evidence before publication.
@@ -1293,8 +1318,16 @@ describe("Mantis Telegram Desktop proof workflow", () => {
expect(wrapper.indexOf(proxyControlShadow)).toBeGreaterThan(
wrapper.indexOf('--mount "type=bind,src=$safe_runtime,dst=$runtime_source"'),
);
const mockControlShadow =
'--mount "type=tmpfs,dst=$runtime_source/mock-control,tmpfs-size=65536,tmpfs-mode=0000"';
expect(wrapper).toContain(mockControlShadow);
expect(wrapper.indexOf(mockControlShadow)).toBeGreaterThan(
wrapper.indexOf('--mount "type=bind,src=$safe_runtime,dst=$runtime_source"'),
);
expect(wrapper).toContain('export TELEGRAM_BOT_TOKEN="$telegram_alias_token"');
expect(wrapper).not.toContain('export TELEGRAM_BOT_TOKEN="$telegram_bot_token"');
expect(wrapper.match(/remove_container_or_fail "\$mock_container_name"/gu)).toHaveLength(2);
expect(wrapper).toContain('remove_container_or_fail "${1}-mock-openai"');
expect(wrapper).toContain('remove_container_or_fail "${1}-telegram-proxy"');
expect(workflow).toContain(
"/usr/local/lib/mantis-toolchain/scripts/e2e/telegram-bot-api-proxy.mjs",
@@ -1338,17 +1371,21 @@ describe("Mantis Telegram Desktop proof workflow", () => {
'const proxyControlDir = path.join(config.tempRoot, "proxy-control")',
);
expect(sutScript).toContain(
'const requestLog = path.join(config.tempRoot, "mock-openai-requests.ndjson")',
'const requestLog = path.join(mockResponseControlDir, "mock-openai-requests.ndjson")',
);
expect(wrapper).toContain(
'export MOCK_RESPONSE_CONTROL="$runtime_source/mock-control/response.json"',
expect(sutScript).toContain(
'const mockLog = path.join(mockResponseControlDir, "mock-openai.log")',
);
const forwardedEnv = wrapper.slice(
wrapper.indexOf("forwarded_env=("),
wrapper.indexOf("docker_env=()"),
);
expect(forwardedEnv).toContain("MOCK_RESPONSE_CONTROL");
expect(forwardedEnv).not.toContain("MOCK_RESPONSE_CONTROL");
expect(forwardedEnv).not.toContain("MOCK_REQUEST_LOG");
expect(forwardedEnv).not.toContain("MOCK_LOG");
expect(forwardedEnv).not.toContain("MOCK_PORT");
expect(wrapper).toContain("refusing to destroy a running SUT container");
expect(wrapper).toContain("refusing to destroy a running mock OpenAI container");
expect(wrapper).toContain('destroy_bounded_filesystem "$runtime_root"');
expect(wrapper).toContain('create_runtime_claim "$container_name" "$runtime_source"');
expect(wrapper).toContain('cancel_runtime_claim "$1" "$runtime_source"');
+77
View File
@@ -922,6 +922,83 @@ exit 1
}
});
it("exposes provider content facts through requests and terminal lane facts", async () => {
const harness = await setupHarness({ userOnlyEvents: true });
const contentFacts = [
{
type: "input_file",
filename: "proof.pdf",
mimeType: "application/pdf",
byteLength: 17,
},
];
fs.writeFileSync(
harness.requestLog,
`${JSON.stringify({
seq: 1,
body: "credential=123456:secret-sut-token",
contentFacts,
path: "/v1/responses",
})}\n`,
);
try {
const requests = JSON.parse(
(await runLane(harness.env, ["requests", "--lane", "candidate"])).stdout,
);
expect(requests).toEqual({
count: 1,
requests: [
{
seq: 1,
body: "credential=[redacted]",
contentFacts,
path: "/v1/responses",
},
],
});
// Tail window: a session with more records than the window must expose
// its newest requests — the ones under proof — with their absolute seq.
fs.writeFileSync(
harness.requestLog,
Array.from(
{ length: 130 },
(_, i) => `${JSON.stringify({ seq: i + 1, body: `turn ${i + 1}` })}\n`,
).join(""),
);
const tail = JSON.parse(
(await runLane(harness.env, ["requests", "--lane", "candidate"])).stdout,
);
expect(tail.count).toBe(128);
expect(tail.requests[0]).toEqual({ seq: 3, body: "turn 3" });
expect(tail.requests.at(-1)).toEqual({ seq: 130, body: "turn 130" });
// Restore the single-record log so terminal lane facts mirror the
// requests assertion above.
fs.writeFileSync(
harness.requestLog,
`${JSON.stringify({
seq: 1,
body: "credential=123456:secret-sut-token",
contentFacts,
path: "/v1/responses",
})}\n`,
);
await runLane(harness.env, ["send", "--lane", "candidate", "--text", "persist facts"]);
await runLane(harness.env, ["finish", "--lane", "candidate"]);
const facts = JSON.parse(
fs.readFileSync(
path.join(harness.outputRoot, "candidate", "mantis-lane-facts.json"),
"utf8",
),
);
expect(facts.providerRequests).toEqual(requests.requests);
expect(JSON.stringify(facts.providerRequests)).not.toContain("secret-sut-token");
} finally {
await harness.close();
}
});
it("finishes an expected-silence proof on the triggering user message", async () => {
const harness = await setupHarness({ userOnlyEvents: true });
try {
+2
View File
@@ -201,6 +201,7 @@ describe("Telegram Mantis SUT", () => {
},
gatewayPort: 19_879,
groupId: "-100123456789",
mockHost: "mock-openai",
mockPort: 19_882,
outputDir,
testerId: "12345",
@@ -212,6 +213,7 @@ describe("Telegram Mantis SUT", () => {
expect(config.channels.telegram.streaming).toEqual({ mode: "partial" });
expect(config.channels.telegram).not.toHaveProperty("replyToMode");
expect(config.commands.ownerAllowFrom).toEqual(["telegram:12345"]);
expect(config.models.providers.openai.baseUrl).toBe("http://mock-openai:19882/v1");
expect(config.session.sendPolicy).toEqual({ default: "deny" });
});
});
@@ -493,6 +493,7 @@ describe("telegram user Crabbox proof log polling", () => {
gatewayPort: 19042,
groupId: "group",
mcpAppFixture: true,
mockHost: "127.0.0.1",
mockPort: 19043,
outputDir: makeTempDir(tempDirs, "openclaw-telegram-proof-"),
repoRoot: "/repo",
@@ -520,6 +521,7 @@ describe("telegram user Crabbox proof log polling", () => {
const configRoot = writeSutConfig({
gatewayPort: 19042,
groupId: "group",
mockHost: "127.0.0.1",
mockPort: 19043,
outputDir: makeTempDir(tempDirs, "openclaw-telegram-proof-"),
testerId: "tester",
@@ -532,6 +534,7 @@ describe("telegram user Crabbox proof log polling", () => {
executionIdentity: true,
messages: "direct",
});
expect(config.models.providers.openai.baseUrl).toBe("http://127.0.0.1:19043/v1");
});
it("injects the requested Telegram link-preview setting before startup", () => {
@@ -539,6 +542,7 @@ describe("telegram user Crabbox proof log polling", () => {
configPatch: { channels: { telegram: { linkPreview: false } } },
gatewayPort: 19042,
groupId: "group",
mockHost: "127.0.0.1",
mockPort: 19043,
outputDir: makeTempDir(tempDirs, "openclaw-telegram-proof-"),
testerId: "tester",
@@ -546,6 +550,7 @@ describe("telegram user Crabbox proof log polling", () => {
const defaultConfigRoot = writeSutConfig({
gatewayPort: 19044,
groupId: "group",
mockHost: "127.0.0.1",
mockPort: 19045,
outputDir: makeTempDir(tempDirs, "openclaw-telegram-proof-"),
testerId: "tester",
@@ -570,6 +575,7 @@ describe("telegram user Crabbox proof log polling", () => {
},
gatewayPort: 19042,
groupId: "group",
mockHost: "127.0.0.1",
mockPort: 19043,
outputDir: makeTempDir(tempDirs, "openclaw-telegram-proof-"),
testerId: "tester",
@@ -577,6 +583,7 @@ describe("telegram user Crabbox proof log polling", () => {
const defaultConfigRoot = writeSutConfig({
gatewayPort: 19044,
groupId: "group",
mockHost: "127.0.0.1",
mockPort: 19045,
outputDir: makeTempDir(tempDirs, "openclaw-telegram-proof-"),
testerId: "tester",