mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-28 13:26:04 -06:00
feat(e2e): record provider media content facts for Mantis proofs (#127830)
* feat(e2e): record provider media content facts * docs(mantis): list staged-media recipe in recipe index * fix(mantis): isolate mock provider evidence * fix(mantis): state provider facts as tamper-evident, not provenance-trusted The mock sidecar makes provider request facts tamper-evident: candidate code cannot alter or remove a recorded request. It cannot make them provenance-authenticated — every process in the SUT container shares the internal network, UID, and gateway config, so nothing distinguishes the gateway flow from any other candidate-originated request. The proof prompt and busy-queue recipe now state exactly that property instead of calling the facts trusted. * fix(mantis): redact unparseable provider request bodies Media redaction walks the parsed JSON body, so a request that fails JSON.parse fell back to logging the raw text — leaking base64 payloads the redactor exists to strip. Unparseable bodies now log a bounded byte-count marker instead; regression posts a malformed body carrying a data URL and asserts the payload never reaches the record. * fix(mantis): expose newest provider records through a seq-stamped tail The lane's requests surface kept the first 100 provider records, so a session longer than the window hid exactly the newest requests a proof asserts on. The mock server now stamps each record with a producer-owned absolute seq ordinal, and the lane reads a bounded 128-record tail — mirroring the sibling botApiRequests window. Regression writes 130 records and asserts the tail keeps seq 3..130; it fails pre-fix.
This commit is contained in:
@@ -17,6 +17,7 @@ const scrubbedEnvKeys = [
|
||||
"CLICKCLACK_FIXTURE_PORT",
|
||||
"CLICKCLACK_FIXTURE_REQUEST_MAX_BYTES",
|
||||
"FIXTURE_PORT",
|
||||
"MOCK_BIND_HOST",
|
||||
"MOCK_PORT",
|
||||
"MOCK_REQUEST_LOG",
|
||||
"MOCK_RESPONSE_CHUNK_DELAY_MS",
|
||||
@@ -370,6 +371,111 @@ describe("mock OpenAI response markers", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("records bounded media facts without provider payload bytes", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "openclaw-mock-content-facts-"));
|
||||
const requestLog = join(root, "requests.ndjson");
|
||||
const pdfBytes = "private-pdf-bytes";
|
||||
const pdfBase64 = Buffer.from(pdfBytes).toString("base64");
|
||||
try {
|
||||
await writeFile(requestLog, "");
|
||||
await withMockServer(mockOpenAiPath, { MOCK_REQUEST_LOG: requestLog }, async (baseUrl) => {
|
||||
const send = async (input: unknown) => {
|
||||
const response = await fetch(`${baseUrl}/v1/responses`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ input, stream: false }),
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
};
|
||||
await send([
|
||||
{
|
||||
type: "message",
|
||||
role: "user",
|
||||
content: Array.from({ length: 128 }, (_, index) => ({
|
||||
type: "input_text",
|
||||
text: `historical turn ${index}`,
|
||||
})),
|
||||
},
|
||||
{
|
||||
type: "message",
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "input_file",
|
||||
filename: "proof.pdf",
|
||||
file_data: `data:application/pdf;base64,${pdfBase64}`,
|
||||
},
|
||||
{ type: "input_text", text: "Summarize the staged document." },
|
||||
],
|
||||
},
|
||||
]);
|
||||
await send([
|
||||
{
|
||||
type: "message",
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "input_text",
|
||||
text: "[media attached: /tmp/session/proof.pdf (application/pdf)]\nSummarize it.",
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
|
||||
const recorded = await readFile(requestLog, "utf8");
|
||||
const entries = recorded
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line));
|
||||
expect(entries[0]?.contentFacts).toHaveLength(128);
|
||||
expect(entries[0]?.contentFactsTruncated).toBe(true);
|
||||
expect(entries[0]?.contentFacts.slice(-2)).toEqual([
|
||||
{
|
||||
type: "input_file",
|
||||
filename: "proof.pdf",
|
||||
mimeType: "application/pdf",
|
||||
byteLength: Buffer.byteLength(pdfBytes),
|
||||
},
|
||||
{ type: "input_text" },
|
||||
]);
|
||||
expect(entries[1]?.contentFacts).toEqual([
|
||||
{ type: "input_text" },
|
||||
{
|
||||
type: "legacy_media",
|
||||
filename: "/tmp/session/proof.pdf",
|
||||
mimeType: "application/pdf",
|
||||
},
|
||||
]);
|
||||
expect(recorded).not.toContain(pdfBase64);
|
||||
expect(entries[0]?.body).toContain("data:application/pdf;base64,[redacted:17 bytes]");
|
||||
expect(entries.map((entry) => entry.seq)).toEqual([1, 2]);
|
||||
|
||||
// Redaction walks parsed JSON, so an unparseable body must never be
|
||||
// logged as raw text — that path would leak the base64 payload.
|
||||
const malformed = `{"input": "data:application/pdf;base64,${pdfBase64}"`;
|
||||
const response = await fetch(`${baseUrl}/v1/responses`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: malformed,
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
const withMalformed = await readFile(requestLog, "utf8");
|
||||
expect(withMalformed).not.toContain(pdfBase64);
|
||||
const malformedEntry = withMalformed
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line))
|
||||
.at(-1);
|
||||
expect(malformedEntry?.body).toBe(
|
||||
`[unparseable request body redacted: ${Buffer.byteLength(malformed)} bytes]`,
|
||||
);
|
||||
expect(malformedEntry?.seq).toBe(3);
|
||||
});
|
||||
} finally {
|
||||
await rm(root, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("supports scripted connection drops", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "openclaw-mock-response-drop-"));
|
||||
const control = join(root, "response.json");
|
||||
|
||||
@@ -5,6 +5,7 @@ import { parse } from "yaml";
|
||||
|
||||
const PROOF_SCRIPT = "scripts/e2e/telegram-user-crabbox-proof.ts";
|
||||
const MANTIS_SUT_SCRIPT = "scripts/e2e/telegram-mantis-sut.ts";
|
||||
const MOCK_OPENAI_SERVER = "scripts/e2e/mock-openai-server.mjs";
|
||||
const MANTIS_LANE_SCRIPT = "scripts/e2e/telegram-mantis-lane.ts";
|
||||
const DESKTOP_CRABBOX_SCRIPT = "scripts/e2e/telegram-desktop-crabbox.ts";
|
||||
const SUT_CONTAINER_WRAPPER = "scripts/mantis/mantis-sut-container.sh";
|
||||
@@ -773,6 +774,13 @@ describe("Mantis Telegram Desktop proof workflow", () => {
|
||||
expect(prompt).toContain("`requests`");
|
||||
expect(prompt).toContain("`finish [--focus-message-id ID]`");
|
||||
expect(prompt).toContain("Identical pixels alone do not force `block`");
|
||||
// Precise trust claim: the sidecar makes facts tamper-evident (candidate
|
||||
// cannot rewrite records), but requests originate inside the untrusted SUT,
|
||||
// so the prompt must not present them as provenance-authenticated.
|
||||
expect(prompt).toContain("Provider request facts are tamper-evident comparison evidence");
|
||||
expect(prompt).toContain("not who sent it");
|
||||
expect(prompt).not.toContain("trusted, tamper-protected");
|
||||
expect(prompt).not.toContain("Provider request logs are diagnostic and pacing signals");
|
||||
expect(prompt).toContain("mantis-recipes/");
|
||||
expect(prompt).toContain("recipe-suggestion.md");
|
||||
expect(prompt).toContain("do not call `finish` and describe the block only in prose");
|
||||
@@ -1157,6 +1165,7 @@ describe("Mantis Telegram Desktop proof workflow", () => {
|
||||
it("does not pass the full workflow environment into the local Telegram SUT", () => {
|
||||
const sutScript = readFileSync(MANTIS_SUT_SCRIPT, "utf8");
|
||||
const laneScript = readFileSync(MANTIS_LANE_SCRIPT, "utf8");
|
||||
const mockServer = readFileSync(MOCK_OPENAI_SERVER, "utf8");
|
||||
const prompt = readFileSync(PROMPT, "utf8");
|
||||
const workflow = readFileSync(WORKFLOW, "utf8");
|
||||
const wrapper = readFileSync(SUT_CONTAINER_WRAPPER, "utf8");
|
||||
@@ -1228,10 +1237,7 @@ describe("Mantis Telegram Desktop proof workflow", () => {
|
||||
expect(workflow).toContain(
|
||||
'sudo install -m 0444 "$toolchain_build/scripts/e2e/mock-openai-server.mjs"',
|
||||
);
|
||||
expect(wrapper).toContain("node /opt/mantis/mock-openai-server.mjs");
|
||||
expect(wrapper).toContain(
|
||||
'--mount "type=bind,src=$mock_server_script,dst=/opt/mantis/mock-openai-server.mjs,readonly"',
|
||||
);
|
||||
expect(wrapper).not.toContain('node /opt/mantis/mock-openai-server.mjs >"$MOCK_LOG"');
|
||||
expect(wrapper).not.toContain("node scripts/e2e/mock-openai-server.mjs");
|
||||
expect(workflow).toContain('sudo usermod -aG mantis-proof "$recorder_user"');
|
||||
expect(workflow).toContain(
|
||||
@@ -1284,6 +1290,25 @@ describe("Mantis Telegram Desktop proof workflow", () => {
|
||||
expect(wrapper).toContain(
|
||||
"--env TELEGRAM_PROXY_RECORD_FILE=/opt/mantis/proxy-control/requests.ndjson",
|
||||
);
|
||||
const mockContainerSpec = wrapper.slice(
|
||||
wrapper.indexOf('"$docker_bin" run --detach --name "$mock_container_name"'),
|
||||
wrapper.indexOf('wait_for_mock_openai "$mock_container_name"'),
|
||||
);
|
||||
expect(mockContainerSpec).toContain('--network "$network_name"');
|
||||
expect(mockContainerSpec).toContain("--network-alias mock-openai");
|
||||
expect(mockContainerSpec).not.toContain("$egress_network_name");
|
||||
expect(mockContainerSpec).toContain(
|
||||
'--mount "type=bind,src=$mock_server_script,dst=/opt/mantis/mock-openai-server.mjs,readonly"',
|
||||
);
|
||||
expect(mockContainerSpec).toContain(
|
||||
'--mount "type=bind,src=$response_control_dir,dst=/opt/mantis/mock-control"',
|
||||
);
|
||||
expect(wrapper).toContain("--env MOCK_BIND_HOST=0.0.0.0");
|
||||
expect(mockContainerSpec).toContain('--user "$(id -u mantis-sut):$(id -g mantis-sut)"');
|
||||
expect(mockServer).toContain('const bindHost = process.env.MOCK_BIND_HOST ?? "127.0.0.1"');
|
||||
expect(mockServer).toContain("server.listen(port, bindHost");
|
||||
expect(wrapper).toContain('wait_for_mock_openai "$mock_container_name" "$mock_log"');
|
||||
expect(wrapper).toContain("mock OpenAI container exited before readiness");
|
||||
// Candidate code shares the mantis-sut UID with the proxy record sink, so
|
||||
// the SUT container must shadow proxy-control; otherwise the lane under
|
||||
// test could rewrite its own trusted Bot API evidence before publication.
|
||||
@@ -1293,8 +1318,16 @@ describe("Mantis Telegram Desktop proof workflow", () => {
|
||||
expect(wrapper.indexOf(proxyControlShadow)).toBeGreaterThan(
|
||||
wrapper.indexOf('--mount "type=bind,src=$safe_runtime,dst=$runtime_source"'),
|
||||
);
|
||||
const mockControlShadow =
|
||||
'--mount "type=tmpfs,dst=$runtime_source/mock-control,tmpfs-size=65536,tmpfs-mode=0000"';
|
||||
expect(wrapper).toContain(mockControlShadow);
|
||||
expect(wrapper.indexOf(mockControlShadow)).toBeGreaterThan(
|
||||
wrapper.indexOf('--mount "type=bind,src=$safe_runtime,dst=$runtime_source"'),
|
||||
);
|
||||
expect(wrapper).toContain('export TELEGRAM_BOT_TOKEN="$telegram_alias_token"');
|
||||
expect(wrapper).not.toContain('export TELEGRAM_BOT_TOKEN="$telegram_bot_token"');
|
||||
expect(wrapper.match(/remove_container_or_fail "\$mock_container_name"/gu)).toHaveLength(2);
|
||||
expect(wrapper).toContain('remove_container_or_fail "${1}-mock-openai"');
|
||||
expect(wrapper).toContain('remove_container_or_fail "${1}-telegram-proxy"');
|
||||
expect(workflow).toContain(
|
||||
"/usr/local/lib/mantis-toolchain/scripts/e2e/telegram-bot-api-proxy.mjs",
|
||||
@@ -1338,17 +1371,21 @@ describe("Mantis Telegram Desktop proof workflow", () => {
|
||||
'const proxyControlDir = path.join(config.tempRoot, "proxy-control")',
|
||||
);
|
||||
expect(sutScript).toContain(
|
||||
'const requestLog = path.join(config.tempRoot, "mock-openai-requests.ndjson")',
|
||||
'const requestLog = path.join(mockResponseControlDir, "mock-openai-requests.ndjson")',
|
||||
);
|
||||
expect(wrapper).toContain(
|
||||
'export MOCK_RESPONSE_CONTROL="$runtime_source/mock-control/response.json"',
|
||||
expect(sutScript).toContain(
|
||||
'const mockLog = path.join(mockResponseControlDir, "mock-openai.log")',
|
||||
);
|
||||
const forwardedEnv = wrapper.slice(
|
||||
wrapper.indexOf("forwarded_env=("),
|
||||
wrapper.indexOf("docker_env=()"),
|
||||
);
|
||||
expect(forwardedEnv).toContain("MOCK_RESPONSE_CONTROL");
|
||||
expect(forwardedEnv).not.toContain("MOCK_RESPONSE_CONTROL");
|
||||
expect(forwardedEnv).not.toContain("MOCK_REQUEST_LOG");
|
||||
expect(forwardedEnv).not.toContain("MOCK_LOG");
|
||||
expect(forwardedEnv).not.toContain("MOCK_PORT");
|
||||
expect(wrapper).toContain("refusing to destroy a running SUT container");
|
||||
expect(wrapper).toContain("refusing to destroy a running mock OpenAI container");
|
||||
expect(wrapper).toContain('destroy_bounded_filesystem "$runtime_root"');
|
||||
expect(wrapper).toContain('create_runtime_claim "$container_name" "$runtime_source"');
|
||||
expect(wrapper).toContain('cancel_runtime_claim "$1" "$runtime_source"');
|
||||
|
||||
@@ -922,6 +922,83 @@ exit 1
|
||||
}
|
||||
});
|
||||
|
||||
it("exposes provider content facts through requests and terminal lane facts", async () => {
|
||||
const harness = await setupHarness({ userOnlyEvents: true });
|
||||
const contentFacts = [
|
||||
{
|
||||
type: "input_file",
|
||||
filename: "proof.pdf",
|
||||
mimeType: "application/pdf",
|
||||
byteLength: 17,
|
||||
},
|
||||
];
|
||||
fs.writeFileSync(
|
||||
harness.requestLog,
|
||||
`${JSON.stringify({
|
||||
seq: 1,
|
||||
body: "credential=123456:secret-sut-token",
|
||||
contentFacts,
|
||||
path: "/v1/responses",
|
||||
})}\n`,
|
||||
);
|
||||
try {
|
||||
const requests = JSON.parse(
|
||||
(await runLane(harness.env, ["requests", "--lane", "candidate"])).stdout,
|
||||
);
|
||||
expect(requests).toEqual({
|
||||
count: 1,
|
||||
requests: [
|
||||
{
|
||||
seq: 1,
|
||||
body: "credential=[redacted]",
|
||||
contentFacts,
|
||||
path: "/v1/responses",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
// Tail window: a session with more records than the window must expose
|
||||
// its newest requests — the ones under proof — with their absolute seq.
|
||||
fs.writeFileSync(
|
||||
harness.requestLog,
|
||||
Array.from(
|
||||
{ length: 130 },
|
||||
(_, i) => `${JSON.stringify({ seq: i + 1, body: `turn ${i + 1}` })}\n`,
|
||||
).join(""),
|
||||
);
|
||||
const tail = JSON.parse(
|
||||
(await runLane(harness.env, ["requests", "--lane", "candidate"])).stdout,
|
||||
);
|
||||
expect(tail.count).toBe(128);
|
||||
expect(tail.requests[0]).toEqual({ seq: 3, body: "turn 3" });
|
||||
expect(tail.requests.at(-1)).toEqual({ seq: 130, body: "turn 130" });
|
||||
|
||||
// Restore the single-record log so terminal lane facts mirror the
|
||||
// requests assertion above.
|
||||
fs.writeFileSync(
|
||||
harness.requestLog,
|
||||
`${JSON.stringify({
|
||||
seq: 1,
|
||||
body: "credential=123456:secret-sut-token",
|
||||
contentFacts,
|
||||
path: "/v1/responses",
|
||||
})}\n`,
|
||||
);
|
||||
await runLane(harness.env, ["send", "--lane", "candidate", "--text", "persist facts"]);
|
||||
await runLane(harness.env, ["finish", "--lane", "candidate"]);
|
||||
const facts = JSON.parse(
|
||||
fs.readFileSync(
|
||||
path.join(harness.outputRoot, "candidate", "mantis-lane-facts.json"),
|
||||
"utf8",
|
||||
),
|
||||
);
|
||||
expect(facts.providerRequests).toEqual(requests.requests);
|
||||
expect(JSON.stringify(facts.providerRequests)).not.toContain("secret-sut-token");
|
||||
} finally {
|
||||
await harness.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("finishes an expected-silence proof on the triggering user message", async () => {
|
||||
const harness = await setupHarness({ userOnlyEvents: true });
|
||||
try {
|
||||
|
||||
@@ -201,6 +201,7 @@ describe("Telegram Mantis SUT", () => {
|
||||
},
|
||||
gatewayPort: 19_879,
|
||||
groupId: "-100123456789",
|
||||
mockHost: "mock-openai",
|
||||
mockPort: 19_882,
|
||||
outputDir,
|
||||
testerId: "12345",
|
||||
@@ -212,6 +213,7 @@ describe("Telegram Mantis SUT", () => {
|
||||
expect(config.channels.telegram.streaming).toEqual({ mode: "partial" });
|
||||
expect(config.channels.telegram).not.toHaveProperty("replyToMode");
|
||||
expect(config.commands.ownerAllowFrom).toEqual(["telegram:12345"]);
|
||||
expect(config.models.providers.openai.baseUrl).toBe("http://mock-openai:19882/v1");
|
||||
expect(config.session.sendPolicy).toEqual({ default: "deny" });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -493,6 +493,7 @@ describe("telegram user Crabbox proof log polling", () => {
|
||||
gatewayPort: 19042,
|
||||
groupId: "group",
|
||||
mcpAppFixture: true,
|
||||
mockHost: "127.0.0.1",
|
||||
mockPort: 19043,
|
||||
outputDir: makeTempDir(tempDirs, "openclaw-telegram-proof-"),
|
||||
repoRoot: "/repo",
|
||||
@@ -520,6 +521,7 @@ describe("telegram user Crabbox proof log polling", () => {
|
||||
const configRoot = writeSutConfig({
|
||||
gatewayPort: 19042,
|
||||
groupId: "group",
|
||||
mockHost: "127.0.0.1",
|
||||
mockPort: 19043,
|
||||
outputDir: makeTempDir(tempDirs, "openclaw-telegram-proof-"),
|
||||
testerId: "tester",
|
||||
@@ -532,6 +534,7 @@ describe("telegram user Crabbox proof log polling", () => {
|
||||
executionIdentity: true,
|
||||
messages: "direct",
|
||||
});
|
||||
expect(config.models.providers.openai.baseUrl).toBe("http://127.0.0.1:19043/v1");
|
||||
});
|
||||
|
||||
it("injects the requested Telegram link-preview setting before startup", () => {
|
||||
@@ -539,6 +542,7 @@ describe("telegram user Crabbox proof log polling", () => {
|
||||
configPatch: { channels: { telegram: { linkPreview: false } } },
|
||||
gatewayPort: 19042,
|
||||
groupId: "group",
|
||||
mockHost: "127.0.0.1",
|
||||
mockPort: 19043,
|
||||
outputDir: makeTempDir(tempDirs, "openclaw-telegram-proof-"),
|
||||
testerId: "tester",
|
||||
@@ -546,6 +550,7 @@ describe("telegram user Crabbox proof log polling", () => {
|
||||
const defaultConfigRoot = writeSutConfig({
|
||||
gatewayPort: 19044,
|
||||
groupId: "group",
|
||||
mockHost: "127.0.0.1",
|
||||
mockPort: 19045,
|
||||
outputDir: makeTempDir(tempDirs, "openclaw-telegram-proof-"),
|
||||
testerId: "tester",
|
||||
@@ -570,6 +575,7 @@ describe("telegram user Crabbox proof log polling", () => {
|
||||
},
|
||||
gatewayPort: 19042,
|
||||
groupId: "group",
|
||||
mockHost: "127.0.0.1",
|
||||
mockPort: 19043,
|
||||
outputDir: makeTempDir(tempDirs, "openclaw-telegram-proof-"),
|
||||
testerId: "tester",
|
||||
@@ -577,6 +583,7 @@ describe("telegram user Crabbox proof log polling", () => {
|
||||
const defaultConfigRoot = writeSutConfig({
|
||||
gatewayPort: 19044,
|
||||
groupId: "group",
|
||||
mockHost: "127.0.0.1",
|
||||
mockPort: 19045,
|
||||
outputDir: makeTempDir(tempDirs, "openclaw-telegram-proof-"),
|
||||
testerId: "tester",
|
||||
|
||||
Reference in New Issue
Block a user