mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-25 20:05:46 -06:00
Ayaan Zaidi
352095882e
* fix(mantis): enforce verdict-expectation coherence and publish agent analysis files Mantis run 32619081130 on #127989 published an overall `pass` while its own manifest recorded that the candidate expectation was not observed: per-lane `status` was mechanical capture success and the agent's judgment lived only in `expected` prose, so nothing reconciled the two before publication. - `mantis-evidence.json` schemaVersion 2: each comparison lane carries a required boolean `expectationMet`. The desktop agent sets it in the same manifest edit as `expected`; mechanical producers (Telegram live, web UI, Slack, Discord) derive it from lane status. - `scripts/mantis/publish-pr-evidence.mjs` is the single enforcement owner: it requires the booleans, recomputes `pass`/`outcome`, downgrades a contradictory pass claim to `fail`, and renders a visible "verdict downgraded" note. The desktop workflow invokes it with `--validate-only` before upload or comment. - Agent top-level `*.json`/`*.md` analysis files (assertions, comparisons, recipe suggestion) now survive the quarantine rebuild and upload, so cited evidence actually exists in the artifact. * fix(mantis): derive expectations from trusted lane facts
107 lines
3.1 KiB
TypeScript
107 lines
3.1 KiB
TypeScript
// Mantis web UI chat evidence tests cover manifest generation.
|
|
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import path from "node:path";
|
|
import { describe, expect, it } from "vitest";
|
|
import {
|
|
buildWebUiChatEvidenceManifest,
|
|
writeWebUiChatEvidence,
|
|
} from "../../scripts/mantis/build-web-ui-chat-evidence.mjs";
|
|
|
|
function withTempDir<T>(fn: (dir: string) => T): T {
|
|
const dir = mkdtempSync(path.join(tmpdir(), "openclaw-mantis-web-ui-chat-"));
|
|
try {
|
|
return fn(dir);
|
|
} finally {
|
|
rmSync(dir, { force: true, recursive: true });
|
|
}
|
|
}
|
|
|
|
describe("build-web-ui-chat-evidence", () => {
|
|
it("marks a passing Control UI chat proof as publishable visible Mantis evidence", () => {
|
|
const manifest = buildWebUiChatEvidenceManifest({
|
|
candidateRef: "HEAD",
|
|
candidateSha: "1234567890abcdef1234567890abcdef12345678",
|
|
status: "pass",
|
|
});
|
|
|
|
expect(manifest).toMatchObject({
|
|
id: "web-ui-chat-proof",
|
|
scenario: "web-ui-chat-proof",
|
|
schemaVersion: 2,
|
|
comparison: {
|
|
candidate: {
|
|
expectationMet: true,
|
|
fixed: true,
|
|
ref: "HEAD",
|
|
sha: "1234567890abcdef1234567890abcdef12345678",
|
|
status: "pass",
|
|
},
|
|
pass: true,
|
|
},
|
|
});
|
|
expect(manifest.artifacts).toContainEqual(
|
|
expect.objectContaining({
|
|
inline: true,
|
|
kind: "desktopScreenshot",
|
|
path: "web-ui-chat.png",
|
|
required: true,
|
|
}),
|
|
);
|
|
});
|
|
|
|
it("writes a failing manifest without requiring a missing screenshot", () => {
|
|
withTempDir((dir) => {
|
|
const result = writeWebUiChatEvidence([
|
|
"--output-dir",
|
|
dir,
|
|
"--candidate-sha",
|
|
"abcdef",
|
|
"--status",
|
|
"fail",
|
|
]);
|
|
|
|
expect(existsSync(result.manifestPath)).toBe(true);
|
|
expect(existsSync(result.reportPath)).toBe(true);
|
|
const manifest = JSON.parse(readFileSync(result.manifestPath, "utf8"));
|
|
expect(manifest.comparison.pass).toBe(false);
|
|
expect(manifest.comparison.candidate.expectationMet).toBe(false);
|
|
expect(
|
|
manifest.artifacts.find(
|
|
(artifact: { path: string }) => artifact.path === "web-ui-chat.png",
|
|
),
|
|
).toMatchObject({
|
|
required: false,
|
|
});
|
|
expect(readFileSync(result.reportPath, "utf8")).toContain("Status: fail");
|
|
});
|
|
});
|
|
|
|
it("uses the explicit pass status and includes report output", () => {
|
|
withTempDir((dir) => {
|
|
writeFileSync(path.join(dir, "web-ui-chat.png"), "png");
|
|
writeFileSync(
|
|
path.join(dir, "web-ui-chat-proof.json"),
|
|
`${JSON.stringify({ status: "pass" })}\n`,
|
|
);
|
|
|
|
const result = writeWebUiChatEvidence([
|
|
"--output-dir",
|
|
dir,
|
|
"--candidate-ref",
|
|
"main",
|
|
"--status",
|
|
"pass",
|
|
]);
|
|
|
|
expect(result.manifest.comparison).toMatchObject({
|
|
candidate: { fixed: true, ref: "main", status: "pass" },
|
|
pass: true,
|
|
});
|
|
expect(readFileSync(result.reportPath, "utf8")).toContain(
|
|
"Screenshot: `web-ui-chat.png` (present)",
|
|
);
|
|
});
|
|
});
|
|
});
|