mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-26 12:26:38 -06:00
352095882e
* fix(mantis): enforce verdict-expectation coherence and publish agent analysis files Mantis run 32619081130 on #127989 published an overall `pass` while its own manifest recorded that the candidate expectation was not observed: per-lane `status` was mechanical capture success and the agent's judgment lived only in `expected` prose, so nothing reconciled the two before publication. - `mantis-evidence.json` schemaVersion 2: each comparison lane carries a required boolean `expectationMet`. The desktop agent sets it in the same manifest edit as `expected`; mechanical producers (Telegram live, web UI, Slack, Discord) derive it from lane status. - `scripts/mantis/publish-pr-evidence.mjs` is the single enforcement owner: it requires the booleans, recomputes `pass`/`outcome`, downgrades a contradictory pass claim to `fail`, and renders a visible "verdict downgraded" note. The desktop workflow invokes it with `--validate-only` before upload or comment. - Agent top-level `*.json`/`*.md` analysis files (assertions, comparisons, recipe suggestion) now survive the quarantine rebuild and upload, so cited evidence actually exists in the artifact. * fix(mantis): derive expectations from trusted lane facts
884 lines
30 KiB
JavaScript
884 lines
30 KiB
JavaScript
#!/usr/bin/env node
|
|
// Publishes evidence manifest artifacts and optional PR comments for Mantis proof.
|
|
import { Buffer } from "node:buffer";
|
|
import { execFileSync } from "node:child_process";
|
|
import { createHash, createHmac } from "node:crypto";
|
|
import { existsSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import { readBoundedResponseText } from "../lib/bounded-response.mjs";
|
|
|
|
/** @typedef {{ mode: "absent" | "contains", target: "botApiRequests" | "observationEvents" | "providerRequests", value: string }} EvidenceAssertion */
|
|
/** @typedef {Record<string, unknown> & { assertion?: EvidenceAssertion, assertionOccurrences?: number, detail?: string, digest?: string, expectationMet: boolean, expected?: string, fixed?: boolean, ref?: string, sha?: string, status?: string }} EvidenceLane */
|
|
/**
|
|
* @typedef {{
|
|
* alt?: string,
|
|
* inline?: boolean,
|
|
* kind: string,
|
|
* label: string,
|
|
* lane: string,
|
|
* path?: string,
|
|
* required?: boolean,
|
|
* source: string,
|
|
* targetPath: string,
|
|
* width?: number,
|
|
* }} EvidenceArtifact
|
|
*/
|
|
/**
|
|
* @typedef {{
|
|
* artifacts: EvidenceArtifact[],
|
|
* comparison: { baseline?: EvidenceLane, candidate: EvidenceLane, differential?: string, outcome: "blocked" | "fail" | "pass", pass: boolean, verdictNote?: string },
|
|
* id: string,
|
|
* manifestDir: string,
|
|
* scenario: string,
|
|
* schemaVersion: number,
|
|
* summary?: string,
|
|
* title: string,
|
|
* }} EvidenceManifest
|
|
*/
|
|
/**
|
|
* @typedef {{
|
|
* alt?: string,
|
|
* inline?: boolean,
|
|
* kind?: string,
|
|
* label?: string,
|
|
* lane?: string,
|
|
* path?: string,
|
|
* required?: boolean,
|
|
* targetPath?: string,
|
|
* width?: number,
|
|
* }} ManifestArtifactEntry
|
|
*/
|
|
/** @typedef {Omit<EvidenceManifest, "artifacts" | "manifestDir"> & { artifacts?: ManifestArtifactEntry[] }} EvidenceManifestFile */
|
|
/** @typedef {{ accessKeyId: string, bucket: string, endpoint: string, publicBaseUrl: string, region: string, secretAccessKey: string }} ObjectStorageConfig */
|
|
/** @typedef {(url: URL, init: { body: Buffer, headers: HeadersInit, method: string, signal: AbortSignal }) => Promise<Response>} ArtifactFetch */
|
|
/** @typedef {{ body: Buffer, headers: HeadersInit, method: string, url: URL }} SignedPutRequest */
|
|
/** @typedef {{ left: EvidenceArtifact, right: EvidenceArtifact }} EvidencePair */
|
|
/**
|
|
* @typedef {{
|
|
* artifactUrl?: string,
|
|
* manifest: EvidenceManifest,
|
|
* marker: string,
|
|
* rawBase: string,
|
|
* requestSource?: string,
|
|
* runUrl?: string,
|
|
* treeUrl?: string,
|
|
* }} RenderEvidenceCommentOptions
|
|
*/
|
|
|
|
// Evidence bundles can include full videos, so allow slow transfers while bounding each PUT.
|
|
const MANTIS_ARTIFACT_UPLOAD_TIMEOUT_MS = 300_000;
|
|
// Untrusted storage error bodies are for diagnostics only; keep them small.
|
|
const MANTIS_UPLOAD_ERROR_BODY_MAX_BYTES = 64 * 1024;
|
|
const COMMENT_GRAPHEME_SEGMENTER = new Intl.Segmenter("en", { granularity: "grapheme" });
|
|
const MANTIS_EVIDENCE_SCHEMA_VERSION = 2;
|
|
const TELEGRAM_ASSERTION_FACT_PATHS = {
|
|
botApiRequests: ["botApiRequests"],
|
|
observationEvents: ["observation", "events"],
|
|
providerRequests: ["providerRequests"],
|
|
};
|
|
|
|
/**
|
|
* @param {string | undefined} value
|
|
* @param {number} maxLength
|
|
* @returns {string | undefined}
|
|
*/
|
|
export function sanitizeCommentText(value, maxLength) {
|
|
const escaped = value
|
|
?.trim()
|
|
.replace(/\s+/gu, " ")
|
|
.replaceAll("&", "&")
|
|
.replaceAll("<", "<")
|
|
.replaceAll(">", ">")
|
|
.replaceAll("`", "`");
|
|
if (!escaped) {
|
|
return undefined;
|
|
}
|
|
const graphemes = Array.from(
|
|
COMMENT_GRAPHEME_SEGMENTER.segment(escaped),
|
|
({ segment }) => segment,
|
|
);
|
|
return graphemes.length > maxLength ? `${graphemes.slice(0, maxLength - 1).join("")}…` : escaped;
|
|
}
|
|
|
|
function parseArgs(argv) {
|
|
const args = {};
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const key = argv[index];
|
|
if (!key?.startsWith("--")) {
|
|
throw new Error(`Unexpected argument: ${key}`);
|
|
}
|
|
const name = key.slice(2).replaceAll("-", "_");
|
|
const value = argv[index + 1];
|
|
if (!value || value.startsWith("--")) {
|
|
throw new Error(`Missing value for ${key}`);
|
|
}
|
|
args[name] = value;
|
|
index += 1;
|
|
}
|
|
return args;
|
|
}
|
|
function requireArg(args, name) {
|
|
const value = args[name];
|
|
if (!value) {
|
|
throw new Error(`Missing --${name.replaceAll("_", "-")}.`);
|
|
}
|
|
return value;
|
|
}
|
|
/** @returns {EvidenceManifestFile} */
|
|
function readJson(filePath) {
|
|
return JSON.parse(readFileSync(filePath, "utf8"));
|
|
}
|
|
function assertInside(parentDir, candidatePath, label) {
|
|
const relative = path.relative(parentDir, candidatePath);
|
|
if (relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative))) {
|
|
return candidatePath;
|
|
}
|
|
throw new Error(`${label} escapes manifest directory: ${candidatePath}`);
|
|
}
|
|
function normalizeTargetPath(targetPath) {
|
|
const normalized = path.posix.normalize(String(targetPath).replaceAll("\\", "/"));
|
|
if (
|
|
normalized === "." ||
|
|
normalized === "" ||
|
|
normalized.startsWith("../") ||
|
|
normalized.includes("/../") ||
|
|
normalized.startsWith("/") ||
|
|
/^[A-Za-z]:/u.test(normalized)
|
|
) {
|
|
throw new Error(`Invalid artifact target path: ${targetPath}`);
|
|
}
|
|
return normalized;
|
|
}
|
|
/**
|
|
* @param {string} manifestDir
|
|
* @param {ManifestArtifactEntry} artifact
|
|
* @returns {EvidenceArtifact | null}
|
|
*/
|
|
function resolveArtifact(manifestDir, artifact) {
|
|
if (!artifact || typeof artifact !== "object") {
|
|
throw new Error("Manifest artifact entries must be objects.");
|
|
}
|
|
if (!artifact.path) {
|
|
throw new Error("Manifest artifact entry is missing path.");
|
|
}
|
|
const source = assertInside(
|
|
manifestDir,
|
|
path.resolve(manifestDir, artifact.path),
|
|
`Artifact ${artifact.label ?? artifact.path}`,
|
|
);
|
|
const required = artifact.required !== false;
|
|
if (!existsSync(source)) {
|
|
if (required) {
|
|
throw new Error(`Missing required artifact: ${artifact.path}`);
|
|
}
|
|
return null;
|
|
}
|
|
if (!statSync(source).isFile()) {
|
|
throw new Error(`Artifact is not a file: ${artifact.path}`);
|
|
}
|
|
return {
|
|
...artifact,
|
|
kind: artifact.kind ?? "attachment",
|
|
lane: artifact.lane ?? "run",
|
|
label: artifact.label ?? artifact.path,
|
|
required,
|
|
source,
|
|
targetPath: normalizeTargetPath(artifact.targetPath ?? path.basename(artifact.path)),
|
|
};
|
|
}
|
|
|
|
function requireExpectationMet(comparison, laneName) {
|
|
const lane = comparison[laneName];
|
|
if (!lane || typeof lane !== "object") {
|
|
throw new Error(`Mantis evidence comparison requires a ${laneName} lane.`);
|
|
}
|
|
if (typeof lane.expectationMet !== "boolean") {
|
|
throw new Error(`Mantis evidence comparison.${laneName}.expectationMet must be a boolean.`);
|
|
}
|
|
return lane.expectationMet;
|
|
}
|
|
|
|
function evaluateTelegramAssertion(manifest, manifestDir, laneName) {
|
|
const lane = manifest.comparison[laneName];
|
|
const assertion = lane?.assertion;
|
|
const validAssertion =
|
|
assertion &&
|
|
typeof assertion === "object" &&
|
|
!Array.isArray(assertion) &&
|
|
Object.keys(assertion).toSorted().join(",") === "mode,target,value" &&
|
|
Object.hasOwn(TELEGRAM_ASSERTION_FACT_PATHS, assertion.target) &&
|
|
(assertion.mode === "contains" || assertion.mode === "absent") &&
|
|
typeof assertion.value === "string" &&
|
|
assertion.value.length >= 1 &&
|
|
assertion.value.length <= 200;
|
|
if (!validAssertion) {
|
|
throw new Error(
|
|
`Telegram Desktop comparison.${laneName}.assertion must be exactly {target: providerRequests|botApiRequests|observationEvents, mode: contains|absent, value: 1..200 character literal}.`,
|
|
);
|
|
}
|
|
const factsPath = `${laneName}/mantis-lane-facts.json`;
|
|
const factsArtifact = (manifest.artifacts ?? []).find(
|
|
(artifact) => artifact?.lane === laneName && artifact?.path === factsPath,
|
|
);
|
|
if (!factsArtifact) {
|
|
throw new Error(`Telegram Desktop ${laneName} lane must list ${factsPath} as its artifact.`);
|
|
}
|
|
const factsSource = resolveArtifact(manifestDir, { ...factsArtifact, required: true }).source;
|
|
const facts = JSON.parse(readFileSync(factsSource, "utf8"));
|
|
const selectedFacts = TELEGRAM_ASSERTION_FACT_PATHS[assertion.target].reduce(
|
|
(value, key) => value?.[key],
|
|
facts,
|
|
);
|
|
if (!Array.isArray(selectedFacts)) {
|
|
throw new Error(
|
|
`Telegram Desktop ${laneName} facts target ${assertion.target} is not an array.`,
|
|
);
|
|
}
|
|
const assertionOccurrences = JSON.stringify(selectedFacts).split(assertion.value).length - 1;
|
|
const expectationMet =
|
|
assertion.mode === "contains" ? assertionOccurrences > 0 : assertionOccurrences === 0;
|
|
return { assertion, assertionOccurrences, expectationMet };
|
|
}
|
|
|
|
/**
|
|
* @param {EvidenceManifestFile} manifest
|
|
* @param {string} manifestDir
|
|
*/
|
|
function reconcileEvidenceVerdict(manifest, manifestDir) {
|
|
if (!manifest.comparison || typeof manifest.comparison !== "object") {
|
|
throw new Error("Mantis evidence manifest requires a comparison.");
|
|
}
|
|
const laneNames = manifest.comparison.baseline ? ["baseline", "candidate"] : ["candidate"];
|
|
// Telegram Desktop judgments are agent-authored, so trusted code derives them from lane facts.
|
|
// Other scenario builders and jq producers are trusted and supply the boolean directly.
|
|
const comparison = { ...manifest.comparison };
|
|
if (isTelegramDesktopProof(manifest)) {
|
|
for (const laneName of laneNames) {
|
|
comparison[laneName] = {
|
|
...comparison[laneName],
|
|
...evaluateTelegramAssertion(manifest, manifestDir, laneName),
|
|
};
|
|
}
|
|
}
|
|
const unmetLanes = laneNames.filter((laneName) => !requireExpectationMet(comparison, laneName));
|
|
const claimedPass = comparison.pass || comparison.outcome === "pass";
|
|
const pass = comparison.pass && unmetLanes.length === 0;
|
|
const outcome = pass ? "pass" : comparison.outcome === "blocked" ? "blocked" : "fail";
|
|
const downgradeNote = `verdict downgraded: ${unmetLanes.join(" and ")} expectation${unmetLanes.length === 1 ? "" : "s"} not met`;
|
|
const verdictNote =
|
|
unmetLanes.length > 0 && (claimedPass || comparison.verdictNote === downgradeNote)
|
|
? downgradeNote
|
|
: undefined;
|
|
const { verdictNote: _untrustedVerdictNote, ...rest } = comparison;
|
|
return {
|
|
...manifest,
|
|
comparison: {
|
|
...rest,
|
|
outcome,
|
|
pass,
|
|
...(verdictNote ? { verdictNote } : {}),
|
|
},
|
|
};
|
|
}
|
|
|
|
/** @param {string} manifestPath */
|
|
export function validateEvidenceManifestFile(manifestPath) {
|
|
const resolvedManifest = path.resolve(manifestPath);
|
|
const manifestDir = path.dirname(resolvedManifest);
|
|
const manifest = validateEvidenceManifest(readJson(resolvedManifest), manifestDir);
|
|
for (const artifact of manifest.artifacts ?? []) {
|
|
resolveArtifact(manifestDir, artifact);
|
|
}
|
|
writeFileSync(resolvedManifest, `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
|
|
return manifest;
|
|
}
|
|
|
|
/**
|
|
* @param {EvidenceManifestFile} manifest
|
|
* @param {string} manifestDir
|
|
*/
|
|
function validateEvidenceManifest(manifest, manifestDir) {
|
|
if (manifest.schemaVersion !== MANTIS_EVIDENCE_SCHEMA_VERSION) {
|
|
throw new Error(
|
|
`Unsupported Mantis evidence manifest schema: ${manifest.schemaVersion}. ${isTelegramDesktopProof(manifest) ? "Rerun the Mantis Telegram Desktop Proof workflow; saved version-1 artifacts are not migrated." : "Rerun the proof to create schema version 2 evidence."}`,
|
|
);
|
|
}
|
|
if (!manifest.id || !manifest.title || !manifest.scenario) {
|
|
throw new Error("Mantis evidence manifest requires id, title, and scenario.");
|
|
}
|
|
return reconcileEvidenceVerdict(manifest, manifestDir);
|
|
}
|
|
/**
|
|
* Loads and validates an evidence manifest from disk.
|
|
*
|
|
* @param {string} manifestPath
|
|
* @returns {EvidenceManifest}
|
|
*/
|
|
export function loadEvidenceManifest(manifestPath) {
|
|
const resolvedManifest = path.resolve(manifestPath);
|
|
const manifestDir = path.dirname(resolvedManifest);
|
|
const manifest = validateEvidenceManifestFile(resolvedManifest);
|
|
const artifacts = (manifest.artifacts ?? [])
|
|
.map((artifact) => resolveArtifact(manifestDir, artifact))
|
|
.filter((artifact) => artifact !== null);
|
|
artifacts.push({
|
|
kind: "metadata",
|
|
lane: "run",
|
|
label: "Mantis evidence manifest",
|
|
source: resolvedManifest,
|
|
targetPath: "mantis-evidence.json",
|
|
});
|
|
return {
|
|
...manifest,
|
|
artifacts,
|
|
manifestDir,
|
|
};
|
|
}
|
|
function encodePathForUrl(input) {
|
|
return input
|
|
.split("/")
|
|
.filter(Boolean)
|
|
.map((part) => encodeURIComponent(part))
|
|
.join("/");
|
|
}
|
|
function artifactUrl(rawBase, artifact) {
|
|
return `${rawBase}/${encodePathForUrl(artifact.targetPath)}`;
|
|
}
|
|
function requireEnv(env, name) {
|
|
const value = env[name]?.trim();
|
|
if (!value) {
|
|
throw new Error(`Missing ${name}.`);
|
|
}
|
|
return value;
|
|
}
|
|
/** @returns {ObjectStorageConfig} */
|
|
function objectStorageConfig(env = process.env) {
|
|
return {
|
|
accessKeyId: requireEnv(env, "MANTIS_ARTIFACT_R2_ACCESS_KEY_ID"),
|
|
bucket: requireEnv(env, "MANTIS_ARTIFACT_R2_BUCKET"),
|
|
endpoint: requireEnv(env, "MANTIS_ARTIFACT_R2_ENDPOINT").replace(/\/+$/u, ""),
|
|
publicBaseUrl: requireEnv(env, "MANTIS_ARTIFACT_R2_PUBLIC_BASE_URL").replace(/\/+$/u, ""),
|
|
region: requireEnv(env, "MANTIS_ARTIFACT_R2_REGION"),
|
|
secretAccessKey: requireEnv(env, "MANTIS_ARTIFACT_R2_SECRET_ACCESS_KEY"),
|
|
};
|
|
}
|
|
function digestHex(value) {
|
|
return createHash("sha256").update(value).digest("hex");
|
|
}
|
|
function hmacBuffer(key, value) {
|
|
return createHmac("sha256", key).update(value).digest();
|
|
}
|
|
function hmacHex(key, value) {
|
|
return createHmac("sha256", key).update(value).digest("hex");
|
|
}
|
|
function signingKey({ date, region, secretAccessKey }) {
|
|
const dateKey = hmacBuffer(`AWS4${secretAccessKey}`, date);
|
|
const regionKey = hmacBuffer(dateKey, region);
|
|
const serviceKey = hmacBuffer(regionKey, "s3");
|
|
return hmacBuffer(serviceKey, "aws4_request");
|
|
}
|
|
function s3Path({ bucket, key }) {
|
|
return `/${encodePathForUrl(bucket)}/${encodePathForUrl(key)}`;
|
|
}
|
|
function contentType(filePath) {
|
|
const extension = path.extname(filePath).toLowerCase();
|
|
return (
|
|
{
|
|
".gif": "image/gif",
|
|
".html": "text/html; charset=utf-8",
|
|
".json": "application/json",
|
|
".md": "text/markdown; charset=utf-8",
|
|
".mp4": "video/mp4",
|
|
".png": "image/png",
|
|
".webm": "video/webm",
|
|
}[extension] ?? "application/octet-stream"
|
|
);
|
|
}
|
|
/** @returns {SignedPutRequest} */
|
|
function signedPutRequest({ artifact, body, config, key, now = new Date() }) {
|
|
const url = new URL(`${config.endpoint}${s3Path({ bucket: config.bucket, key })}`);
|
|
const amzDate = now.toISOString().replace(/[:-]|\.\d{3}/gu, "");
|
|
const date = amzDate.slice(0, 8);
|
|
const payloadHash = digestHex(body);
|
|
const headers = {
|
|
"content-type": contentType(artifact.targetPath),
|
|
host: url.host,
|
|
"x-amz-content-sha256": payloadHash,
|
|
"x-amz-date": amzDate,
|
|
};
|
|
const canonicalHeaders = Object.entries(headers)
|
|
.toSorted(([left], [right]) => left.localeCompare(right))
|
|
.map(([name, value]) => `${name}:${value}\n`)
|
|
.join("");
|
|
const signedHeaders = Object.keys(headers).toSorted().join(";");
|
|
const canonicalRequest = [
|
|
"PUT",
|
|
url.pathname,
|
|
"",
|
|
canonicalHeaders,
|
|
signedHeaders,
|
|
payloadHash,
|
|
].join("\n");
|
|
const scope = `${date}/${config.region}/s3/aws4_request`;
|
|
const stringToSign = ["AWS4-HMAC-SHA256", amzDate, scope, digestHex(canonicalRequest)].join("\n");
|
|
const signature = hmacHex(
|
|
signingKey({ date, region: config.region, secretAccessKey: config.secretAccessKey }),
|
|
stringToSign,
|
|
);
|
|
return {
|
|
body,
|
|
headers: {
|
|
"content-type": headers["content-type"],
|
|
"x-amz-content-sha256": headers["x-amz-content-sha256"],
|
|
"x-amz-date": headers["x-amz-date"],
|
|
authorization: `AWS4-HMAC-SHA256 Credential=${config.accessKeyId}/${scope}, SignedHeaders=${signedHeaders}, Signature=${signature}`,
|
|
},
|
|
method: "PUT",
|
|
url,
|
|
};
|
|
}
|
|
function byLane(artifacts, kind) {
|
|
const lanes = new Map();
|
|
for (const artifact of artifacts) {
|
|
if (artifact.kind !== kind) {
|
|
continue;
|
|
}
|
|
lanes.set(artifact.lane, artifact);
|
|
}
|
|
return lanes;
|
|
}
|
|
function findPair(artifacts, kind, leftLane, rightLane) {
|
|
const lanes = byLane(artifacts, kind);
|
|
const left = lanes.get(leftLane);
|
|
const right = lanes.get(rightLane);
|
|
return left && right ? { left, right } : null;
|
|
}
|
|
function renderPairTable({ pair, rawBase }) {
|
|
const { left, right } = pair;
|
|
if (!left || !right) {
|
|
return "";
|
|
}
|
|
return [
|
|
'<table width="100%">',
|
|
" <thead>",
|
|
" <tr>",
|
|
` <th width="50%">${left.label}</th>`,
|
|
` <th width="50%">${right.label}</th>`,
|
|
" </tr>",
|
|
" </thead>",
|
|
" <tbody>",
|
|
" <tr>",
|
|
` <td width="50%" align="center"><img src="${artifactUrl(rawBase, left)}" width="100%" alt="${left.alt ?? left.label}"></td>`,
|
|
` <td width="50%" align="center"><img src="${artifactUrl(rawBase, right)}" width="100%" alt="${right.alt ?? right.label}"></td>`,
|
|
" </tr>",
|
|
" </tbody>",
|
|
"</table>",
|
|
"",
|
|
].join("\n");
|
|
}
|
|
function renderSingleImageTables({ artifacts, rawBase, pairedKeys }) {
|
|
const renderedPairs = new Set(pairedKeys);
|
|
return artifacts
|
|
.filter(
|
|
(artifact) => artifact.inline && !renderedPairs.has(`${artifact.kind}:${artifact.lane}`),
|
|
)
|
|
.map((artifact) => {
|
|
const width = Math.min(Number(artifact.width ?? 720) || 720, 900);
|
|
return [
|
|
`**${artifact.label}**`,
|
|
"",
|
|
`<img src="${artifactUrl(rawBase, artifact)}" width="${width}" alt="${artifact.alt ?? artifact.label}">`,
|
|
"",
|
|
].join("\n");
|
|
})
|
|
.join("\n");
|
|
}
|
|
function renderLinkList({ artifacts, kind, rawBase, title }) {
|
|
const links = artifacts
|
|
.filter((artifact) => artifact.kind === kind)
|
|
.map((artifact) => `- [${artifact.label}](${artifactUrl(rawBase, artifact)})`);
|
|
if (links.length === 0) {
|
|
return "";
|
|
}
|
|
return [`${title}:`, ...links, ""].join("\n");
|
|
}
|
|
function laneLine(label, lane) {
|
|
if (!lane) {
|
|
return "";
|
|
}
|
|
const pieces = [`- ${label}: \`${lane.status ?? "unknown"}\``];
|
|
if (lane.sha) {
|
|
pieces.push(` at \`${lane.sha}\``);
|
|
} else if (lane.ref) {
|
|
pieces.push(` at \`${lane.ref}\``);
|
|
}
|
|
if (lane.digest) {
|
|
const judgment = lane.detail ?? sanitizeCommentText(lane.expected, 1_000);
|
|
if (judgment) {
|
|
pieces.push(` — ${judgment}`);
|
|
}
|
|
pieces.push(` · facts: ${lane.digest}`);
|
|
} else if (lane.detail) {
|
|
pieces.push(` — ${lane.detail}`);
|
|
} else if (lane.expected) {
|
|
pieces.push(`, expected ${lane.expected}`);
|
|
}
|
|
return pieces.join("");
|
|
}
|
|
function laneAssertionLine(label, lane) {
|
|
if (!lane?.assertion || typeof lane.assertionOccurrences !== "number") {
|
|
return "";
|
|
}
|
|
const value = sanitizeCommentText(lane.assertion.value, 200);
|
|
return `- ${label} assertion: \`${lane.assertion.target}\` \`${lane.assertion.mode}\` "${value}" · occurrences: ${lane.assertionOccurrences} · ${lane.expectationMet ? "met" : "unmet"}`;
|
|
}
|
|
function hasVisibleProofArtifacts(manifest) {
|
|
return manifest.artifacts.some((artifact) =>
|
|
["desktopScreenshot", "fullVideo", "motionClip", "motionPreview", "timeline"].includes(
|
|
artifact.kind,
|
|
),
|
|
);
|
|
}
|
|
function isTelegramDesktopProof(manifest) {
|
|
return manifest.id === "telegram-desktop-proof" || manifest.scenario === "telegram-desktop-proof";
|
|
}
|
|
function publicSummary(manifest) {
|
|
return manifest.summary ?? "Mantis captured QA evidence for this scenario.";
|
|
}
|
|
function overallStatus(manifest) {
|
|
const outcome = manifest.comparison?.outcome;
|
|
if (outcome === "blocked" || outcome === "fail" || outcome === "pass") {
|
|
return outcome;
|
|
}
|
|
const pass = manifest.comparison?.pass;
|
|
return typeof pass === "boolean" ? String(pass) : "";
|
|
}
|
|
/**
|
|
* @param {EvidenceManifest} manifest
|
|
* @param {{ requestSource?: string }} [options]
|
|
*/
|
|
export function shouldPublishPrComment(manifest, { requestSource } = {}) {
|
|
if (!isTelegramDesktopProof(manifest) || hasVisibleProofArtifacts(manifest)) {
|
|
return true;
|
|
}
|
|
if (manifest.comparison?.outcome === "blocked") {
|
|
return true;
|
|
}
|
|
if (requestSource === "pull_request_target") {
|
|
return false;
|
|
}
|
|
return manifest.comparison.pass;
|
|
}
|
|
/** @param {RenderEvidenceCommentOptions} options */
|
|
export function renderEvidenceComment({
|
|
artifactUrl: actionsArtifactUrl,
|
|
manifest,
|
|
marker,
|
|
rawBase,
|
|
requestSource,
|
|
runUrl,
|
|
treeUrl,
|
|
}) {
|
|
const comparison = manifest.comparison ?? {};
|
|
const baseline = comparison.baseline;
|
|
const candidate = comparison.candidate;
|
|
const pairs = [
|
|
findPair(manifest.artifacts, "timeline", "baseline", "candidate"),
|
|
findPair(manifest.artifacts, "desktopScreenshot", "baseline", "candidate"),
|
|
findPair(manifest.artifacts, "motionPreview", "baseline", "candidate"),
|
|
].filter((pair) => pair !== null);
|
|
const pairedKeys = pairs.flatMap((pair) => [
|
|
`${pair.left.kind}:${pair.left.lane}`,
|
|
`${pair.right.kind}:${pair.right.lane}`,
|
|
]);
|
|
const lines = [
|
|
marker,
|
|
`## ${manifest.title}`,
|
|
"",
|
|
`Summary: ${publicSummary(manifest)}`,
|
|
"",
|
|
`- Scenario: \`${manifest.scenario}\``,
|
|
];
|
|
if (requestSource) {
|
|
lines.push(`- Trigger: \`${requestSource}\``);
|
|
}
|
|
if (runUrl) {
|
|
lines.push(`- Run: ${runUrl}`);
|
|
}
|
|
if (actionsArtifactUrl) {
|
|
lines.push(`- Artifact: ${actionsArtifactUrl}`);
|
|
}
|
|
for (const { assertionLabel, lane, laneLabel } of [
|
|
{ assertionLabel: "Baseline", lane: baseline, laneLabel: "Baseline" },
|
|
{
|
|
assertionLabel: "Candidate",
|
|
lane: candidate,
|
|
laneLabel: "Candidate (PR merged onto main)",
|
|
},
|
|
]) {
|
|
const laneSummary = laneLine(laneLabel, lane);
|
|
const assertionSummary = laneAssertionLine(assertionLabel, lane);
|
|
lines.push(...[laneSummary, assertionSummary].filter(Boolean));
|
|
}
|
|
if (comparison.differential) {
|
|
lines.push(`- Differential (trusted facts): ${comparison.differential}`);
|
|
}
|
|
if (comparison.verdictNote) {
|
|
lines.push(`- Note: ${comparison.verdictNote}`);
|
|
}
|
|
const overall = overallStatus(manifest);
|
|
if (overall) {
|
|
lines.push(`- Overall: \`${overall}\``);
|
|
}
|
|
lines.push("");
|
|
const pairedSections = pairs.map((pair) => renderPairTable({ pair, rawBase }));
|
|
lines.push(...pairedSections);
|
|
const singleTables = renderSingleImageTables({
|
|
artifacts: manifest.artifacts,
|
|
pairedKeys,
|
|
rawBase,
|
|
});
|
|
if (singleTables) {
|
|
lines.push(singleTables);
|
|
}
|
|
const motionClips = renderLinkList({
|
|
artifacts: manifest.artifacts,
|
|
kind: "motionClip",
|
|
rawBase,
|
|
title: "Motion-trimmed clips",
|
|
});
|
|
if (motionClips) {
|
|
lines.push(motionClips);
|
|
}
|
|
const fullVideos = renderLinkList({
|
|
artifacts: manifest.artifacts,
|
|
kind: "fullVideo",
|
|
rawBase,
|
|
title: "Full videos",
|
|
});
|
|
if (fullVideos) {
|
|
lines.push(fullVideos);
|
|
}
|
|
lines.push(`Raw QA files: ${treeUrl ?? rawBase}`);
|
|
return `${lines.join("\n").replace(/\n{3,}/gu, "\n\n")}\n`;
|
|
}
|
|
function run(command, args, options = {}) {
|
|
return execFileSync(command, args, {
|
|
encoding: "utf8",
|
|
stdio: options.stdio ?? ["ignore", "pipe", "inherit"],
|
|
});
|
|
}
|
|
async function uploadArtifact({ artifact, fetchImpl, request, timeoutMs }) {
|
|
const signal = AbortSignal.timeout(timeoutMs);
|
|
try {
|
|
const response = await fetchImpl(request.url, {
|
|
body: request.body,
|
|
headers: request.headers,
|
|
method: request.method,
|
|
signal,
|
|
});
|
|
if (response.ok) {
|
|
await response.body?.cancel().catch(() => undefined);
|
|
return;
|
|
}
|
|
const failurePrefix = `Failed to upload Mantis artifact ${artifact.targetPath}: ${response.status} ${response.statusText}`;
|
|
const responseText = await readBoundedResponseText(
|
|
response,
|
|
"Mantis upload error",
|
|
MANTIS_UPLOAD_ERROR_BODY_MAX_BYTES,
|
|
{
|
|
signal,
|
|
formatTooLargeMessage: (_label, maxBytes) =>
|
|
`${failurePrefix}\nMantis upload error response body exceeded ${maxBytes} bytes`,
|
|
},
|
|
);
|
|
throw new Error(`${failurePrefix}\n${responseText}`);
|
|
} catch (error) {
|
|
if (signal.aborted) {
|
|
throw new Error(
|
|
`Timed out uploading Mantis artifact ${artifact.targetPath} after ${timeoutMs}ms.`,
|
|
{ cause: error },
|
|
);
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
/** @type {ArtifactFetch} */
|
|
const defaultArtifactFetch = (url, init) => {
|
|
const body =
|
|
init.body.buffer instanceof ArrayBuffer
|
|
? new Uint8Array(init.body.buffer, init.body.byteOffset, init.body.byteLength)
|
|
: Uint8Array.from(init.body);
|
|
return fetch(url, { ...init, body });
|
|
};
|
|
/**
|
|
* @param {{ artifactRoot: string, fetchImpl?: ArtifactFetch, manifest: EvidenceManifest, storageConfig?: ObjectStorageConfig, timeoutMs?: number }} options
|
|
* @returns {Promise<{ artifactRoot: string, rawBase: string, treeUrl: string }>}
|
|
*/
|
|
export async function publishArtifactFiles({
|
|
artifactRoot,
|
|
fetchImpl = defaultArtifactFetch,
|
|
manifest,
|
|
storageConfig = objectStorageConfig(),
|
|
timeoutMs = MANTIS_ARTIFACT_UPLOAD_TIMEOUT_MS,
|
|
}) {
|
|
const safeArtifactRoot = normalizeTargetPath(artifactRoot);
|
|
const publicRoot = `${storageConfig.publicBaseUrl}/${encodePathForUrl(safeArtifactRoot)}`;
|
|
for (const artifact of manifest.artifacts) {
|
|
const key = normalizeTargetPath(`${safeArtifactRoot}/${artifact.targetPath}`);
|
|
const request = signedPutRequest({
|
|
artifact,
|
|
body: readFileSync(artifact.source),
|
|
config: storageConfig,
|
|
key,
|
|
});
|
|
await uploadArtifact({ artifact, fetchImpl, request, timeoutMs });
|
|
}
|
|
const indexArtifact = {
|
|
targetPath: "index.json",
|
|
};
|
|
const indexRequest = signedPutRequest({
|
|
artifact: indexArtifact,
|
|
body: Buffer.from(
|
|
`${JSON.stringify(
|
|
{
|
|
artifacts: manifest.artifacts.map((artifact) => ({
|
|
kind: artifact.kind,
|
|
label: artifact.label,
|
|
lane: artifact.lane,
|
|
targetPath: artifact.targetPath,
|
|
url: artifactUrl(publicRoot, artifact),
|
|
})),
|
|
comparison: manifest.comparison,
|
|
id: manifest.id,
|
|
rawBase: publicRoot,
|
|
scenario: manifest.scenario,
|
|
summary: manifest.summary,
|
|
title: manifest.title,
|
|
},
|
|
null,
|
|
2,
|
|
)}\n`,
|
|
),
|
|
config: storageConfig,
|
|
key: normalizeTargetPath(`${safeArtifactRoot}/${indexArtifact.targetPath}`),
|
|
});
|
|
await uploadArtifact({ artifact: indexArtifact, fetchImpl, request: indexRequest, timeoutMs });
|
|
return {
|
|
artifactRoot: safeArtifactRoot,
|
|
rawBase: publicRoot,
|
|
treeUrl: artifactUrl(publicRoot, indexArtifact),
|
|
};
|
|
}
|
|
function upsertPrComment({ body, createMissing, marker, prNumber, repo }) {
|
|
run("gh", ["api", `repos/${repo}/pulls/${prNumber}`, "--jq", ".number"]);
|
|
const commentId = run("gh", [
|
|
"api",
|
|
"--paginate",
|
|
`repos/${repo}/issues/${prNumber}/comments`,
|
|
"--jq",
|
|
`.[] | select(.user.login == "openclaw-mantis[bot]" and (.body | contains("${marker}"))) | .id`,
|
|
])
|
|
.trim()
|
|
.split("\n")
|
|
.findLast((line) => line.length > 0);
|
|
const bodyFile = path.join(mkdtempSync(path.join(tmpdir(), "mantis-comment-")), "body.md");
|
|
writeFileSync(bodyFile, body);
|
|
try {
|
|
if (commentId) {
|
|
const payloadFile = `${bodyFile}.json`;
|
|
writeFileSync(payloadFile, JSON.stringify({ body }));
|
|
try {
|
|
run("gh", [
|
|
"api",
|
|
"--method",
|
|
"PATCH",
|
|
`repos/${repo}/issues/comments/${commentId}`,
|
|
"--input",
|
|
payloadFile,
|
|
]);
|
|
console.log(`Updated Mantis QA evidence comment on PR #${prNumber}.`);
|
|
return;
|
|
} catch {
|
|
if (!createMissing) {
|
|
console.log(
|
|
`Could not update existing Mantis QA evidence comment ${commentId}; create-missing is false.`,
|
|
);
|
|
return;
|
|
}
|
|
console.warn(
|
|
`Could not update existing Mantis QA evidence comment ${commentId}; creating a new one.`,
|
|
);
|
|
}
|
|
}
|
|
if (!createMissing) {
|
|
console.log("No existing Mantis QA evidence comment found and create-missing is false.");
|
|
return;
|
|
}
|
|
run("gh", ["pr", "comment", prNumber, "--body-file", bodyFile], { stdio: "inherit" });
|
|
console.log(`Created Mantis QA evidence comment on PR #${prNumber}.`);
|
|
} finally {
|
|
rmSync(path.dirname(bodyFile), { force: true, recursive: true });
|
|
}
|
|
}
|
|
/** @param {string[]} [rawArgs] */
|
|
export async function publishEvidence(rawArgs = process.argv.slice(2)) {
|
|
const args = parseArgs(rawArgs);
|
|
const manifestPath = requireArg(args, "manifest");
|
|
if (args.validate_only === "true") {
|
|
validateEvidenceManifestFile(manifestPath);
|
|
console.log(`Validated Mantis evidence manifest: ${manifestPath}`);
|
|
return;
|
|
}
|
|
const targetPr = requireArg(args, "target_pr");
|
|
const artifactRoot = requireArg(args, "artifact_root");
|
|
const marker = requireArg(args, "marker");
|
|
if (!/^[0-9]+$/u.test(targetPr)) {
|
|
throw new Error(`--target-pr must be numeric, got ${targetPr}.`);
|
|
}
|
|
const repo = args.repo ?? process.env.GITHUB_REPOSITORY;
|
|
const ghToken = process.env.GH_TOKEN ?? process.env.GITHUB_TOKEN;
|
|
if (!repo) {
|
|
throw new Error("Missing --repo or GITHUB_REPOSITORY.");
|
|
}
|
|
if (!ghToken) {
|
|
throw new Error("Missing GH_TOKEN or GITHUB_TOKEN.");
|
|
}
|
|
const manifest = loadEvidenceManifest(manifestPath);
|
|
const published = await publishArtifactFiles({
|
|
artifactRoot,
|
|
manifest,
|
|
});
|
|
const body = renderEvidenceComment({
|
|
artifactUrl: args.artifact_url,
|
|
manifest,
|
|
marker,
|
|
rawBase: published.rawBase,
|
|
requestSource: args.request_source,
|
|
runUrl: args.run_url,
|
|
treeUrl: published.treeUrl,
|
|
});
|
|
if (!shouldPublishPrComment(manifest, { requestSource: args.request_source })) {
|
|
console.log("Skipped Mantis QA evidence PR comment because the run did not capture proof.");
|
|
return;
|
|
}
|
|
upsertPrComment({
|
|
body,
|
|
createMissing: args.create_missing !== "false",
|
|
marker,
|
|
prNumber: targetPr,
|
|
repo,
|
|
});
|
|
}
|
|
const executedPath = process.argv[1] ? path.resolve(process.argv[1]) : "";
|
|
if (executedPath === fileURLToPath(import.meta.url)) {
|
|
try {
|
|
await publishEvidence();
|
|
} catch (error) {
|
|
console.error(error instanceof Error ? error.message : String(error));
|
|
process.exit(1);
|
|
}
|
|
}
|