import { describe, expect, it } from "vitest";
import { buildAnthropicCliBackend } from "./cli-backend.js";
const MOCK_RAW_TOOL_OUTPUT = [
"I'll inspect the synthetic report.",
"",
'',
'wc -l /tmp/mock-report.md',
'Verify the mock report',
"",
"",
"12 /tmp/mock-report.md",
"",
"The synthetic report has 12 lines.",
].join("\n");
function parseResult(result: string) {
return buildAnthropicCliBackend().parseJsonlEvent?.(
JSON.stringify({ type: "result", subtype: "success", result }),
{
backendId: "claude-cli",
backend: buildAnthropicCliBackend().config,
},
);
}
describe("Claude CLI output validation", () => {
it("rejects mocked raw tool protocol returned as terminal assistant text", () => {
expect(parseResult(MOCK_RAW_TOOL_OUTPUT)).toEqual({
kind: "result",
errorText: expect.stringContaining("raw tool protocol appeared as assistant text"),
});
});
it.each(["\\u003c", "\\u003C"])(
"rejects the %s-escaped JSON form reported by upstream Claude Code",
(escapedLessThan) => {
const line = JSON.stringify({ type: "result", result: MOCK_RAW_TOOL_OUTPUT }).replaceAll(
"<",
escapedLessThan,
);
const backend = buildAnthropicCliBackend();
expect(
backend.parseJsonlEvent?.(line, { backendId: backend.id, backend: backend.config }),
).toEqual({
kind: "result",
errorText: expect.stringContaining("raw tool protocol appeared as assistant text"),
});
},
);
it("rejects standalone protocol with CRLF line endings", () => {
expect(parseResult(MOCK_RAW_TOOL_OUTPUT.replaceAll("\n", "\r\n"))).toEqual({
kind: "result",
errorText: expect.stringContaining("raw tool protocol appeared as assistant text"),
});
});
it("rejects a complete invocation whose parameter payload exceeds the opening-tag lookahead", () => {
expect(
parseResult(
[
"Writing the generated fixture.",
'',
`${"x".repeat(3_000)}`,
"",
].join("\n"),
),
).toEqual({
kind: "result",
errorText: expect.stringContaining("raw tool protocol appeared as assistant text"),
});
});
it.each(["call", "count", "court", "Bash"])(
"rejects the upstream-observed %s prefix when the protocol block is truncated",
(prefix) => {
expect(
parseResult(
[
"I will inspect it.",
prefix,
'',
'wc -l /tmp/mock-report.md',
].join("\n"),
),
).toEqual({
kind: "result",
errorText: expect.stringContaining("raw tool protocol appeared as assistant text"),
});
},
);
it("does not let a later inline close token mask an observed truncated leak", () => {
expect(
parseResult(
[
"call",
'',
'pwd',
"Documentation may mention inline.",
].join("\n"),
),
).toEqual({
kind: "result",
errorText: expect.stringContaining("raw tool protocol appeared as assistant text"),
});
});
it("does not let inline close prose before the parameter mask an observed truncated leak", () => {
expect(
parseResult(
[
"call",
'',
"Documentation may mention inline.",
'pwd',
].join("\n"),
),
).toEqual({
kind: "result",
errorText: expect.stringContaining("raw tool protocol appeared as assistant text"),
});
});
it("continues to a valid parameter after a non-evidentiary parameter-like tag", () => {
expect(
parseResult(
[
'',
'ignored',
'pwd',
"",
].join("\n"),
),
).toEqual({
kind: "result",
errorText: expect.stringContaining("raw tool protocol appeared as assistant text"),
});
});
it("rejects a complete unfenced protocol example as the accepted false-positive tradeoff", () => {
expect(
parseResult(
[
"Here is the exact raw protocol for documentation:",
'',
'pwd',
"",
].join("\n"),
),
).toEqual({
kind: "result",
errorText: expect.stringContaining("raw tool protocol appeared as assistant text"),
});
});
it.each([
["ordinary chat", "The washer report is ready. Here are three recommendations."],
[
"inline protocol discussion",
'Claude printed `pwd`.',
],
[
"fenced protocol example",
[
"Example:",
"```xml",
'',
'pwd',
"",
"```",
].join("\n"),
],
[
"unterminated fenced protocol example",
[
"Example:",
"~~~xml",
'',
'pwd',
"",
].join("\n"),
],
[
"same-line XML prose",
'Use pwd only as an example.',
],
[
"line-leading protocol prose",
[
'... denotes a call.',
'... denotes its argument.',
].join("\n"),
],
[
"indented protocol example",
[
' ',
' pwd',
" ",
].join("\n"),
],
[
"unrelated lowercase XML",
['', 'text', ""].join(
"\n",
),
],
[
"similarly named XML elements",
[
'',
'pwd',
"",
].join("\n"),
],
[
"similarly named XML attributes",
[
'',
'pwd',
"",
].join("\n"),
],
[
"name text inside unrelated quoted attributes",
[
"",
"pwd",
"",
].join("\n"),
],
["standalone invoke without parameters", 'no parameter block'],
[
"complete parameterless invocation not observed upstream",
['', ""].join("\n"),
],
[
"parameter evidence belonging to a later lowercase invocation",
[
'',
"",
'',
'text',
"",
].join("\n"),
],
[
"later lowercase invocation after a truncated parameterless invocation",
[
"call",
'',
'',
'text',
"",
].join("\n"),
],
[
"unprefixed truncated protocol example",
[
"Here is an incomplete protocol example:",
'',
'pwd',
].join("\n"),
],
[
"namespaced protocol example not observed upstream",
[
'',
'pwd',
"",
].join("\n"),
],
["long ordinary report", `Summary\n\n${"Normal report text. ".repeat(20_000)}`],
])("preserves %s", (_name, text) => {
expect(parseResult(text)).toBeNull();
});
it("ignores malformed and non-terminal JSONL frames", () => {
const backend = buildAnthropicCliBackend();
const context = { backendId: backend.id, backend: backend.config };
expect(backend.parseJsonlEvent?.("not json