import { describe, expect, it } from "vitest"; import { buildAnthropicCliBackend } from "./cli-backend.js"; const MOCK_RAW_TOOL_OUTPUT = [ "I'll inspect the synthetic report.", "", '', 'wc -l /tmp/mock-report.md', 'Verify the mock report', "", "", "12 /tmp/mock-report.md", "", "The synthetic report has 12 lines.", ].join("\n"); function parseResult(result: string) { return buildAnthropicCliBackend().parseJsonlEvent?.( JSON.stringify({ type: "result", subtype: "success", result }), { backendId: "claude-cli", backend: buildAnthropicCliBackend().config, }, ); } describe("Claude CLI output validation", () => { it("rejects mocked raw tool protocol returned as terminal assistant text", () => { expect(parseResult(MOCK_RAW_TOOL_OUTPUT)).toEqual({ kind: "result", errorText: expect.stringContaining("raw tool protocol appeared as assistant text"), }); }); it.each(["\\u003c", "\\u003C"])( "rejects the %s-escaped JSON form reported by upstream Claude Code", (escapedLessThan) => { const line = JSON.stringify({ type: "result", result: MOCK_RAW_TOOL_OUTPUT }).replaceAll( "<", escapedLessThan, ); const backend = buildAnthropicCliBackend(); expect( backend.parseJsonlEvent?.(line, { backendId: backend.id, backend: backend.config }), ).toEqual({ kind: "result", errorText: expect.stringContaining("raw tool protocol appeared as assistant text"), }); }, ); it("rejects standalone protocol with CRLF line endings", () => { expect(parseResult(MOCK_RAW_TOOL_OUTPUT.replaceAll("\n", "\r\n"))).toEqual({ kind: "result", errorText: expect.stringContaining("raw tool protocol appeared as assistant text"), }); }); it("rejects a complete invocation whose parameter payload exceeds the opening-tag lookahead", () => { expect( parseResult( [ "Writing the generated fixture.", '', `${"x".repeat(3_000)}`, "", ].join("\n"), ), ).toEqual({ kind: "result", errorText: expect.stringContaining("raw tool protocol appeared as assistant text"), }); }); it.each(["call", "count", "court", "Bash"])( "rejects the upstream-observed %s prefix when the protocol block is truncated", (prefix) => { expect( parseResult( [ "I will inspect it.", prefix, '', 'wc -l /tmp/mock-report.md', ].join("\n"), ), ).toEqual({ kind: "result", errorText: expect.stringContaining("raw tool protocol appeared as assistant text"), }); }, ); it("does not let a later inline close token mask an observed truncated leak", () => { expect( parseResult( [ "call", '', 'pwd', "Documentation may mention inline.", ].join("\n"), ), ).toEqual({ kind: "result", errorText: expect.stringContaining("raw tool protocol appeared as assistant text"), }); }); it("does not let inline close prose before the parameter mask an observed truncated leak", () => { expect( parseResult( [ "call", '', "Documentation may mention inline.", 'pwd', ].join("\n"), ), ).toEqual({ kind: "result", errorText: expect.stringContaining("raw tool protocol appeared as assistant text"), }); }); it("continues to a valid parameter after a non-evidentiary parameter-like tag", () => { expect( parseResult( [ '', 'ignored', 'pwd', "", ].join("\n"), ), ).toEqual({ kind: "result", errorText: expect.stringContaining("raw tool protocol appeared as assistant text"), }); }); it("rejects a complete unfenced protocol example as the accepted false-positive tradeoff", () => { expect( parseResult( [ "Here is the exact raw protocol for documentation:", '', 'pwd', "", ].join("\n"), ), ).toEqual({ kind: "result", errorText: expect.stringContaining("raw tool protocol appeared as assistant text"), }); }); it.each([ ["ordinary chat", "The washer report is ready. Here are three recommendations."], [ "inline protocol discussion", 'Claude printed `pwd`.', ], [ "fenced protocol example", [ "Example:", "```xml", '', 'pwd', "", "```", ].join("\n"), ], [ "unterminated fenced protocol example", [ "Example:", "~~~xml", '', 'pwd', "", ].join("\n"), ], [ "same-line XML prose", 'Use pwd only as an example.', ], [ "line-leading protocol prose", [ '... denotes a call.', '... denotes its argument.', ].join("\n"), ], [ "indented protocol example", [ ' ', ' pwd', " ", ].join("\n"), ], [ "unrelated lowercase XML", ['', 'text', ""].join( "\n", ), ], [ "similarly named XML elements", [ '', 'pwd', "", ].join("\n"), ], [ "similarly named XML attributes", [ '', 'pwd', "", ].join("\n"), ], [ "name text inside unrelated quoted attributes", [ "", "pwd", "", ].join("\n"), ], ["standalone invoke without parameters", 'no parameter block'], [ "complete parameterless invocation not observed upstream", ['', ""].join("\n"), ], [ "parameter evidence belonging to a later lowercase invocation", [ '', "", '', 'text', "", ].join("\n"), ], [ "later lowercase invocation after a truncated parameterless invocation", [ "call", '', '', 'text', "", ].join("\n"), ], [ "unprefixed truncated protocol example", [ "Here is an incomplete protocol example:", '', 'pwd', ].join("\n"), ], [ "namespaced protocol example not observed upstream", [ '', 'pwd', "", ].join("\n"), ], ["long ordinary report", `Summary\n\n${"Normal report text. ".repeat(20_000)}`], ])("preserves %s", (_name, text) => { expect(parseResult(text)).toBeNull(); }); it("ignores malformed and non-terminal JSONL frames", () => { const backend = buildAnthropicCliBackend(); const context = { backendId: backend.id, backend: backend.config }; expect(backend.parseJsonlEvent?.("not json