diff --git a/extensions/anthropic/cli-backend.ts b/extensions/anthropic/cli-backend.ts index 852736d78c43..4939b90b650c 100644 --- a/extensions/anthropic/cli-backend.ts +++ b/extensions/anthropic/cli-backend.ts @@ -11,6 +11,7 @@ import { CLI_FRESH_WATCHDOG_DEFAULTS, CLI_RESUME_WATCHDOG_DEFAULTS, } from "openclaw/plugin-sdk/cli-backend"; +import { parseClaudeCliJsonlEvent } from "./cli-output.js"; import { CLAUDE_CLI_BACKEND_ID, CLAUDE_CLI_DEFAULT_MODEL_REF, @@ -237,6 +238,7 @@ export function buildAnthropicCliBackend(): CliBackendPlugin { } : undefined; }, + parseJsonlEvent: parseClaudeCliJsonlEvent, resolveExecutionArgs: resolveClaudeCliExecutionArgs, }; } diff --git a/extensions/anthropic/cli-output.test.ts b/extensions/anthropic/cli-output.test.ts new file mode 100644 index 000000000000..cf9b19516506 --- /dev/null +++ b/extensions/anthropic/cli-output.test.ts @@ -0,0 +1,293 @@ +import { describe, expect, it } from "vitest"; +import { buildAnthropicCliBackend } from "./cli-backend.js"; + +const MOCK_RAW_TOOL_OUTPUT = [ + "I'll inspect the synthetic report.", + "", + '', + 'wc -l /tmp/mock-report.md', + 'Verify the mock report', + "", + "", + "12 /tmp/mock-report.md", + "", + "The synthetic report has 12 lines.", +].join("\n"); + +function parseResult(result: string) { + return buildAnthropicCliBackend().parseJsonlEvent?.( + JSON.stringify({ type: "result", subtype: "success", result }), + { + backendId: "claude-cli", + backend: buildAnthropicCliBackend().config, + }, + ); +} + +describe("Claude CLI output validation", () => { + it("rejects mocked raw tool protocol returned as terminal assistant text", () => { + expect(parseResult(MOCK_RAW_TOOL_OUTPUT)).toEqual({ + kind: "result", + errorText: expect.stringContaining("raw tool protocol appeared as assistant text"), + }); + }); + + it.each(["\\u003c", "\\u003C"])( + "rejects the %s-escaped JSON form reported by upstream Claude Code", + (escapedLessThan) => { + const line = JSON.stringify({ type: "result", result: MOCK_RAW_TOOL_OUTPUT }).replaceAll( + "<", + escapedLessThan, + ); + const backend = buildAnthropicCliBackend(); + + expect( + backend.parseJsonlEvent?.(line, { backendId: backend.id, backend: backend.config }), + ).toEqual({ + kind: "result", + errorText: expect.stringContaining("raw tool protocol appeared as assistant text"), + }); + }, + ); + + it("rejects standalone protocol with CRLF line endings", () => { + expect(parseResult(MOCK_RAW_TOOL_OUTPUT.replaceAll("\n", "\r\n"))).toEqual({ + kind: "result", + errorText: expect.stringContaining("raw tool protocol appeared as assistant text"), + }); + }); + + it("rejects a complete invocation whose parameter payload exceeds the opening-tag lookahead", () => { + expect( + parseResult( + [ + "Writing the generated fixture.", + '', + `${"x".repeat(3_000)}`, + "", + ].join("\n"), + ), + ).toEqual({ + kind: "result", + errorText: expect.stringContaining("raw tool protocol appeared as assistant text"), + }); + }); + + it.each(["call", "count", "court", "Bash"])( + "rejects the upstream-observed %s prefix when the protocol block is truncated", + (prefix) => { + expect( + parseResult( + [ + "I will inspect it.", + prefix, + '', + 'wc -l /tmp/mock-report.md', + ].join("\n"), + ), + ).toEqual({ + kind: "result", + errorText: expect.stringContaining("raw tool protocol appeared as assistant text"), + }); + }, + ); + + it("does not let a later inline close token mask an observed truncated leak", () => { + expect( + parseResult( + [ + "call", + '', + 'pwd', + "Documentation may mention inline.", + ].join("\n"), + ), + ).toEqual({ + kind: "result", + errorText: expect.stringContaining("raw tool protocol appeared as assistant text"), + }); + }); + + it("does not let inline close prose before the parameter mask an observed truncated leak", () => { + expect( + parseResult( + [ + "call", + '', + "Documentation may mention inline.", + 'pwd', + ].join("\n"), + ), + ).toEqual({ + kind: "result", + errorText: expect.stringContaining("raw tool protocol appeared as assistant text"), + }); + }); + + it("continues to a valid parameter after a non-evidentiary parameter-like tag", () => { + expect( + parseResult( + [ + '', + 'ignored', + 'pwd', + "", + ].join("\n"), + ), + ).toEqual({ + kind: "result", + errorText: expect.stringContaining("raw tool protocol appeared as assistant text"), + }); + }); + + it("rejects a complete unfenced protocol example as the accepted false-positive tradeoff", () => { + expect( + parseResult( + [ + "Here is the exact raw protocol for documentation:", + '', + 'pwd', + "", + ].join("\n"), + ), + ).toEqual({ + kind: "result", + errorText: expect.stringContaining("raw tool protocol appeared as assistant text"), + }); + }); + + it.each([ + ["ordinary chat", "The washer report is ready. Here are three recommendations."], + [ + "inline protocol discussion", + 'Claude printed `pwd`.', + ], + [ + "fenced protocol example", + [ + "Example:", + "```xml", + '', + 'pwd', + "", + "```", + ].join("\n"), + ], + [ + "unterminated fenced protocol example", + [ + "Example:", + "~~~xml", + '', + 'pwd', + "", + ].join("\n"), + ], + [ + "same-line XML prose", + 'Use pwd only as an example.', + ], + [ + "line-leading protocol prose", + [ + '... denotes a call.', + '... denotes its argument.', + ].join("\n"), + ], + [ + "indented protocol example", + [ + ' ', + ' pwd', + " ", + ].join("\n"), + ], + [ + "unrelated lowercase XML", + ['', 'text', ""].join( + "\n", + ), + ], + [ + "similarly named XML elements", + [ + '', + 'pwd', + "", + ].join("\n"), + ], + [ + "similarly named XML attributes", + [ + '', + 'pwd', + "", + ].join("\n"), + ], + [ + "name text inside unrelated quoted attributes", + [ + "", + "pwd", + "", + ].join("\n"), + ], + ["standalone invoke without parameters", 'no parameter block'], + [ + "complete parameterless invocation not observed upstream", + ['', ""].join("\n"), + ], + [ + "parameter evidence belonging to a later lowercase invocation", + [ + '', + "", + '', + 'text', + "", + ].join("\n"), + ], + [ + "later lowercase invocation after a truncated parameterless invocation", + [ + "call", + '', + '', + 'text', + "", + ].join("\n"), + ], + [ + "unprefixed truncated protocol example", + [ + "Here is an incomplete protocol example:", + '', + 'pwd', + ].join("\n"), + ], + [ + "namespaced protocol example not observed upstream", + [ + '', + 'pwd', + "", + ].join("\n"), + ], + ["long ordinary report", `Summary\n\n${"Normal report text. ".repeat(20_000)}`], + ])("preserves %s", (_name, text) => { + expect(parseResult(text)).toBeNull(); + }); + + it("ignores malformed and non-terminal JSONL frames", () => { + const backend = buildAnthropicCliBackend(); + const context = { backendId: backend.id, backend: backend.config }; + + expect(backend.parseJsonlEvent?.("not json ])[^<>\r\n]*>/gu; +const RAW_PARAMETER_TAG_RE = /])[^<>\r\n]*>/gu; +const RAW_INVOKE_CLOSE_TAG_RE = /<\/invoke[ \t]*>/gu; +const RAW_PARAMETER_LOOKAHEAD_CHARS = 2_048; +const OBSERVED_TRUNCATED_LEAK_PREFIXES = new Set(["call", "count", "court"]); + +function readTagNameAttribute(tag: string, elementName: "invoke" | "parameter"): string { + const attribute = /[ \t]+([^\s=/>]+)[ \t]*=[ \t]*(?:"([^"]*)"|'([^']*)')/gy; + const tagEnd = tag.length - 1; + let cursor = elementName.length + 1; + while (cursor < tagEnd) { + attribute.lastIndex = cursor; + const match = attribute.exec(tag); + if (!match) { + return ""; + } + if (match[1] === "name") { + return match[2] ?? match[3] ?? ""; + } + cursor = attribute.lastIndex; + } + return ""; +} + +function isStandaloneLineTag(text: string, index: number): boolean { + const lineStart = text.lastIndexOf("\n", index - 1) + 1; + return /^[ ]{0,3}$/u.test(text.slice(lineStart, index)); +} + +function isStandaloneOpeningTagLine(text: string, index: number, tagLength: number): boolean { + if (!isStandaloneLineTag(text, index)) { + return false; + } + const lineEnd = text.indexOf("\n", index + tagLength); + const trailingText = text.slice(index + tagLength, lineEnd === -1 ? text.length : lineEnd); + return /^[ \t]*\r?$/u.test(trailingText); +} + +function isInsideCodeRegion(index: number, regions: CodeRegion[]): boolean { + let low = 0; + let high = regions.length - 1; + while (low <= high) { + const middle = Math.floor((low + high) / 2); + const region = regions[middle]; + if (!region) { + return false; + } + if (index < region.start) { + high = middle - 1; + } else if (index >= region.end) { + low = middle + 1; + } else { + return true; + } + } + return false; +} + +function hasObservedTruncatedLeakPrefix(text: string, index: number, toolName: string): boolean { + const previousNonEmptyLine = text + .slice(0, index) + .split(/\r?\n/gu) + .findLast((line) => line.trim().length > 0) + ?.trim(); + return ( + previousNonEmptyLine === toolName || + (previousNonEmptyLine !== undefined && + OBSERVED_TRUNCATED_LEAK_PREFIXES.has(previousNonEmptyLine)) + ); +} + +function findNextStandaloneTag( + text: string, + index: number, + regions: CodeRegion[], + pattern: RegExp, +): number | null { + const scanner = new RegExp(pattern.source, pattern.flags); + scanner.lastIndex = index; + for (let match = scanner.exec(text); match; match = scanner.exec(text)) { + if ( + isStandaloneOpeningTagLine(text, match.index, match[0].length) && + !isInsideCodeRegion(match.index, regions) + ) { + return match.index; + } + } + return null; +} + +/** Detect Claude's legacy tool protocol only when it occupies standalone assistant lines. */ +function hasClaudeRawToolInvocation(text: string): boolean { + if (!text.includes(" { + const mightContainRawToolProtocol = + (line.includes(" { }); describe("parseCliOutput", () => { + it("applies a backend JSONL hook when reparsing complete output", () => { + const parseJsonlEvent = vi.fn(() => ({ + kind: "result" as const, + errorText: "invalid request format: malformed backend result", + })); + + expect( + parseCliOutput({ + raw: JSON.stringify({ type: "result", result: "malformed" }), + backend: { command: "acme", output: "jsonl" }, + providerId: "acme-cli", + parseJsonlEvent, + outputMode: "jsonl", + }), + ).toEqual({ + text: "", + sessionId: undefined, + usage: undefined, + errorText: "invalid request format: malformed backend result", + }); + expect(parseJsonlEvent).toHaveBeenCalledOnce(); + }); + it.each([ { name: "uses streamed Claude assistant text when the result envelope is missing", diff --git a/src/agents/cli-output.ts b/src/agents/cli-output.ts index 64d89ebfa726..243a7fada216 100644 --- a/src/agents/cli-output.ts +++ b/src/agents/cli-output.ts @@ -2263,6 +2263,7 @@ export function parseCliOutput(params: { raw: string; backend: CliBackendConfig; providerId: string; + parseJsonlEvent?: CliBackendParseJsonlEvent; outputMode?: "json" | "jsonl" | "text"; fallbackSessionId?: string; }): CliOutput { @@ -2271,7 +2272,20 @@ export function parseCliOutput(params: { return { text: params.raw.trim(), sessionId: params.fallbackSessionId }; } if (outputMode === "jsonl") { - const parsed = parseCliJsonl(params.raw, params.backend, params.providerId); + let parsed: CliOutput | null; + if (params.parseJsonlEvent) { + const parser = createCliJsonlStreamingParser({ + backend: params.backend, + providerId: params.providerId, + parseJsonlEvent: params.parseJsonlEvent, + onAssistantDelta: () => {}, + }); + parser.push(params.raw); + parser.finish(); + parsed = parser.getOutput(); + } else { + parsed = parseCliJsonl(params.raw, params.backend, params.providerId); + } if (parsed) { return parsed; } diff --git a/src/agents/cli-runner/claude-live-session.capability.test.ts b/src/agents/cli-runner/claude-live-session.capability.test.ts index 4ea5c097ed3c..7b5c10396065 100644 --- a/src/agents/cli-runner/claude-live-session.capability.test.ts +++ b/src/agents/cli-runner/claude-live-session.capability.test.ts @@ -1,5 +1,6 @@ /** Claude live-session capability negotiation and input ownership tests. */ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { CliBackendParseJsonlEvent } from "../../plugins/cli-backend.types.js"; import type { getProcessSupervisor } from "../../process/supervisor/index.js"; import { buildClaudeLiveRunContext, mockClaudeLiveRun } from "../cli-runner.test-helpers.js"; import { supervisorSpawnMock } from "../cli-runner.test-support.js"; @@ -36,13 +37,21 @@ function getProcessSupervisorForTest() { }; } -function startLiveTurn(runId: string, useResume: boolean) { +function startLiveTurn( + runId: string, + useResume: boolean, + options: { + onPhase?: (phase: "send" | "resolve") => void; + parseJsonlEvent?: CliBackendParseJsonlEvent; + } = {}, +) { const context = buildClaudeLiveRunContext({ runId, timeoutMs: 60_000, liveSessionRequirement, backend: { resumeArgs: ["-p", "--resume", "{sessionId}"] }, }); + context.backendResolved.parseJsonlEvent = options.parseJsonlEvent; return runClaudeLiveSessionTurn({ context, args: context.preparedBackend.backend.args ?? [], @@ -52,11 +61,69 @@ function startLiveTurn(runId: string, useResume: boolean) { noOutputTimeoutMs: 5_000, getProcessSupervisor: getProcessSupervisorForTest, onAssistantDelta: () => {}, + onPhase: options.onPhase, cleanup: async () => {}, }); } describe("Claude live-session capability negotiation", () => { + it("rejects a malformed terminal result before background-task deferral", async () => { + const parseJsonlEvent = vi.fn((line) => { + const parsed = JSON.parse(line) as { type?: string; result?: string }; + if (parsed.type !== "result" || !parsed.result?.includes('')) { + return null; + } + return { + kind: "result", + errorText: + "Claude CLI returned malformed tool output (invalid request format): raw tool protocol appeared as assistant text.", + }; + }); + const phases: Array<"send" | "resolve"> = []; + const fixture = mockClaudeLiveRun(supervisorSpawnMock, { + events: [ + { + type: "system", + subtype: "init", + session_id: "live-malformed", + capabilities: ["msg_lifecycle_v1"], + }, + { + type: "system", + subtype: "background_tasks_changed", + tasks: [{ task_id: "task-1", task_type: "local_agent", description: "still running" }], + }, + { + type: "result", + subtype: "success", + session_id: "live-malformed", + result: [ + '', + 'pwd', + "", + ].join("\n"), + }, + ], + }); + + await expect( + startLiveTurn("run-malformed-result", false, { + parseJsonlEvent, + onPhase: (phase) => phases.push(phase), + }), + ).rejects.toMatchObject({ + name: "FailoverError", + reason: "format", + status: 400, + rawError: expect.stringContaining("raw tool protocol appeared as assistant text"), + }); + expect(phases).toEqual(["resolve"]); + expect(fixture.writes.filter((line) => line.includes('"type":"user"'))).toHaveLength(1); + expect( + parseJsonlEvent.mock.calls.filter(([line]) => line.includes('"type":"result"')), + ).toHaveLength(1); + }); + it.each([ { label: "fresh", useResume: false }, { label: "resumed", useResume: true }, diff --git a/src/agents/cli-runner/claude-live-session.ts b/src/agents/cli-runner/claude-live-session.ts index 8229f3ded393..01ba10d640ec 100644 --- a/src/agents/cli-runner/claude-live-session.ts +++ b/src/agents/cli-runner/claude-live-session.ts @@ -31,6 +31,7 @@ import { KeyedAsyncQueue } from "../../plugin-sdk/keyed-async-queue.js"; import type { CliBackendConfig, CliBackendLiveSessionRequirement, + CliBackendParseJsonlEvent, } from "../../plugins/cli-backend.types.js"; import { LEGACY_IMPLICIT_AGENT_ID, @@ -79,6 +80,7 @@ type ProcessSupervisor = ReturnType< type ManagedRun = Awaited>; type ClaudeLiveTurn = { backend: CliBackendConfig; + parseJsonlEvent?: CliBackendParseJsonlEvent; diagnosticRefs: ClaudeLiveDiagnosticRefs; /** Enclosing run abort signal; authoritative for tool terminal reason on turn failure. */ abortSignal?: AbortSignal; @@ -1410,6 +1412,7 @@ function handleClaudeLiveLine(session: ClaudeLiveSession, line: string): void { raw, backend: turn.backend, providerId: session.providerId, + parseJsonlEvent: turn.parseJsonlEvent, outputMode: "jsonl", fallbackSessionId: turn.sessionId, }); @@ -1685,6 +1688,7 @@ function createTurn(params: { }): ClaudeLiveTurn { const turn: ClaudeLiveTurn = { backend: params.context.preparedBackend.backend, + parseJsonlEvent: params.context.backendResolved.parseJsonlEvent, diagnosticRefs: { runId: params.context.params.runId, sessionId: params.context.params.sessionId, @@ -1710,6 +1714,7 @@ function createTurn(params: { streamingParser: createCliJsonlStreamingParser({ backend: params.context.preparedBackend.backend, providerId: params.context.backendResolved.id, + parseJsonlEvent: params.context.backendResolved.parseJsonlEvent, onAssistantDelta: params.onAssistantDelta, onThinkingDelta: params.onThinkingDelta, onThinkingProgress: params.onThinkingProgress,