fix(anthropic): reject leaked Claude tool protocol output (#120734)

Co-authored-by: VACInc <3279061+VACInc@users.noreply.github.com>
This commit is contained in:
Vito Cappello
2026-08-09 18:30:38 -04:00
committed by GitHub
parent aa0bad1ebe
commit 67e653e580
7 changed files with 605 additions and 2 deletions
+2
View File
@@ -11,6 +11,7 @@ import {
CLI_FRESH_WATCHDOG_DEFAULTS,
CLI_RESUME_WATCHDOG_DEFAULTS,
} from "openclaw/plugin-sdk/cli-backend";
import { parseClaudeCliJsonlEvent } from "./cli-output.js";
import {
CLAUDE_CLI_BACKEND_ID,
CLAUDE_CLI_DEFAULT_MODEL_REF,
@@ -237,6 +238,7 @@ export function buildAnthropicCliBackend(): CliBackendPlugin {
}
: undefined;
},
parseJsonlEvent: parseClaudeCliJsonlEvent,
resolveExecutionArgs: resolveClaudeCliExecutionArgs,
};
}
+293
View File
@@ -0,0 +1,293 @@
import { describe, expect, it } from "vitest";
import { buildAnthropicCliBackend } from "./cli-backend.js";
const MOCK_RAW_TOOL_OUTPUT = [
"I'll inspect the synthetic report.",
"",
'<invoke name="Bash">',
'<parameter name="command">wc -l /tmp/mock-report.md</parameter>',
'<parameter name="description">Verify the mock report</parameter>',
"</invoke>",
"",
"12 /tmp/mock-report.md",
"",
"The synthetic report has 12 lines.",
].join("\n");
function parseResult(result: string) {
return buildAnthropicCliBackend().parseJsonlEvent?.(
JSON.stringify({ type: "result", subtype: "success", result }),
{
backendId: "claude-cli",
backend: buildAnthropicCliBackend().config,
},
);
}
describe("Claude CLI output validation", () => {
it("rejects mocked raw tool protocol returned as terminal assistant text", () => {
expect(parseResult(MOCK_RAW_TOOL_OUTPUT)).toEqual({
kind: "result",
errorText: expect.stringContaining("raw tool protocol appeared as assistant text"),
});
});
it.each(["\\u003c", "\\u003C"])(
"rejects the %s-escaped JSON form reported by upstream Claude Code",
(escapedLessThan) => {
const line = JSON.stringify({ type: "result", result: MOCK_RAW_TOOL_OUTPUT }).replaceAll(
"<",
escapedLessThan,
);
const backend = buildAnthropicCliBackend();
expect(
backend.parseJsonlEvent?.(line, { backendId: backend.id, backend: backend.config }),
).toEqual({
kind: "result",
errorText: expect.stringContaining("raw tool protocol appeared as assistant text"),
});
},
);
it("rejects standalone protocol with CRLF line endings", () => {
expect(parseResult(MOCK_RAW_TOOL_OUTPUT.replaceAll("\n", "\r\n"))).toEqual({
kind: "result",
errorText: expect.stringContaining("raw tool protocol appeared as assistant text"),
});
});
it("rejects a complete invocation whose parameter payload exceeds the opening-tag lookahead", () => {
expect(
parseResult(
[
"Writing the generated fixture.",
'<invoke name="Write">',
`<parameter name="content">${"x".repeat(3_000)}</parameter>`,
"</invoke>",
].join("\n"),
),
).toEqual({
kind: "result",
errorText: expect.stringContaining("raw tool protocol appeared as assistant text"),
});
});
it.each(["call", "count", "court", "Bash"])(
"rejects the upstream-observed %s prefix when the protocol block is truncated",
(prefix) => {
expect(
parseResult(
[
"I will inspect it.",
prefix,
'<invoke name="Bash">',
'<parameter name="command">wc -l /tmp/mock-report.md',
].join("\n"),
),
).toEqual({
kind: "result",
errorText: expect.stringContaining("raw tool protocol appeared as assistant text"),
});
},
);
it("does not let a later inline close token mask an observed truncated leak", () => {
expect(
parseResult(
[
"call",
'<invoke name="Bash">',
'<parameter name="command">pwd',
"Documentation may mention </invoke> inline.",
].join("\n"),
),
).toEqual({
kind: "result",
errorText: expect.stringContaining("raw tool protocol appeared as assistant text"),
});
});
it("does not let inline close prose before the parameter mask an observed truncated leak", () => {
expect(
parseResult(
[
"call",
'<invoke name="Bash">',
"Documentation may mention </invoke> inline.",
'<parameter name="command">pwd',
].join("\n"),
),
).toEqual({
kind: "result",
errorText: expect.stringContaining("raw tool protocol appeared as assistant text"),
});
});
it("continues to a valid parameter after a non-evidentiary parameter-like tag", () => {
expect(
parseResult(
[
'<invoke name="Bash">',
'<parameter data-name="example">ignored</parameter>',
'<parameter name="command">pwd</parameter>',
"</invoke>",
].join("\n"),
),
).toEqual({
kind: "result",
errorText: expect.stringContaining("raw tool protocol appeared as assistant text"),
});
});
it("rejects a complete unfenced protocol example as the accepted false-positive tradeoff", () => {
expect(
parseResult(
[
"Here is the exact raw protocol for documentation:",
'<invoke name="Bash">',
'<parameter name="command">pwd</parameter>',
"</invoke>",
].join("\n"),
),
).toEqual({
kind: "result",
errorText: expect.stringContaining("raw tool protocol appeared as assistant text"),
});
});
it.each([
["ordinary chat", "The washer report is ready. Here are three recommendations."],
[
"inline protocol discussion",
'Claude printed `<invoke name="Bash"><parameter name="command">pwd</parameter></invoke>`.',
],
[
"fenced protocol example",
[
"Example:",
"```xml",
'<invoke name="Bash">',
'<parameter name="command">pwd</parameter>',
"</invoke>",
"```",
].join("\n"),
],
[
"unterminated fenced protocol example",
[
"Example:",
"~~~xml",
'<invoke name="Bash">',
'<parameter name="command">pwd</parameter>',
"</invoke>",
].join("\n"),
],
[
"same-line XML prose",
'Use <invoke name="Bash"><parameter name="command">pwd</parameter></invoke> only as an example.',
],
[
"line-leading protocol prose",
[
'<invoke name="Bash">...</invoke> denotes a call.',
'<parameter name="command">...</parameter> denotes its argument.',
].join("\n"),
],
[
"indented protocol example",
[
' <invoke name="Bash">',
' <parameter name="command">pwd</parameter>',
" </invoke>",
].join("\n"),
],
[
"unrelated lowercase XML",
['<invoke name="transform">', '<parameter name="input">text</parameter>', "</invoke>"].join(
"\n",
),
],
[
"similarly named XML elements",
[
'<invoke-example name="Bash">',
'<parameter-example name="command">pwd</parameter-example>',
"</invoke-example>",
].join("\n"),
],
[
"similarly named XML attributes",
[
'<invoke data-name="Bash">',
'<parameter data-name="command">pwd</parameter>',
"</invoke>",
].join("\n"),
],
[
"name text inside unrelated quoted attributes",
[
"<invoke description=\"tool name='Bash'\">",
"<parameter description=\"argument name='command'\">pwd</parameter>",
"</invoke>",
].join("\n"),
],
["standalone invoke without parameters", '<invoke name="Bash">no parameter block</invoke>'],
[
"complete parameterless invocation not observed upstream",
['<invoke name="mcp__server__get_current_time">', "</invoke>"].join("\n"),
],
[
"parameter evidence belonging to a later lowercase invocation",
[
'<invoke name="Bash">',
"</invoke>",
'<invoke name="transform">',
'<parameter name="input">text</parameter>',
"</invoke>",
].join("\n"),
],
[
"later lowercase invocation after a truncated parameterless invocation",
[
"call",
'<invoke name="Bash">',
'<invoke name="transform">',
'<parameter name="input">text</parameter>',
"</invoke>",
].join("\n"),
],
[
"unprefixed truncated protocol example",
[
"Here is an incomplete protocol example:",
'<invoke name="Bash">',
'<parameter name="command">pwd',
].join("\n"),
],
[
"namespaced protocol example not observed upstream",
[
'<antml:invoke name="Bash">',
'<antml:parameter name="command">pwd</antml:parameter>',
"</antml:invoke>",
].join("\n"),
],
["long ordinary report", `Summary\n\n${"Normal report text. ".repeat(20_000)}`],
])("preserves %s", (_name, text) => {
expect(parseResult(text)).toBeNull();
});
it("ignores malformed and non-terminal JSONL frames", () => {
const backend = buildAnthropicCliBackend();
const context = { backendId: backend.id, backend: backend.config };
expect(backend.parseJsonlEvent?.("not json <invoke <parameter", context)).toBeNull();
expect(
backend.parseJsonlEvent?.(
JSON.stringify({ type: "assistant", result: MOCK_RAW_TOOL_OUTPUT }),
context,
),
).toBeNull();
});
});
+199
View File
@@ -0,0 +1,199 @@
import type { CliBackendParseJsonlEvent } from "openclaw/plugin-sdk/cli-backend";
import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
import { findCodeRegions, type CodeRegion } from "openclaw/plugin-sdk/text-chunking";
const CLAUDE_RAW_TOOL_OUTPUT_ERROR =
"Claude CLI returned malformed tool output (invalid request format): raw tool protocol appeared as assistant text. OpenClaw refused to persist or deliver it.";
const RAW_INVOKE_TAG_RE = /<invoke(?=[ \t>])[^<>\r\n]*>/gu;
const RAW_PARAMETER_TAG_RE = /<parameter(?=[ \t>])[^<>\r\n]*>/gu;
const RAW_INVOKE_CLOSE_TAG_RE = /<\/invoke[ \t]*>/gu;
const RAW_PARAMETER_LOOKAHEAD_CHARS = 2_048;
const OBSERVED_TRUNCATED_LEAK_PREFIXES = new Set(["call", "count", "court"]);
function readTagNameAttribute(tag: string, elementName: "invoke" | "parameter"): string {
const attribute = /[ \t]+([^\s=/>]+)[ \t]*=[ \t]*(?:"([^"]*)"|'([^']*)')/gy;
const tagEnd = tag.length - 1;
let cursor = elementName.length + 1;
while (cursor < tagEnd) {
attribute.lastIndex = cursor;
const match = attribute.exec(tag);
if (!match) {
return "";
}
if (match[1] === "name") {
return match[2] ?? match[3] ?? "";
}
cursor = attribute.lastIndex;
}
return "";
}
function isStandaloneLineTag(text: string, index: number): boolean {
const lineStart = text.lastIndexOf("\n", index - 1) + 1;
return /^[ ]{0,3}$/u.test(text.slice(lineStart, index));
}
function isStandaloneOpeningTagLine(text: string, index: number, tagLength: number): boolean {
if (!isStandaloneLineTag(text, index)) {
return false;
}
const lineEnd = text.indexOf("\n", index + tagLength);
const trailingText = text.slice(index + tagLength, lineEnd === -1 ? text.length : lineEnd);
return /^[ \t]*\r?$/u.test(trailingText);
}
function isInsideCodeRegion(index: number, regions: CodeRegion[]): boolean {
let low = 0;
let high = regions.length - 1;
while (low <= high) {
const middle = Math.floor((low + high) / 2);
const region = regions[middle];
if (!region) {
return false;
}
if (index < region.start) {
high = middle - 1;
} else if (index >= region.end) {
low = middle + 1;
} else {
return true;
}
}
return false;
}
function hasObservedTruncatedLeakPrefix(text: string, index: number, toolName: string): boolean {
const previousNonEmptyLine = text
.slice(0, index)
.split(/\r?\n/gu)
.findLast((line) => line.trim().length > 0)
?.trim();
return (
previousNonEmptyLine === toolName ||
(previousNonEmptyLine !== undefined &&
OBSERVED_TRUNCATED_LEAK_PREFIXES.has(previousNonEmptyLine))
);
}
function findNextStandaloneTag(
text: string,
index: number,
regions: CodeRegion[],
pattern: RegExp,
): number | null {
const scanner = new RegExp(pattern.source, pattern.flags);
scanner.lastIndex = index;
for (let match = scanner.exec(text); match; match = scanner.exec(text)) {
if (
isStandaloneOpeningTagLine(text, match.index, match[0].length) &&
!isInsideCodeRegion(match.index, regions)
) {
return match.index;
}
}
return null;
}
/** Detect Claude's legacy tool protocol only when it occupies standalone assistant lines. */
function hasClaudeRawToolInvocation(text: string): boolean {
if (!text.includes("<invoke") || !text.includes("<parameter")) {
return false;
}
const codeRegions = findCodeRegions(text);
RAW_INVOKE_TAG_RE.lastIndex = 0;
for (const match of text.matchAll(RAW_INVOKE_TAG_RE)) {
const index = match.index;
const toolName = readTagNameAttribute(match[0], "invoke");
if (
(!/^[A-Z]/u.test(toolName) && !toolName.startsWith("mcp__")) ||
!isStandaloneOpeningTagLine(text, index, match[0].length) ||
isInsideCodeRegion(index, codeRegions)
) {
continue;
}
const invocationBodyStart = index + match[0].length;
const invokeCloseIndex = findNextStandaloneTag(
text,
invocationBodyStart,
codeRegions,
RAW_INVOKE_CLOSE_TAG_RE,
);
const nextInvokeIndex = findNextStandaloneTag(
text,
invocationBodyStart,
codeRegions,
RAW_INVOKE_TAG_RE,
);
const completeInvokeCloseIndex =
invokeCloseIndex !== null && (nextInvokeIndex === null || invokeCloseIndex < nextInvokeIndex)
? invokeCloseIndex
: null;
const parameterSearchEnd = Math.min(
text.length,
invocationBodyStart + RAW_PARAMETER_LOOKAHEAD_CHARS,
completeInvokeCloseIndex ?? text.length,
nextInvokeIndex ?? text.length,
);
const parameterText = text.slice(invocationBodyStart, parameterSearchEnd);
let parameterIndex: number | null = null;
for (const parameterMatch of parameterText.matchAll(RAW_PARAMETER_TAG_RE)) {
const candidateIndex = invocationBodyStart + parameterMatch.index;
if (
readTagNameAttribute(parameterMatch[0], "parameter") &&
isStandaloneLineTag(text, candidateIndex) &&
!isInsideCodeRegion(candidateIndex, codeRegions)
) {
parameterIndex = candidateIndex;
break;
}
}
if (parameterIndex === null) {
continue;
}
if (
// Upstream shows both complete blocks and prefix-led truncated blocks. Requiring a close
// misses the latter; requiring a prefix misses the complete leak reproduced in this PR.
// Complete unfenced examples remain the accepted false positive and surface as format errors.
completeInvokeCloseIndex !== null ||
(completeInvokeCloseIndex === null && hasObservedTruncatedLeakPrefix(text, index, toolName))
) {
return true;
}
}
return false;
}
/** Reject malformed terminal Claude results before the generic CLI runner accepts them as prose. */
export const parseClaudeCliJsonlEvent: CliBackendParseJsonlEvent = (line) => {
const mightContainRawToolProtocol =
(line.includes("<invoke") ||
line.includes("\\u003cinvoke") ||
line.includes("\\u003Cinvoke")) &&
(line.includes("<parameter") ||
line.includes("\\u003cparameter") ||
line.includes("\\u003Cparameter"));
if (!mightContainRawToolProtocol) {
return null;
}
let parsed: unknown;
try {
parsed = JSON.parse(line);
} catch {
return null;
}
if (
!isRecord(parsed) ||
parsed.type !== "result" ||
typeof parsed.result !== "string" ||
!hasClaudeRawToolInvocation(parsed.result)
) {
return null;
}
// Core classifies this exact phrase as a format failure, enabling safe fallback instead of
// accepting the malformed terminal result as a successful assistant response.
return { kind: "result", errorText: CLAUDE_RAW_TOOL_OUTPUT_ERROR };
};
+23
View File
@@ -1211,6 +1211,29 @@ describe("parseCliJsonl", () => {
});
describe("parseCliOutput", () => {
it("applies a backend JSONL hook when reparsing complete output", () => {
const parseJsonlEvent = vi.fn(() => ({
kind: "result" as const,
errorText: "invalid request format: malformed backend result",
}));
expect(
parseCliOutput({
raw: JSON.stringify({ type: "result", result: "malformed" }),
backend: { command: "acme", output: "jsonl" },
providerId: "acme-cli",
parseJsonlEvent,
outputMode: "jsonl",
}),
).toEqual({
text: "",
sessionId: undefined,
usage: undefined,
errorText: "invalid request format: malformed backend result",
});
expect(parseJsonlEvent).toHaveBeenCalledOnce();
});
it.each([
{
name: "uses streamed Claude assistant text when the result envelope is missing",
+15 -1
View File
@@ -2263,6 +2263,7 @@ export function parseCliOutput(params: {
raw: string;
backend: CliBackendConfig;
providerId: string;
parseJsonlEvent?: CliBackendParseJsonlEvent;
outputMode?: "json" | "jsonl" | "text";
fallbackSessionId?: string;
}): CliOutput {
@@ -2271,7 +2272,20 @@ export function parseCliOutput(params: {
return { text: params.raw.trim(), sessionId: params.fallbackSessionId };
}
if (outputMode === "jsonl") {
const parsed = parseCliJsonl(params.raw, params.backend, params.providerId);
let parsed: CliOutput | null;
if (params.parseJsonlEvent) {
const parser = createCliJsonlStreamingParser({
backend: params.backend,
providerId: params.providerId,
parseJsonlEvent: params.parseJsonlEvent,
onAssistantDelta: () => {},
});
parser.push(params.raw);
parser.finish();
parsed = parser.getOutput();
} else {
parsed = parseCliJsonl(params.raw, params.backend, params.providerId);
}
if (parsed) {
return parsed;
}
@@ -1,5 +1,6 @@
/** Claude live-session capability negotiation and input ownership tests. */
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { CliBackendParseJsonlEvent } from "../../plugins/cli-backend.types.js";
import type { getProcessSupervisor } from "../../process/supervisor/index.js";
import { buildClaudeLiveRunContext, mockClaudeLiveRun } from "../cli-runner.test-helpers.js";
import { supervisorSpawnMock } from "../cli-runner.test-support.js";
@@ -36,13 +37,21 @@ function getProcessSupervisorForTest() {
};
}
function startLiveTurn(runId: string, useResume: boolean) {
function startLiveTurn(
runId: string,
useResume: boolean,
options: {
onPhase?: (phase: "send" | "resolve") => void;
parseJsonlEvent?: CliBackendParseJsonlEvent;
} = {},
) {
const context = buildClaudeLiveRunContext({
runId,
timeoutMs: 60_000,
liveSessionRequirement,
backend: { resumeArgs: ["-p", "--resume", "{sessionId}"] },
});
context.backendResolved.parseJsonlEvent = options.parseJsonlEvent;
return runClaudeLiveSessionTurn({
context,
args: context.preparedBackend.backend.args ?? [],
@@ -52,11 +61,69 @@ function startLiveTurn(runId: string, useResume: boolean) {
noOutputTimeoutMs: 5_000,
getProcessSupervisor: getProcessSupervisorForTest,
onAssistantDelta: () => {},
onPhase: options.onPhase,
cleanup: async () => {},
});
}
describe("Claude live-session capability negotiation", () => {
it("rejects a malformed terminal result before background-task deferral", async () => {
const parseJsonlEvent = vi.fn<CliBackendParseJsonlEvent>((line) => {
const parsed = JSON.parse(line) as { type?: string; result?: string };
if (parsed.type !== "result" || !parsed.result?.includes('<invoke name="Bash">')) {
return null;
}
return {
kind: "result",
errorText:
"Claude CLI returned malformed tool output (invalid request format): raw tool protocol appeared as assistant text.",
};
});
const phases: Array<"send" | "resolve"> = [];
const fixture = mockClaudeLiveRun(supervisorSpawnMock, {
events: [
{
type: "system",
subtype: "init",
session_id: "live-malformed",
capabilities: ["msg_lifecycle_v1"],
},
{
type: "system",
subtype: "background_tasks_changed",
tasks: [{ task_id: "task-1", task_type: "local_agent", description: "still running" }],
},
{
type: "result",
subtype: "success",
session_id: "live-malformed",
result: [
'<invoke name="Bash">',
'<parameter name="command">pwd</parameter>',
"</invoke>",
].join("\n"),
},
],
});
await expect(
startLiveTurn("run-malformed-result", false, {
parseJsonlEvent,
onPhase: (phase) => phases.push(phase),
}),
).rejects.toMatchObject({
name: "FailoverError",
reason: "format",
status: 400,
rawError: expect.stringContaining("raw tool protocol appeared as assistant text"),
});
expect(phases).toEqual(["resolve"]);
expect(fixture.writes.filter((line) => line.includes('"type":"user"'))).toHaveLength(1);
expect(
parseJsonlEvent.mock.calls.filter(([line]) => line.includes('"type":"result"')),
).toHaveLength(1);
});
it.each([
{ label: "fresh", useResume: false },
{ label: "resumed", useResume: true },
@@ -31,6 +31,7 @@ import { KeyedAsyncQueue } from "../../plugin-sdk/keyed-async-queue.js";
import type {
CliBackendConfig,
CliBackendLiveSessionRequirement,
CliBackendParseJsonlEvent,
} from "../../plugins/cli-backend.types.js";
import {
LEGACY_IMPLICIT_AGENT_ID,
@@ -79,6 +80,7 @@ type ProcessSupervisor = ReturnType<
type ManagedRun = Awaited<ReturnType<ProcessSupervisor["spawn"]>>;
type ClaudeLiveTurn = {
backend: CliBackendConfig;
parseJsonlEvent?: CliBackendParseJsonlEvent;
diagnosticRefs: ClaudeLiveDiagnosticRefs;
/** Enclosing run abort signal; authoritative for tool terminal reason on turn failure. */
abortSignal?: AbortSignal;
@@ -1410,6 +1412,7 @@ function handleClaudeLiveLine(session: ClaudeLiveSession, line: string): void {
raw,
backend: turn.backend,
providerId: session.providerId,
parseJsonlEvent: turn.parseJsonlEvent,
outputMode: "jsonl",
fallbackSessionId: turn.sessionId,
});
@@ -1685,6 +1688,7 @@ function createTurn(params: {
}): ClaudeLiveTurn {
const turn: ClaudeLiveTurn = {
backend: params.context.preparedBackend.backend,
parseJsonlEvent: params.context.backendResolved.parseJsonlEvent,
diagnosticRefs: {
runId: params.context.params.runId,
sessionId: params.context.params.sessionId,
@@ -1710,6 +1714,7 @@ function createTurn(params: {
streamingParser: createCliJsonlStreamingParser({
backend: params.context.preparedBackend.backend,
providerId: params.context.backendResolved.id,
parseJsonlEvent: params.context.backendResolved.parseJsonlEvent,
onAssistantDelta: params.onAssistantDelta,
onThinkingDelta: params.onThinkingDelta,
onThinkingProgress: params.onThinkingProgress,