diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index b296977d96b6..a6b1d12256dc 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -376,16 +376,12 @@ src/agents/embedded-agent-runner/replay-history.ts src/agents/embedded-agent-runner/run.incomplete-turn.test.ts src/agents/embedded-agent-runner/run.overflow-compaction.harness.ts src/agents/embedded-agent-runner/run/attempt-spawn-workspace.test-support.ts -src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.test.ts -src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.ts src/agents/embedded-agent-runner/run/attempt.spawn-workspace.context-engine.test.ts src/agents/embedded-agent-runner/run/attempt.test.ts src/agents/embedded-agent-runner/run/attempt.tool-call-argument-repair.ts src/agents/embedded-agent-runner/run/attempt.tool-call-normalization.test.ts src/agents/embedded-agent-runner/run/attempt.tool-call-normalization.ts src/agents/embedded-agent-runner/run/incomplete-turn.ts -src/agents/embedded-agent-runner/run/payloads.errors.test.ts -src/agents/embedded-agent-runner/run/payloads.ts src/agents/embedded-agent-runner/runs.ts src/agents/embedded-agent-runner/thinking.test.ts src/agents/embedded-agent-runner/tool-result-context-guard.test.ts diff --git a/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.test.ts b/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.test.ts index 304ac01b9bd9..284fc69f8225 100644 --- a/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.test.ts +++ b/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.test.ts @@ -1,39 +1,22 @@ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; // Coverage for model-call diagnostic events around attempt stream functions. -import { isRecord } from "@openclaw/normalization-core/record-coerce"; import type { StreamFn } from "openclaw/plugin-sdk/agent-core"; import { createRequireRecord } from "openclaw/plugin-sdk/test-fixtures"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { useAutoCleanupTempDirTracker } from "../../../../test/helpers/temp-dir.js"; import { onInternalDiagnosticEvent, onTrustedInternalDiagnosticEvent, resetDiagnosticEventsForTest, - setDiagnosticsEnabledForProcess, type DiagnosticEventPrivateData, type DiagnosticEventPayload, - waitForDiagnosticEventsDrained, } from "../../../infra/diagnostic-events.js"; -import { isCoreSemanticRunProgressDiagnosticMetadata } from "../../../infra/diagnostic-semantic-run-progress.js"; import { createDiagnosticTraceContext } from "../../../infra/diagnostic-trace-context.js"; -import { registerDiagnosticTracePropagationBridge } from "../../../infra/diagnostic-trace-propagation.js"; import { - getDiagnosticSessionActivitySnapshot, - markDiagnosticEmbeddedRunStarted, resetDiagnosticRunActivityForTest, startDiagnosticRunActivityTracking, } from "../../../logging/diagnostic-run-activity.js"; -import { - initializeGlobalHookRunner, - resetGlobalHookRunner, -} from "../../../plugins/hook-runner-global.js"; -import { createHookRunnerWithRegistry } from "../../../plugins/hooks.test-fixtures.js"; -import { withEnvAsync } from "../../../test-utils/env.js"; +import { resetGlobalHookRunner } from "../../../plugins/hook-runner-global.js"; import { wrapStreamFnWithDiagnosticModelCallEvents } from "./attempt.model-diagnostic-events.js"; -const tempDirs = useAutoCleanupTempDirTracker(afterEach); - async function collectModelCallEvents(run: () => Promise): Promise { // Diagnostics are emitted asynchronously; collect only public model-call // events and flush one tick after the stream completes. @@ -80,30 +63,6 @@ async function collectTrustedModelCallEvents(run: () => Promise): Promise< } } -async function collectSemanticProgressEvents(run: () => Promise) { - const events: DiagnosticEventPayload[] = []; - const stop = onInternalDiagnosticEvent((event, metadata) => { - if ( - isCoreSemanticRunProgressDiagnosticMetadata(metadata) && - event.type === "run.progress" && - event.reason === "model_call:semantic_result" - ) { - events.push(event); - } - }); - try { - await run(); - await waitForDiagnosticEventsDrained(); - return events; - } finally { - stop(); - } -} - -function assistantResult(stopReason: string, content: unknown[]) { - return { role: "assistant", stopReason, content }; -} - async function drain(stream: AsyncIterable): Promise { // Force stream iteration so completion events include response byte and timing // accounting. @@ -114,14 +73,6 @@ async function drain(stream: AsyncIterable): Promise { const requireRecord = createRequireRecord("record", "expected-label-object-capitalized"); -function readRecordField(record: Record, key: string, label: string) { - const value = record[key]; - if (!isRecord(value)) { - throw new Error(`Expected ${label} to be an object`); - } - return value; -} - function expectNumberField(record: Record, key: string) { expect(typeof record[key]).toBe("number"); } @@ -130,34 +81,7 @@ function getEvent(events: readonly DiagnosticEventPayload[], index: number) { return requireRecord(events[index], `event ${index}`); } -function requireMockRecordArg( - mock: ReturnType, - callIndex: number, - argIndex: number, - label: string, -) { - return requireRecord(mock.mock.calls[callIndex]?.[argIndex], label); -} - -async function collectProviderTimelineEvents(run: () => Promise) { - const root = tempDirs.make("openclaw-provider-timeline-"); - const timelinePath = join(root, "timeline.jsonl"); - await withEnvAsync( - { - OPENCLAW_DIAGNOSTICS: "1", - OPENCLAW_DIAGNOSTICS_TIMELINE_PATH: timelinePath, - }, - run, - ); - return readFileSync(timelinePath, "utf8") - .trim() - .split("\n") - .filter(Boolean) - .map((line) => requireRecord(JSON.parse(line), "provider timeline event")) - .filter((event) => event.type === "provider.request"); -} - -describe("wrapStreamFnWithDiagnosticModelCallEvents", () => { +describe("wrapStreamFnWithDiagnosticModelCallEvents stream proxy", () => { beforeEach(() => { resetDiagnosticEventsForTest(); resetDiagnosticRunActivityForTest(); @@ -251,844 +175,6 @@ describe("wrapStreamFnWithDiagnosticModelCallEvents", () => { expect(JSON.stringify(events)).not.toContain("sk-test-secret-value"); }); - it.each([ - { - name: "visible text", - result: assistantResult("stop", [{ type: "text", text: "done" }]), - expected: 1, - }, - { - name: "tool-use call", - result: assistantResult("toolUse", [ - { type: "toolCall", id: "call-1", name: "read", arguments: { path: "README.md" } }, - ]), - expected: 1, - }, - { - name: "error text", - result: assistantResult("error", [{ type: "text", text: "provider failed" }]), - expected: 0, - }, - { - name: "aborted text", - result: assistantResult("aborted", [{ type: "text", text: "partial" }]), - expected: 0, - }, - { - name: "reasoning only", - result: assistantResult("stop", [{ type: "thinking", thinking: "working" }]), - expected: 0, - }, - { - name: "blank text", - result: assistantResult("stop", [{ type: "text", text: " \n" }]), - expected: 0, - }, - { - name: "non-executable tool block", - result: assistantResult("stop", [ - { type: "toolCall", id: "call-1", name: "read", arguments: {} }, - ]), - expected: 0, - }, - { - name: "malformed tool-use call", - result: assistantResult("toolUse", [{ type: "toolCall", id: "", name: "read" }]), - expected: 0, - }, - ])("emits semantic progress once for $name final results", async ({ result, expected }) => { - const stream = { - async *[Symbol.asyncIterator]() {}, - result: async () => result, - }; - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (() => stream) as unknown as StreamFn, - { - runId: "run-semantic-result", - sessionId: "session-semantic-result", - provider: "openai", - model: "gpt-5.4", - trace: createDiagnosticTraceContext(), - nextCallId: () => "call-semantic-result", - }, - ); - - const events = await collectSemanticProgressEvents(async () => { - const observed = wrapped({} as never, {} as never, {} as never) as unknown as typeof stream; - await observed.result(); - await observed.result(); - }); - - expect(events).toHaveLength(expected); - }); - - it("orders semantic results between repeated request observations", async () => { - const ref = { - sessionId: "session-semantic-order", - sessionKey: "agent:main:semantic-order", - }; - const runId = "run-semantic-order"; - const results = [ - assistantResult("error", [{ type: "text", text: "retry one" }]), - assistantResult("error", [{ type: "text", text: "retry two" }]), - assistantResult("stop", [{ type: "text", text: "made progress" }]), - assistantResult("error", [{ type: "text", text: "retry after progress" }]), - ]; - let callSequence = 0; - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (() => { - const result = results.shift(); - return { - async *[Symbol.asyncIterator]() {}, - result: async () => result, - }; - }) as unknown as StreamFn, - { - ...ref, - runId, - provider: "openai", - model: "gpt-5.4", - trace: createDiagnosticTraceContext(), - nextCallId: () => `${runId}:${(callSequence += 1)}`, - }, - ); - markDiagnosticEmbeddedRunStarted({ ...ref, runId }); - - const repeatedRequestAges: Array = []; - for (let index = 0; index < 4; index += 1) { - const observed = wrapped({} as never, {} as never, {} as never) as unknown as { - result: () => Promise; - }; - await observed.result(); - await waitForDiagnosticEventsDrained(); - repeatedRequestAges.push( - getDiagnosticSessionActivitySnapshot(ref).repeatedRequestNoProgressAgeMs, - ); - } - - expect(repeatedRequestAges).toEqual([undefined, expect.any(Number), undefined, undefined]); - - expect(getDiagnosticSessionActivitySnapshot(ref)).toMatchObject({ - hasActiveEmbeddedRun: true, - repeatedRequestNoProgressAgeMs: undefined, - }); - }); - - it("emits one successful provider timeline event for result and iterator completion", async () => { - let now = Date.parse("2026-07-09T18:30:00.000Z"); - vi.spyOn(Date, "now").mockImplementation(() => now); - async function* stream() { - yield { type: "text", text: "ok" }; - } - const originalStream = stream() as unknown as AsyncIterable & { - result: () => Promise; - }; - originalStream.result = async () => { - now += 125; - return "kept"; - }; - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (() => originalStream) as unknown as StreamFn, - { - runId: "run-timeline-success", - provider: "openai", - model: "gpt-5.5", - api: "openai-responses", - transport: "http", - trace: createDiagnosticTraceContext(), - nextCallId: () => "call-timeline-success", - }, - ); - - const events = await collectProviderTimelineEvents(async () => { - const returned = wrapped( - {} as never, - {} as never, - {} as never, - ) as unknown as typeof originalStream; - await returned.result(); - await drain(returned); - }); - - expect(events).toHaveLength(1); - expect(events[0]).toMatchObject({ - type: "provider.request", - name: "provider.request", - timestamp: "2026-07-09T18:30:00.000Z", - runId: "run-timeline-success", - spanId: "call-timeline-success", - durationMs: 125, - provider: "openai", - operation: "openai-responses", - ok: true, - attributes: { - model: "gpt-5.5", - api: "openai-responses", - transport: "http", - }, - }); - expect(events[0]?.status).toBeUndefined(); - }); - - it("records provider response status and preserves the original response callback", async () => { - const originalOnResponse = vi.fn(async () => undefined); - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (( - model: Parameters[0], - _context: Parameters[1], - options: Parameters[2], - ) => { - return options?.onResponse?.({ status: 200, headers: { "x-request-id": "req-1" } }, model); - }) as unknown as StreamFn, - { - runId: "run-timeline-status", - provider: "openai", - model: "gpt-5.6", - api: "openai-responses", - transport: "http", - trace: createDiagnosticTraceContext(), - nextCallId: () => "call-timeline-status", - }, - ); - - const events = await collectProviderTimelineEvents(async () => { - await wrapped( - { id: "gpt-5.6" } as never, - {} as never, - { - onResponse: originalOnResponse, - } as never, - ); - }); - - expect(originalOnResponse).toHaveBeenCalledWith( - { status: 200, headers: { "x-request-id": "req-1" } }, - { id: "gpt-5.6" }, - ); - expect(events).toHaveLength(1); - expect(events[0]).toMatchObject({ - type: "provider.request", - ok: true, - status: 200, - }); - }); - - it("writes Unicode-safe bounded attributes to the provider timeline JSONL", async () => { - const modelPrefix = "m".repeat(255); - const exactBoundary = "b".repeat(256); - const events = await collectProviderTimelineEvents(async () => { - const cases: Array<{ callId: string; model: string }> = [ - { callId: "call-timeline-unicode-boundary", model: `${modelPrefix}😀tail` }, - { callId: "call-timeline-exact-boundary", model: exactBoundary }, - ]; - for (const { callId, model } of cases) { - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (() => undefined) as unknown as StreamFn, - { - runId: "run-timeline-unicode-boundary", - provider: "openai", - model, - trace: createDiagnosticTraceContext(), - nextCallId: () => callId, - }, - ); - await wrapped({} as never, {} as never, {} as never); - } - }); - - expect(events).toHaveLength(2); - const splitBoundaryModel = readRecordField(events[0]!, "attributes", "attributes").model; - expect(splitBoundaryModel).toBe(modelPrefix); - expect(splitBoundaryModel).toHaveLength(255); - expect(splitBoundaryModel).not.toContain("�"); - expect(splitBoundaryModel).not.toMatch(/[\uD800-\uDFFF]/u); - const exactBoundaryModel = readRecordField(events[1]!, "attributes", "attributes").model; - expect(exactBoundaryModel).toBe(exactBoundary); - expect(exactBoundaryModel).toHaveLength(256); - }); - - it("emits one failed provider timeline event for a thrown model call", async () => { - let now = Date.parse("2026-07-09T18:31:00.000Z"); - vi.spyOn(Date, "now").mockImplementation(() => now); - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (() => { - now += 75; - throw new Error("provider failed"); - }) as unknown as StreamFn, - { - runId: "run-timeline-error", - provider: "anthropic", - model: "claude-sonnet-4-6", - transport: "sse", - trace: createDiagnosticTraceContext(), - nextCallId: () => "call-timeline-error", - }, - ); - - const events = await collectProviderTimelineEvents(async () => { - expect(() => wrapped({} as never, {} as never, {} as never)).toThrow("provider failed"); - }); - - expect(events).toHaveLength(1); - expect(events[0]).toMatchObject({ - type: "provider.request", - name: "provider.request", - timestamp: "2026-07-09T18:31:00.000Z", - runId: "run-timeline-error", - spanId: "call-timeline-error", - durationMs: 75, - provider: "anthropic", - operation: "sse", - ok: false, - attributes: { - model: "claude-sonnet-4-6", - transport: "sse", - }, - }); - }); - - it("records a non-2xx provider response on a failed model call", async () => { - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (() => { - throw Object.assign(new Error("rate limited"), { status: 429 }); - }) as unknown as StreamFn, - { - runId: "run-timeline-http-error", - provider: "openai", - model: "gpt-5.6", - api: "openai-responses", - transport: "http", - trace: createDiagnosticTraceContext(), - nextCallId: () => "call-timeline-http-error", - }, - ); - - const events = await collectProviderTimelineEvents(async () => { - expect(() => wrapped({} as never, {} as never, {} as never)).toThrow("rate limited"); - }); - - expect(events).toHaveLength(1); - expect(events[0]).toMatchObject({ - type: "provider.request", - ok: false, - status: 429, - }); - }); - - it("keeps an observed response status when the terminal error has another status", async () => { - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (( - model: Parameters[0], - _context: Parameters[1], - options: Parameters[2], - ) => { - void options?.onResponse?.({ status: 503, headers: {} }, model); - throw Object.assign(new Error("retry failed"), { status: 429 }); - }) as unknown as StreamFn, - { - runId: "run-timeline-observed-http-error", - provider: "openai", - model: "gpt-5.6", - api: "openai-responses", - transport: "http", - trace: createDiagnosticTraceContext(), - nextCallId: () => "call-timeline-observed-http-error", - }, - ); - - const events = await collectProviderTimelineEvents(async () => { - expect(() => wrapped({} as never, {} as never, {} as never)).toThrow("retry failed"); - }); - - expect(events).toHaveLength(1); - expect(events[0]).toMatchObject({ - type: "provider.request", - ok: false, - status: 503, - }); - }); - - it("updates diagnostic run activity from throttled stream chunks", async () => { - let now = 1_000_000; - vi.spyOn(Date, "now").mockImplementation(() => now); - async function* stream() { - yield { type: "text_delta", delta: "first" }; - yield { type: "text_delta", delta: "second" }; - yield { type: "text_delta", delta: "third" }; - } - const runProgressEvents: DiagnosticEventPayload[] = []; - const stop = onInternalDiagnosticEvent((event) => { - if (event.type === "run.progress") { - runProgressEvents.push(event); - } - }); - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (() => stream()) as unknown as StreamFn, - { - runId: "run-1", - sessionKey: "session-key", - sessionId: "session-id", - provider: "vllm", - model: "qwen/qwen3.5-9b", - trace: createDiagnosticTraceContext(), - nextCallId: () => "call-stream", - }, - ); - - const returned = wrapped({} as never, {} as never, {} as never) as AsyncIterable; - const iterator = returned[Symbol.asyncIterator](); - - try { - await iterator.next(); - await waitForDiagnosticEventsDrained(); - let snapshot = getDiagnosticSessionActivitySnapshot({ - sessionKey: "session-key", - sessionId: "session-id", - }); - expect(snapshot.activeWorkKind).toBe("model_call"); - expect(snapshot.lastProgressReason).toBe("model_call:stream_progress"); - expect(snapshot.lastProgressAgeMs).toBe(0); - expect(runProgressEvents).toHaveLength(1); - - now += 10_000; - await iterator.next(); - await waitForDiagnosticEventsDrained(); - snapshot = getDiagnosticSessionActivitySnapshot({ - sessionKey: "session-key", - sessionId: "session-id", - }); - expect(snapshot.lastProgressReason).toBe("model_call:stream_progress"); - expect(snapshot.lastProgressAgeMs).toBe(0); - expect(runProgressEvents).toHaveLength(1); - - now += 30_000; - await iterator.next(); - await waitForDiagnosticEventsDrained(); - snapshot = getDiagnosticSessionActivitySnapshot({ - sessionKey: "session-key", - sessionId: "session-id", - }); - expect(snapshot.lastProgressReason).toBe("model_call:stream_progress"); - expect(snapshot.lastProgressAgeMs).toBe(0); - expect(runProgressEvents).toHaveLength(2); - expect(runProgressEvents.every((event) => event.type === "run.progress")).toBe(true); - expect(runProgressEvents.every((event) => !("progressKind" in event))).toBe(true); - } finally { - await iterator.return?.(); - await waitForDiagnosticEventsDrained(); - stop(); - } - }); - - it("does not retain stream progress activity when diagnostics are disabled", async () => { - setDiagnosticsEnabledForProcess(false); - const runProgressEvents: DiagnosticEventPayload[] = []; - const stop = onInternalDiagnosticEvent((event) => { - if (event.type === "run.progress") { - runProgressEvents.push(event); - } - }); - async function* stream() { - yield { type: "text_delta", delta: "first" }; - yield { type: "text_delta", delta: "second" }; - } - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (() => stream()) as unknown as StreamFn, - { - runId: "run-1", - sessionKey: "session-key", - sessionId: "session-id", - provider: "vllm", - model: "qwen/qwen3.5-9b", - trace: createDiagnosticTraceContext(), - nextCallId: () => "call-disabled-diagnostics", - }, - ); - - try { - await drain(wrapped({} as never, {} as never, {} as never) as AsyncIterable); - await waitForDiagnosticEventsDrained(); - } finally { - stop(); - } - - expect( - getDiagnosticSessionActivitySnapshot({ - sessionKey: "session-key", - sessionId: "session-id", - }), - ).toEqual({}); - expect(runProgressEvents).toEqual([]); - }); - - it("counts async onPayload replacements instead of raw payload content", async () => { - async function* stream() { - yield { type: "text_delta", delta: "safe" }; - } - const originalPayload = { input: "secret sk-original-secret" }; - const replacementPayload = { input: "redacted" }; - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (async ( - model: Parameters[0], - _context: Parameters[1], - options: Parameters[2], - ) => { - await options?.onPayload?.(originalPayload, model); - return stream(); - }) as unknown as StreamFn, - { - runId: "run-1", - provider: "openai", - model: "gpt-5.4", - trace: createDiagnosticTraceContext(), - nextCallId: () => "call-payload", - }, - ); - - const events = await collectModelCallEvents(async () => { - const streamResult = await wrapped({} as never, {} as never, { - onPayload: async () => replacementPayload, - }); - await drain(streamResult as unknown as AsyncIterable); - }); - - const completedEvent = getEvent(events, 1); - expect(completedEvent.type).toBe("model.call.completed"); - expect(completedEvent.callId).toBe("call-payload"); - expect(completedEvent.requestPayloadBytes).toBe( - Buffer.byteLength(JSON.stringify(replacementPayload), "utf8"), - ); - expectNumberField(completedEvent, "responseStreamBytes"); - expectNumberField(completedEvent, "timeToFirstByteMs"); - expect(JSON.stringify(events)).not.toContain("sk-original-secret"); - }); - - it("counts text deltas without serializing full partial snapshots", async () => { - const serializedPartial = vi.fn(() => { - throw new Error("partial snapshot should not be serialized for text deltas"); - }); - async function* stream() { - yield { - type: "text_delta", - contentIndex: 0, - delta: "a", - partial: { - toJSON: serializedPartial, - role: "assistant", - content: [{ type: "text", text: "a".repeat(200_000) }], - }, - }; - yield { - type: "text_delta", - contentIndex: 0, - delta: "bc", - partial: { - toJSON: serializedPartial, - role: "assistant", - content: [{ type: "text", text: "abc".repeat(200_000) }], - }, - }; - } - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (() => stream()) as unknown as StreamFn, - { - runId: "run-1", - provider: "openai", - model: "gpt-5.4", - trace: createDiagnosticTraceContext(), - nextCallId: () => "call-delta-bytes", - }, - ); - - const events = await collectModelCallEvents(async () => { - await drain(wrapped({} as never, {} as never, {} as never) as AsyncIterable); - }); - - const completedEvent = getEvent(events, 1); - expect(completedEvent.type).toBe("model.call.completed"); - expect(completedEvent.responseStreamBytes).toBe(Buffer.byteLength("abc", "utf8")); - expect(serializedPartial).not.toHaveBeenCalled(); - }); - - it("keeps streams alive when diagnostic byte inspection cannot read a chunk", async () => { - const opaqueChunk = new Proxy( - {}, - { - get(_target, property) { - if (property === "then") { - return undefined; - } - throw new Error("chunk should not be inspected"); - }, - }, - ); - async function* stream() { - yield opaqueChunk; - yield { type: "text_delta", delta: "ok" }; - } - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (() => stream()) as unknown as StreamFn, - { - runId: "run-1", - provider: "openai", - model: "gpt-5.4", - trace: createDiagnosticTraceContext(), - nextCallId: () => "call-opaque-chunk", - }, - ); - - const chunks: unknown[] = []; - const events = await collectModelCallEvents(async () => { - for await (const chunk of wrapped( - {} as never, - {} as never, - {} as never, - ) as AsyncIterable) { - chunks.push(chunk); - } - }); - - expect(chunks).toHaveLength(2); - expect(chunks[0]).toBe(opaqueChunk); - expect(chunks[1]).toEqual({ type: "text_delta", delta: "ok" }); - const completedEvent = getEvent(events, 1); - expect(completedEvent.type).toBe("model.call.completed"); - expect(completedEvent.responseStreamBytes).toBe(Buffer.byteLength("ok", "utf8")); - }); - - it("captures model input, tools, and output only when content capture is enabled", async () => { - const assistant = { - role: "assistant", - content: [{ type: "text", text: "trace reply" }], - api: "openai-responses", - provider: "openai", - model: "gpt-5.4", - usage: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 2 }, - stopReason: "stop", - timestamp: 1, - }; - async function* stream() { - yield { type: "done", reason: "stop", message: assistant }; - } - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (() => stream()) as unknown as StreamFn, - { - runId: "run-1", - provider: "openai", - model: "gpt-5.4", - trace: createDiagnosticTraceContext(), - contentCapture: { - inputMessages: true, - outputMessages: true, - toolInputs: false, - toolOutputs: false, - systemPrompt: true, - toolDefinitions: true, - anyModelContent: true, - }, - nextCallId: () => "call-content", - }, - ); - - const inputMessages = [{ role: "user", content: "trace prompt", timestamp: 1 }]; - const tools = [{ name: "lookup", description: "Lookup data", parameters: { type: "object" } }]; - const events = await collectTrustedModelCallEvents(async () => { - const streamResult = wrapped( - {} as never, - { - systemPrompt: "trace system", - messages: inputMessages, - tools, - } as never, - {}, - ); - await drain(streamResult as unknown as AsyncIterable); - }); - - const startedEvent = getEvent( - events.map((entry) => entry.event), - 0, - ); - expect(startedEvent.type).toBe("model.call.started"); - expect(startedEvent.inputMessages).toBeUndefined(); - expect(startedEvent.systemPrompt).toBeUndefined(); - expect(startedEvent.toolDefinitions).toBeUndefined(); - expect(events[0]?.privateData.modelContent?.inputMessages).toEqual(inputMessages); - expect(events[0]?.privateData.modelContent?.systemPrompt).toBe("trace system"); - expect(events[0]?.privateData.modelContent?.toolDefinitions).toEqual(tools); - const completedEvent = getEvent( - events.map((entry) => entry.event), - 1, - ); - expect(completedEvent.type).toBe("model.call.completed"); - expect(completedEvent.outputMessages).toBeUndefined(); - expect(events[1]?.privateData.modelContent?.inputMessages).toEqual(inputMessages); - expect(events[1]?.privateData.modelContent?.outputMessages).toEqual([assistant]); - }); - - it("emits safe prompt stats and per-call usage without content capture", async () => { - const assistant = { - role: "assistant", - content: [{ type: "text", text: "trace reply" }], - usage: { - input: 11, - output: 7, - cacheRead: 3, - cacheWrite: 2, - reasoningTokens: 5, - totalTokens: 28, - }, - timestamp: 1, - }; - async function* stream() { - yield { type: "done", reason: "stop", message: assistant }; - } - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (() => stream()) as unknown as StreamFn, - { - runId: "run-1", - provider: "openai", - model: "gpt-5.4", - trace: createDiagnosticTraceContext(), - nextCallId: () => "call-stats", - }, - ); - - const inputMessages = [{ role: "user", content: "private prompt text", timestamp: 1 }]; - const tools = [ - { name: "lookup", description: "private tool description", parameters: { type: "object" } }, - ]; - const systemPrompt = "private system prompt"; - const events = await collectModelCallEvents(async () => { - const streamResult = wrapped( - {} as never, - { - systemPrompt, - messages: inputMessages, - tools, - } as never, - {}, - ); - await drain(streamResult as unknown as AsyncIterable); - }); - - const startedEvent = getEvent(events, 0); - const completedEvent = getEvent(events, 1); - const expectedPromptStats = { - inputMessagesCount: inputMessages.length, - inputMessagesChars: JSON.stringify(inputMessages).length, - systemPromptChars: systemPrompt.length, - toolDefinitionsCount: tools.length, - toolDefinitionsChars: JSON.stringify(tools).length, - totalChars: - JSON.stringify(inputMessages).length + systemPrompt.length + JSON.stringify(tools).length, - }; - expect(startedEvent.promptStats).toEqual(expectedPromptStats); - expect(completedEvent.promptStats).toEqual(expectedPromptStats); - expect(completedEvent.usage).toEqual({ - input: 11, - output: 7, - cacheRead: 3, - cacheWrite: 2, - reasoningTokens: 5, - total: 28, - promptTokens: 16, - }); - expect(JSON.stringify(events)).not.toContain("private prompt text"); - expect(JSON.stringify(events)).not.toContain("private system prompt"); - expect(JSON.stringify(events)).not.toContain("private tool description"); - }); - - it("captures per-call usage from terminal error events", async () => { - // Aborted/error streams terminate with an `error` event carrying the final - // AssistantMessage and its usage. Iterating to completion without awaiting - // result() must still surface per-call usage, matching the `done` path and - // the usage field already emitted on model.call.error and its OTel span. - const assistant = { - role: "assistant", - content: [{ type: "text", text: "partial reply" }], - usage: { - input: 11, - output: 7, - cacheRead: 3, - cacheWrite: 2, - reasoningTokens: 5, - totalTokens: 28, - }, - stopReason: "aborted", - timestamp: 1, - }; - async function* stream() { - yield { type: "error", reason: "aborted", error: assistant }; - } - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (() => stream()) as unknown as StreamFn, - { - runId: "run-1", - provider: "openrouter", - model: "openrouter/auto", - trace: createDiagnosticTraceContext(), - nextCallId: () => "call-error-usage", - }, - ); - - const events = await collectModelCallEvents(async () => { - await drain(wrapped({} as never, {} as never, {} as never) as AsyncIterable); - }); - - // An in-band error event is data, not a throw, so iteration completes - // normally; the per-call usage rides on the terminal completion event. - const completedEvent = getEvent(events, 1); - expect(completedEvent.type).toBe("model.call.completed"); - expect(completedEvent.usage).toEqual({ - input: 11, - output: 7, - cacheRead: 3, - cacheWrite: 2, - reasoningTokens: 5, - total: 28, - promptTokens: 16, - }); - }); - - it("skips prompt stat computation when diagnostics are disabled", async () => { - // Prompt stats are only attached to diagnostic events; when diagnostics are - // off those events are dropped, so the JSON.stringify of input messages and - // tool definitions must not run on the model-call hot path. - setDiagnosticsEnabledForProcess(false); - let promptInspected = false; - const streamContext = { - systemPrompt: "system", - get messages() { - promptInspected = true; - return [{ role: "user", content: "x", timestamp: 1 }]; - }, - get tools() { - promptInspected = true; - return [{ name: "lookup", description: "d", parameters: { type: "object" } }]; - }, - }; - async function* stream() { - yield { type: "text_delta", delta: "ok" }; - } - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (() => stream()) as unknown as StreamFn, - { - runId: "run-1", - provider: "openai", - model: "gpt-5.4", - trace: createDiagnosticTraceContext(), - nextCallId: () => "call-disabled-prompt-stats", - }, - ); - - await drain( - wrapped({} as never, streamContext as never, {} as never) as AsyncIterable, - ); - - expect(promptInspected).toBe(false); - }); - it("captures output and completes when callers only await stream.result()", async () => { const assistant = { role: "assistant", @@ -1218,105 +304,6 @@ describe("wrapStreamFnWithDiagnosticModelCallEvents", () => { ]); }); - it("propagates the trusted model-call traceparent without mutating caller headers", async () => { - async function* stream() { - yield { type: "text", text: "ok" }; - } - const capturedOptions: Array[2]> = []; - const callerOptions = { - headers: { - "X-Custom": "kept", - TraceParent: "00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-01", - }, - sessionId: "provider-session", - }; - const exportedTrace = createDiagnosticTraceContext({ - traceId: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - spanId: "bbbbbbbbbbbbbbbb", - traceFlags: "01", - }); - registerDiagnosticTracePropagationBridge({ - resolveTraceContext: () => exportedTrace, - }); - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (( - _model: Parameters[0], - _context: Parameters[1], - options: Parameters[2], - ) => { - capturedOptions.push(options); - return stream(); - }) as unknown as StreamFn, - { - runId: "run-1", - provider: "openai", - model: "gpt-5.4", - trace: createDiagnosticTraceContext({ - traceId: "4bf92f3577b34da6a3ce929d0e0e4736", - spanId: "00f067aa0ba902b7", - traceFlags: "01", - }), - nextCallId: () => "call-traceparent", - }, - ); - - await drain( - wrapped({} as never, {} as never, callerOptions) as unknown as AsyncIterable, - ); - - expect(capturedOptions).toHaveLength(1); - expect(capturedOptions[0]).not.toBe(callerOptions); - const capturedOption = requireRecord(capturedOptions[0], "captured stream options"); - expect(capturedOption.sessionId).toBe("provider-session"); - expect(capturedOption.requestId).toBe("call-traceparent"); - const headers = readRecordField(capturedOption, "headers", "captured stream headers"); - expect(headers["X-Custom"]).toBe("kept"); - expect(headers.traceparent).toBe(`00-${exportedTrace.traceId}-${exportedTrace.spanId}-01`); - expect(capturedOptions[0]?.headers).not.toHaveProperty("TraceParent"); - expect(callerOptions.headers).toEqual({ - "X-Custom": "kept", - TraceParent: "00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-01", - }); - }); - - it("removes caller traceparent when the active exporter cannot resolve a span", async () => { - async function* stream() { - yield { type: "text", text: "ok" }; - } - const capturedOptions: Array[2]> = []; - registerDiagnosticTracePropagationBridge({ - resolveTraceContext: () => undefined, - }); - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (( - _model: Parameters[0], - _context: Parameters[1], - options: Parameters[2], - ) => { - capturedOptions.push(options); - return stream(); - }) as unknown as StreamFn, - { - runId: "run-1", - provider: "openai", - model: "gpt-5.4", - trace: createDiagnosticTraceContext(), - nextCallId: () => "call-no-exported-span", - }, - ); - - await drain( - wrapped({} as never, {} as never, { - headers: { - "X-Custom": "kept", - TraceParent: "00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-01", - }, - }) as unknown as AsyncIterable, - ); - - expect(capturedOptions[0]?.headers).toEqual({ "X-Custom": "kept" }); - }); - it("emits error events when stream iteration fails", async () => { const requestId = "req_provider_123"; const stream = { @@ -1356,47 +343,6 @@ describe("wrapStreamFnWithDiagnosticModelCallEvents", () => { expect(JSON.stringify(events[1])).not.toContain(requestId); }); - it("adds failure kind and memory diagnostics for terminated model calls", async () => { - const stream = { - [Symbol.asyncIterator]() { - return { - async next(): Promise> { - throw new Error("terminated"); - }, - }; - }, - }; - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (() => stream) as unknown as StreamFn, - { - runId: "run-1", - provider: "lmstudio", - model: "qwen/qwen3.5-9b", - trace: createDiagnosticTraceContext(), - nextCallId: () => "call-terminated", - }, - ); - - const events = await collectModelCallEvents(async () => { - await expect( - drain(wrapped({} as never, {} as never, {} as never) as AsyncIterable), - ).rejects.toThrow("terminated"); - }); - - expect(events.map((event) => event.type)).toEqual(["model.call.started", "model.call.error"]); - const errorEvent = getEvent(events, 1); - expect(errorEvent.type).toBe("model.call.error"); - expect(errorEvent.callId).toBe("call-terminated"); - expect(errorEvent.errorCategory).toBe("Error"); - expect(errorEvent.failureKind).toBe("terminated"); - const memory = readRecordField(errorEvent, "memory", "error event memory"); - expectNumberField(memory, "rssBytes"); - expectNumberField(memory, "heapTotalBytes"); - expectNumberField(memory, "heapUsedBytes"); - expectNumberField(memory, "externalBytes"); - expectNumberField(memory, "arrayBuffersBytes"); - }); - it("does not mutate non-configurable provider streams", async () => { const stream = {}; Object.defineProperty(stream, Symbol.asyncIterator, { @@ -1433,125 +379,6 @@ describe("wrapStreamFnWithDiagnosticModelCallEvents", () => { ]); }); - it("fires frozen sanitized model-call plugin hooks", async () => { - const started = vi.fn(); - const ended = vi.fn(); - const { registry } = createHookRunnerWithRegistry([ - { hookName: "model_call_started", handler: started }, - { hookName: "model_call_ended", handler: ended }, - ]); - initializeGlobalHookRunner(registry); - const secretChunk = "secret response with Bearer sk-test-secret-value"; - - async function* stream() { - yield { type: "text", text: secretChunk }; - } - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (() => stream()) as unknown as StreamFn, - { - runId: "run-1", - sessionKey: "session-key", - sessionId: "session-id", - provider: "openai", - model: "gpt-5.4", - api: "openai-responses", - transport: "http", - contextTokenBudget: 150_000, - contextWindowSource: "agentContextTokens", - contextWindowReferenceTokens: 200_000, - trace: createDiagnosticTraceContext(), - nextCallId: () => "call-hook", - }, - ); - - const events = await collectModelCallEvents(async () => { - await drain(wrapped({} as never, {} as never, {} as never) as AsyncIterable); - }); - await new Promise((resolve) => { - setImmediate(resolve); - }); - - expect(events.map((event) => event.type)).toEqual([ - "model.call.started", - "model.call.completed", - ]); - const startedEvent = requireMockRecordArg(started, 0, 0, "started hook event"); - expect(startedEvent.runId).toBe("run-1"); - expect(startedEvent.callId).toBe("call-hook"); - expect(startedEvent.sessionKey).toBe("session-key"); - expect(startedEvent.sessionId).toBe("session-id"); - expect(startedEvent.provider).toBe("openai"); - expect(startedEvent.model).toBe("gpt-5.4"); - expect(startedEvent.api).toBe("openai-responses"); - expect(startedEvent.transport).toBe("http"); - expect(startedEvent.contextTokenBudget).toBe(150_000); - expect(startedEvent.contextWindowSource).toBe("agentContextTokens"); - expect(startedEvent.contextWindowReferenceTokens).toBe(200_000); - const startedCtx = requireMockRecordArg(started, 0, 1, "started hook context"); - expect(startedCtx.runId).toBe("run-1"); - expect(startedCtx.sessionKey).toBe("session-key"); - expect(startedCtx.sessionId).toBe("session-id"); - expect(startedCtx.modelProviderId).toBe("openai"); - expect(startedCtx.modelId).toBe("gpt-5.4"); - expect(startedCtx.contextTokenBudget).toBe(150_000); - expect(startedCtx.contextWindowSource).toBe("agentContextTokens"); - expect(startedCtx.contextWindowReferenceTokens).toBe(200_000); - const endedEvent = requireMockRecordArg(ended, 0, 0, "ended hook event"); - expect(endedEvent.runId).toBe("run-1"); - expect(endedEvent.callId).toBe("call-hook"); - expect(endedEvent.outcome).toBe("completed"); - expect(endedEvent.contextTokenBudget).toBe(150_000); - expect(endedEvent.contextWindowSource).toBe("agentContextTokens"); - expect(endedEvent.contextWindowReferenceTokens).toBe(200_000); - expectNumberField(endedEvent, "durationMs"); - expectNumberField(endedEvent, "responseStreamBytes"); - expectNumberField(endedEvent, "timeToFirstByteMs"); - const endedCtx = requireMockRecordArg(ended, 0, 1, "ended hook context"); - expect(endedCtx.runId).toBe("run-1"); - expect(Object.isFrozen(startedEvent)).toBe(true); - expect(Object.isFrozen(startedCtx)).toBe(true); - expect(Object.isFrozen(startedCtx.trace)).toBe(true); - expect(JSON.stringify([started.mock.calls, ended.mock.calls])).not.toContain(secretChunk); - }); - - it("keeps core model-call diagnostics while suppressing finalization plugin hooks", async () => { - const started = vi.fn(); - const ended = vi.fn(); - const { registry } = createHookRunnerWithRegistry([ - { hookName: "model_call_started", handler: started }, - { hookName: "model_call_ended", handler: ended }, - ]); - initializeGlobalHookRunner(registry); - async function* stream() { - yield { type: "text", text: "final answer" }; - } - const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( - (() => stream()) as unknown as StreamFn, - { - runId: "run-finalization", - provider: "openai", - model: "gpt-5.4", - trace: createDiagnosticTraceContext(), - nextCallId: () => "call-finalization", - suppressPluginHooks: true, - }, - ); - - const events = await collectModelCallEvents(async () => { - await drain(wrapped({} as never, {} as never, {} as never) as AsyncIterable); - }); - await new Promise((resolve) => { - setImmediate(resolve); - }); - - expect(events.map((event) => event.type)).toEqual([ - "model.call.started", - "model.call.completed", - ]); - expect(started).not.toHaveBeenCalled(); - expect(ended).not.toHaveBeenCalled(); - }); - it("emits completed events when stream consumption stops early", async () => { async function* stream() { yield { type: "text", text: "first" }; @@ -1589,4 +416,3 @@ describe("wrapStreamFnWithDiagnosticModelCallEvents", () => { expect(events[1]).not.toHaveProperty("errorCategory"); }); }); -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.ts b/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.ts index 1a2dac6183a3..23c180a3a25a 100644 --- a/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.ts +++ b/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.ts @@ -1,422 +1,17 @@ import { isPromiseLike } from "@openclaw/normalization-core/promise-like"; import { isRecord } from "@openclaw/normalization-core/record-coerce"; -import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; /** * Emits diagnostic model-call events around embedded-agent stream functions. */ -import { fireAndForgetBoundedHook } from "../../../hooks/fire-and-forget.js"; -import { - diagnosticErrorCategory, - diagnosticErrorFailureKind, - diagnosticHttpStatusCode, - diagnosticProviderRequestIdHash, -} from "../../../infra/diagnostic-error-metadata.js"; -import { - areDiagnosticsEnabledForProcess, - emitTrustedDiagnosticEvent, - type DiagnosticEventInput, - type DiagnosticModelCallContent, - type DiagnosticMemoryUsage, - emitTrustedDiagnosticEventWithPrivateData, -} from "../../../infra/diagnostic-events.js"; -import { - cloneDiagnosticContentValue, - type DiagnosticModelContentCapturePolicy, -} from "../../../infra/diagnostic-llm-content.js"; -import { emitCoreModelRequestStartedDiagnosticEvent } from "../../../infra/diagnostic-model-request.js"; -import { emitCoreSemanticRunProgressDiagnosticEvent } from "../../../infra/diagnostic-semantic-run-progress.js"; -import { - createChildDiagnosticTraceContext, - freezeDiagnosticTraceContext, - type DiagnosticTraceContext, -} from "../../../infra/diagnostic-trace-context.js"; -import { formatPropagatedDiagnosticTraceparent } from "../../../infra/diagnostic-trace-propagation.js"; -import { emitDiagnosticsTimelineEvent } from "../../../infra/diagnostics-timeline.js"; -import { markDiagnosticRunProgress } from "../../../logging/diagnostic-run-activity.js"; -import { getGlobalHookRunner } from "../../../plugins/hook-runner-global.js"; -import type { - PluginHookAgentContext, - PluginHookContextWindowSource, - PluginHookModelCallEndedEvent, - PluginHookModelCallStartedEvent, -} from "../../../plugins/hook-types.js"; import type { StreamFn } from "../../runtime/index.js"; -import { derivePromptTokens, normalizeUsage, type UsageLike } from "../../usage.js"; +import { + createModelLifecycle, + type ModelCallDiagnosticContext, + type ModelCallLifecycle, +} from "./attempt.model-diagnostic-lifecycle.js"; +import { createModelObserver } from "./attempt.model-diagnostic-observation.js"; -type ModelCallDiagnosticContext = { - runId: string; - sessionKey?: string; - sessionId?: string; - provider: string; - model: string; - api?: string; - transport?: string; - contextTokenBudget?: number; - contextWindowSource?: PluginHookContextWindowSource; - contextWindowReferenceTokens?: number; - trace: DiagnosticTraceContext; - contentCapture?: DiagnosticModelContentCapturePolicy; - nextCallId: () => string; - onStarted?: () => void; - suppressPluginHooks?: boolean; -}; - -type ModelCallEventBase = Omit< - Extract, - "type" ->; -type ModelCallErrorFields = Pick< - Extract, - "errorCategory" | "failureKind" | "memory" | "upstreamRequestIdHash" ->; -type ModelCallEndedHookFields = Pick< - PluginHookModelCallEndedEvent, - | "durationMs" - | "outcome" - | "errorCategory" - | "requestPayloadBytes" - | "responseStreamBytes" - | "timeToFirstByteMs" - | "failureKind" - | "upstreamRequestIdHash" ->; -type ModelCallSizeTimingFields = Pick< - Extract, - "requestPayloadBytes" | "responseStreamBytes" | "timeToFirstByteMs" ->; -type ModelCallPromptStats = NonNullable< - Extract["promptStats"] ->; -type ModelCallUsage = NonNullable< - Extract["usage"] ->; -type ModelCallObservationState = { - requestPayloadBytes?: number; - responseStatus?: number; - responseStreamBytes: number; - timeToFirstByteMs?: number; - modelContent?: DiagnosticModelCallContent; - outputMessages?: unknown[]; - usage?: ModelCallUsage; - contentCapture?: DiagnosticModelContentCapturePolicy; - lastStreamProgressAt?: number; - semanticProgressEmitted?: boolean; - terminalEventEmitted?: boolean; - suppressPluginHooks?: boolean; -}; - -const MODEL_CALL_STREAM_PROGRESS_INTERVAL_MS = 30_000; -const MODEL_CALL_STREAM_PROGRESS_REASON = "model_call:stream_progress"; -const MODEL_CALL_SEMANTIC_PROGRESS_REASON = "model_call:semantic_result"; const MODEL_CALL_STREAM_RETURN_TIMEOUT_MS = 1000; -const TRACEPARENT_HEADER_NAME = "traceparent"; -const TIMELINE_ATTRIBUTE_MAX_LENGTH = 256; -type ModelCallStreamOptions = Parameters[2]; - -function utf8JsonByteLength(value: unknown): number | undefined { - try { - return Buffer.byteLength(JSON.stringify(value), "utf8"); - } catch { - return undefined; - } -} - -function assignRequestPayloadBytes(state: ModelCallObservationState, payload: unknown): void { - const bytes = utf8JsonByteLength(payload); - if (bytes !== undefined) { - state.requestPayloadBytes = bytes; - } -} - -function utf8StringByteLength(value: string): number { - return Buffer.byteLength(value, "utf8"); -} - -function jsonCharLength(value: unknown): number | undefined { - try { - return JSON.stringify(value)?.length; - } catch { - return undefined; - } -} - -function streamDeltaByteLength(chunk: Record): number | undefined { - const type = chunk.type; - if ( - (type === "text_delta" || type === "thinking_delta" || type === "toolcall_delta") && - typeof chunk.delta === "string" - ) { - return utf8StringByteLength(chunk.delta); - } - return undefined; -} - -function responseStreamChunkByteLengthUnchecked(chunk: unknown): number | undefined { - if (!isRecord(chunk)) { - return utf8JsonByteLength(chunk); - } - const deltaBytes = streamDeltaByteLength(chunk); - if (deltaBytes !== undefined) { - return deltaBytes; - } - if (!("partial" in chunk)) { - return utf8JsonByteLength(chunk); - } - // Plain stream deltas can carry an accumulated partial snapshot. Byte metrics - // count the new stream payload, not the answer-so-far replay. - const { partial: _partial, ...snapshotlessChunk } = chunk; - return utf8JsonByteLength(snapshotlessChunk); -} - -function responseStreamChunkByteLength(chunk: unknown): number | undefined { - try { - return responseStreamChunkByteLengthUnchecked(chunk); - } catch { - return undefined; - } -} - -function streamContextModelContentFields( - policy: DiagnosticModelContentCapturePolicy | undefined, - streamContext: unknown, -): DiagnosticModelCallContent | undefined { - if (!policy?.anyModelContent || !isRecord(streamContext)) { - return undefined; - } - const content = { - ...(policy.inputMessages && Array.isArray(streamContext.messages) - ? { inputMessages: cloneDiagnosticContentValue(streamContext.messages) } - : {}), - ...(policy.systemPrompt && typeof streamContext.systemPrompt === "string" - ? { systemPrompt: streamContext.systemPrompt } - : {}), - ...(policy.toolDefinitions && Array.isArray(streamContext.tools) - ? { toolDefinitions: cloneDiagnosticContentValue(streamContext.tools) } - : {}), - }; - return Object.keys(content).length > 0 ? content : undefined; -} - -function streamContextModelPromptStats(streamContext: unknown): ModelCallPromptStats | undefined { - if (!isRecord(streamContext)) { - return undefined; - } - const messages = Array.isArray(streamContext.messages) ? streamContext.messages : undefined; - const tools = Array.isArray(streamContext.tools) ? streamContext.tools : undefined; - const systemPrompt = - typeof streamContext.systemPrompt === "string" ? streamContext.systemPrompt : undefined; - const inputMessagesChars = messages ? jsonCharLength(messages) : undefined; - const toolDefinitionsChars = tools ? jsonCharLength(tools) : undefined; - const systemPromptChars = systemPrompt?.length; - if ( - messages === undefined && - tools === undefined && - systemPromptChars === undefined && - inputMessagesChars === undefined && - toolDefinitionsChars === undefined - ) { - return undefined; - } - const totalChars = - (inputMessagesChars ?? 0) + (systemPromptChars ?? 0) + (toolDefinitionsChars ?? 0); - return { - ...(messages ? { inputMessagesCount: messages.length } : {}), - ...(inputMessagesChars !== undefined ? { inputMessagesChars } : {}), - ...(systemPromptChars !== undefined ? { systemPromptChars } : {}), - ...(tools ? { toolDefinitionsCount: tools.length } : {}), - ...(toolDefinitionsChars !== undefined ? { toolDefinitionsChars } : {}), - totalChars, - }; -} - -function normalizedModelCallUsage(rawUsage: unknown): ModelCallUsage | undefined { - if (!isRecord(rawUsage)) { - return undefined; - } - const usage = normalizeUsage(rawUsage as UsageLike); - if (!usage) { - return undefined; - } - const promptTokens = derivePromptTokens(usage); - return { - ...usage, - ...(promptTokens !== undefined ? { promptTokens } : {}), - }; -} - -function observeModelCallUsage(state: ModelCallObservationState, value: unknown): void { - if (!isRecord(value)) { - return; - } - let rawUsage: unknown; - try { - rawUsage = value.usage; - } catch { - return; - } - const usage = normalizedModelCallUsage(rawUsage); - if (usage) { - state.usage = usage; - } -} - -function observeOutputMessageContent(state: ModelCallObservationState, chunk: unknown): void { - if (!isRecord(chunk)) { - return; - } - let type: unknown; - let message: unknown; - try { - type = chunk.type; - message = type === "done" ? chunk.message : type === "error" ? chunk.error : undefined; - } catch { - return; - } - // Terminal events carry the final AssistantMessage with usage — `done` for - // success, `error` for aborted/error streams. Capture usage from either so - // iterated error-terminated calls still report the per-call usage that the - // model.call.error event and its OTel span already expose. - if (message !== undefined) { - observeModelCallUsage(state, message); - if (state.contentCapture?.outputMessages) { - state.outputMessages = [cloneDiagnosticContentValue(message)]; - } - } -} - -function observeResultMessageContent( - state: ModelCallObservationState, - startedAt: number, - result: unknown, -): void { - state.timeToFirstByteMs ??= Math.max(0, Date.now() - startedAt); - observeModelCallUsage(state, result); - if (state.contentCapture?.outputMessages && state.outputMessages === undefined) { - state.outputMessages = [cloneDiagnosticContentValue(result)]; - } - if (state.responseStreamBytes === 0) { - const bytes = utf8JsonByteLength(result); - if (bytes !== undefined) { - state.responseStreamBytes = bytes; - } - } -} - -function isNormalizedToolCall(value: unknown): boolean { - if (!isRecord(value) || value.type !== "toolCall") { - return false; - } - return ( - typeof value.id === "string" && - value.id.trim().length > 0 && - typeof value.name === "string" && - value.name.trim().length > 0 && - isRecord(value.arguments) - ); -} - -function isSemanticModelCallResult(result: unknown): boolean { - try { - if ( - !isRecord(result) || - result.role !== "assistant" || - result.stopReason === "error" || - result.stopReason === "aborted" || - !Array.isArray(result.content) - ) { - return false; - } - const hasExecutableToolCall = - result.stopReason === "toolUse" && result.content.some(isNormalizedToolCall); - return ( - hasExecutableToolCall || - result.content.some( - (item) => - isRecord(item) && - item.type === "text" && - typeof item.text === "string" && - item.text.trim().length > 0, - ) - ); - } catch { - return false; - } -} - -function maybeEmitModelCallSemanticProgress( - eventBase: ModelCallEventBase, - state: ModelCallObservationState, - result: unknown, -): void { - if (state.semanticProgressEmitted || !isSemanticModelCallResult(result)) { - return; - } - state.semanticProgressEmitted = true; - emitCoreSemanticRunProgressDiagnosticEvent({ - runId: eventBase.runId, - ...(eventBase.sessionKey ? { sessionKey: eventBase.sessionKey } : {}), - ...(eventBase.sessionId ? { sessionId: eventBase.sessionId } : {}), - reason: MODEL_CALL_SEMANTIC_PROGRESS_REASON, - }); -} - -function observeResponseChunk( - state: ModelCallObservationState, - startedAt: number, - chunk: unknown, -): void { - state.timeToFirstByteMs ??= Math.max(0, Date.now() - startedAt); - observeOutputMessageContent(state, chunk); - const bytes = responseStreamChunkByteLength(chunk); - if (bytes !== undefined) { - state.responseStreamBytes += bytes; - } -} - -function maybeEmitModelCallStreamProgress( - eventBase: ModelCallEventBase, - state: ModelCallObservationState, -): void { - if (!areDiagnosticsEnabledForProcess()) { - return; - } - const now = Date.now(); - const progressFields = { - runId: eventBase.runId, - ...(eventBase.sessionKey ? { sessionKey: eventBase.sessionKey } : {}), - ...(eventBase.sessionId ? { sessionId: eventBase.sessionId } : {}), - reason: MODEL_CALL_STREAM_PROGRESS_REASON, - }; - markDiagnosticRunProgress(progressFields); - if ( - state.lastStreamProgressAt !== undefined && - now - state.lastStreamProgressAt < MODEL_CALL_STREAM_PROGRESS_INTERVAL_MS - ) { - return; - } - state.lastStreamProgressAt = now; - // Streaming providers, local or remote, are expected to produce chunks or - // heartbeat-style progress. The in-memory freshness clock is refreshed for - // each chunk, while diagnostic events are throttled so token streams do not - // spam observers; silent/non-streaming calls remain recoverable after the - // configured stuck-session timeout. - emitTrustedDiagnosticEvent({ - type: "run.progress", - ...progressFields, - }); -} - -function modelCallSizeTimingFields(state: ModelCallObservationState): ModelCallSizeTimingFields { - return { - ...(state.requestPayloadBytes !== undefined - ? { requestPayloadBytes: state.requestPayloadBytes } - : {}), - ...(state.responseStreamBytes > 0 ? { responseStreamBytes: state.responseStreamBytes } : {}), - ...(state.timeToFirstByteMs !== undefined - ? { timeToFirstByteMs: state.timeToFirstByteMs } - : {}), - }; -} function asyncIteratorFactory(value: unknown): (() => AsyncIterator) | undefined { if (value === null || typeof value !== "object") { @@ -433,314 +28,6 @@ function asyncIteratorFactory(value: unknown): (() => AsyncIterator) | } } -function baseModelCallEvent( - ctx: ModelCallDiagnosticContext, - callId: string, - trace: DiagnosticTraceContext, - promptStats: ModelCallPromptStats | undefined, -): ModelCallEventBase { - return { - runId: ctx.runId, - callId, - ...(ctx.sessionKey && { sessionKey: ctx.sessionKey }), - ...(ctx.sessionId && { sessionId: ctx.sessionId }), - provider: ctx.provider, - model: ctx.model, - ...(ctx.api && { api: ctx.api }), - ...(ctx.transport && { transport: ctx.transport }), - observationUnit: "request", - ...(ctx.contextTokenBudget ? { contextTokenBudget: ctx.contextTokenBudget } : {}), - ...(ctx.contextWindowSource ? { contextWindowSource: ctx.contextWindowSource } : {}), - ...(ctx.contextWindowReferenceTokens - ? { contextWindowReferenceTokens: ctx.contextWindowReferenceTokens } - : {}), - ...(promptStats ? { promptStats } : {}), - trace, - }; -} - -function modelContentPrivateData(modelContent: DiagnosticModelCallContent | undefined) { - return modelContent ? { modelContent } : undefined; -} - -function modelCallCompletedContent(state: ModelCallObservationState) { - if (!state.modelContent && !state.outputMessages) { - return undefined; - } - return { - ...state.modelContent, - ...(state.outputMessages ? { outputMessages: state.outputMessages } : {}), - }; -} - -function modelCallUsageField(state: ModelCallObservationState) { - return state.usage ? { usage: state.usage } : {}; -} - -function boundedTimelineAttribute(value: string | undefined): string | undefined { - return truncateUtf16Safe(value?.trim() ?? "", TIMELINE_ATTRIBUTE_MAX_LENGTH) || undefined; -} - -function emitProviderRequestTimelineEvent( - eventBase: ModelCallEventBase, - startedAt: number, - durationMs: number, - ok: boolean, - responseStatus: number | undefined, -): void { - const provider = boundedTimelineAttribute(eventBase.provider); - const model = boundedTimelineAttribute(eventBase.model); - const api = boundedTimelineAttribute(eventBase.api); - const transport = boundedTimelineAttribute(eventBase.transport); - emitDiagnosticsTimelineEvent({ - type: "provider.request", - name: "provider.request", - timestamp: new Date(startedAt).toISOString(), - runId: eventBase.runId, - spanId: eventBase.callId, - durationMs, - provider, - operation: api ?? transport ?? "model.call", - ok, - ...(responseStatus !== undefined ? { status: responseStatus } : {}), - attributes: { - ...(model ? { model } : {}), - ...(api ? { api } : {}), - ...(transport ? { transport } : {}), - }, - }); -} - -function modelCallErrorFields(err: unknown): ModelCallErrorFields { - const upstreamRequestIdHash = diagnosticProviderRequestIdHash(err); - const failureKind = diagnosticErrorFailureKind(err); - return { - errorCategory: diagnosticErrorCategory(err), - ...(failureKind ? { failureKind, memory: processMemoryUsageSnapshot() } : {}), - ...(upstreamRequestIdHash ? { upstreamRequestIdHash } : {}), - }; -} - -function processMemoryUsageSnapshot(): DiagnosticMemoryUsage | undefined { - try { - const memory = process.memoryUsage(); - return { - rssBytes: memory.rss, - heapTotalBytes: memory.heapTotal, - heapUsedBytes: memory.heapUsed, - externalBytes: memory.external, - arrayBuffersBytes: memory.arrayBuffers, - }; - } catch { - return undefined; - } -} - -function modelCallHookEventBase(eventBase: ModelCallEventBase): PluginHookModelCallStartedEvent { - return { - runId: eventBase.runId, - callId: eventBase.callId, - ...(eventBase.sessionKey ? { sessionKey: eventBase.sessionKey } : {}), - ...(eventBase.sessionId ? { sessionId: eventBase.sessionId } : {}), - provider: eventBase.provider, - model: eventBase.model, - ...(eventBase.api ? { api: eventBase.api } : {}), - ...(eventBase.transport ? { transport: eventBase.transport } : {}), - ...(eventBase.contextTokenBudget ? { contextTokenBudget: eventBase.contextTokenBudget } : {}), - ...(eventBase.contextWindowSource - ? { contextWindowSource: eventBase.contextWindowSource } - : {}), - ...(eventBase.contextWindowReferenceTokens - ? { contextWindowReferenceTokens: eventBase.contextWindowReferenceTokens } - : {}), - }; -} - -function modelCallHookContext(eventBase: ModelCallEventBase): PluginHookAgentContext { - return Object.freeze({ - runId: eventBase.runId, - trace: eventBase.trace, - ...(eventBase.sessionKey ? { sessionKey: eventBase.sessionKey } : {}), - ...(eventBase.sessionId ? { sessionId: eventBase.sessionId } : {}), - modelProviderId: eventBase.provider, - modelId: eventBase.model, - ...(eventBase.contextTokenBudget ? { contextTokenBudget: eventBase.contextTokenBudget } : {}), - ...(eventBase.contextWindowSource - ? { contextWindowSource: eventBase.contextWindowSource } - : {}), - ...(eventBase.contextWindowReferenceTokens - ? { contextWindowReferenceTokens: eventBase.contextWindowReferenceTokens } - : {}), - }) as PluginHookAgentContext; -} - -function dispatchModelCallStartedHook(eventBase: ModelCallEventBase): void { - const hookRunner = getGlobalHookRunner(); - if (!hookRunner?.hasHooks("model_call_started")) { - return; - } - const event = Object.freeze(modelCallHookEventBase(eventBase)) as PluginHookModelCallStartedEvent; - const hookCtx = modelCallHookContext(eventBase); - fireAndForgetBoundedHook( - () => hookRunner.runModelCallStarted(event, hookCtx), - "model_call_started plugin hook failed", - ); -} - -function dispatchModelCallEndedHook( - eventBase: ModelCallEventBase, - fields: ModelCallEndedHookFields, -): void { - const hookRunner = getGlobalHookRunner(); - if (!hookRunner?.hasHooks("model_call_ended")) { - return; - } - const event = Object.freeze({ - ...modelCallHookEventBase(eventBase), - ...fields, - }) as PluginHookModelCallEndedEvent; - const hookCtx = modelCallHookContext(eventBase); - fireAndForgetBoundedHook( - () => hookRunner.runModelCallEnded(event, hookCtx), - "model_call_ended plugin hook failed", - ); -} - -function emitModelCallStarted( - eventBase: ModelCallEventBase, - modelContent: DiagnosticModelCallContent | undefined, - suppressPluginHooks: boolean, -): void { - emitCoreModelRequestStartedDiagnosticEvent( - { - ...eventBase, - }, - modelContentPrivateData(modelContent), - ); - if (!suppressPluginHooks) { - dispatchModelCallStartedHook(eventBase); - } -} - -function emitModelCallCompleted( - eventBase: ModelCallEventBase, - startedAt: number, - state: ModelCallObservationState, -): void { - if (state.terminalEventEmitted) { - return; - } - state.terminalEventEmitted = true; - const durationMs = Date.now() - startedAt; - const sizeTimingFields = modelCallSizeTimingFields(state); - emitProviderRequestTimelineEvent(eventBase, startedAt, durationMs, true, state.responseStatus); - emitTrustedDiagnosticEventWithPrivateData( - { - type: "model.call.completed", - ...eventBase, - durationMs, - ...sizeTimingFields, - ...modelCallUsageField(state), - }, - modelContentPrivateData(modelCallCompletedContent(state)), - ); - if (!state.suppressPluginHooks) { - dispatchModelCallEndedHook(eventBase, { - durationMs, - outcome: "completed", - ...sizeTimingFields, - }); - } -} - -function emitModelCallError( - eventBase: ModelCallEventBase, - startedAt: number, - state: ModelCallObservationState, - err: unknown, -): void { - if (state.terminalEventEmitted) { - return; - } - state.terminalEventEmitted = true; - const durationMs = Date.now() - startedAt; - const sizeTimingFields = modelCallSizeTimingFields(state); - const fields = modelCallErrorFields(err); - const errorStatus = diagnosticHttpStatusCode(err); - const responseStatus = - state.responseStatus ?? (errorStatus === undefined ? undefined : Number(errorStatus)); - emitProviderRequestTimelineEvent(eventBase, startedAt, durationMs, false, responseStatus); - emitTrustedDiagnosticEventWithPrivateData( - { - type: "model.call.error", - ...eventBase, - durationMs, - ...sizeTimingFields, - ...fields, - ...modelCallUsageField(state), - }, - modelContentPrivateData(modelCallCompletedContent(state)), - ); - if (!state.suppressPluginHooks) { - dispatchModelCallEndedHook(eventBase, { - durationMs, - outcome: "error", - ...sizeTimingFields, - ...fields, - }); - } -} - -function withDiagnosticRequestContext( - options: ModelCallStreamOptions, - trace: DiagnosticTraceContext, - state: ModelCallObservationState, - callId: string, -): ModelCallStreamOptions { - const traceparent = formatPropagatedDiagnosticTraceparent(trace); - const originalOnPayload = options?.onPayload; - const originalOnResponse = options?.onResponse; - const onPayload: NonNullable["onPayload"] = (payload, model) => { - if (!originalOnPayload) { - assignRequestPayloadBytes(state, payload); - return undefined; - } - const result = originalOnPayload(payload, model); - if (isPromiseLike(result)) { - return result.then((replacement) => { - assignRequestPayloadBytes(state, replacement ?? payload); - return replacement; - }); - } - assignRequestPayloadBytes(state, result ?? payload); - return result; - }; - const onResponse: NonNullable["onResponse"] = (response, model) => { - // Retrying providers can expose several responses; the terminal request status - // is the latest response observed before the model call completes or fails. - state.responseStatus = response.status; - return originalOnResponse?.(response, model); - }; - - const headers: Record = {}; - for (const [key, value] of Object.entries(options?.headers ?? {})) { - if (key.toLowerCase() === TRACEPARENT_HEADER_NAME) { - continue; - } - headers[key] = value; - } - if (traceparent) { - headers[TRACEPARENT_HEADER_NAME] = traceparent; - } - return { - ...options, - requestId: callId, - ...((options?.headers || traceparent) && { headers }), - onPayload, - onResponse, - }; -} - async function safeReturnIterator(iterator: AsyncIterator): Promise { let returnResult: unknown; try { @@ -777,9 +64,7 @@ async function safeReturnIterator(iterator: AsyncIterator): Promise( iterator: AsyncIterator, - eventBase: ModelCallEventBase, - startedAt: number, - state: ModelCallObservationState, + lifecycle: ModelCallLifecycle, ): AsyncIterable { // Tracks whether the underlying iterator terminated on its own (done or threw). // This is independent of state.terminalEventEmitted: result() can emit the @@ -792,14 +77,14 @@ async function* observeModelCallIterator( iteratorSettled = true; break; } - observeResponseChunk(state, startedAt, next.value); - maybeEmitModelCallStreamProgress(eventBase, state); + lifecycle.observer.observeResponseChunk(lifecycle.startedAt, next.value); + lifecycle.observer.maybeEmitStreamProgress(lifecycle.eventBase); yield next.value; } - emitModelCallCompleted(eventBase, startedAt, state); + lifecycle.emitCompleted(); } catch (err) { iteratorSettled = true; - emitModelCallError(eventBase, startedAt, state, err); + lifecycle.emitError(err); throw err; } finally { if (!iteratorSettled) { @@ -809,30 +94,20 @@ async function* observeModelCallIterator( // listeners, SSE readers) even when result() already emitted the terminal // event; emitModelCallCompleted self-dedupes via state.terminalEventEmitted. await safeReturnIterator(iterator); - emitModelCallCompleted(eventBase, startedAt, state); + lifecycle.emitCompleted(); } } } -function observeModelCallFinalResult( - result: T, - eventBase: ModelCallEventBase, - startedAt: number, - state: ModelCallObservationState, -): T { - observeResultMessageContent(state, startedAt, result); - // Queue semantic progress beside model lifecycle events so request starts, - // progress, and the next request retain their authoritative FIFO ordering. - maybeEmitModelCallSemanticProgress(eventBase, state, result); - emitModelCallCompleted(eventBase, startedAt, state); +function observeModelCallFinalResult(result: T, lifecycle: ModelCallLifecycle): T { + lifecycle.observer.observeFinalResult(lifecycle.eventBase, lifecycle.startedAt, result); + lifecycle.emitCompleted(); return result; } function createObservedResultFunction( stream: unknown, - eventBase: ModelCallEventBase, - startedAt: number, - state: ModelCallObservationState, + lifecycle: ModelCallLifecycle, ): ((...args: unknown[]) => unknown) | undefined { if (!isRecord(stream) || typeof stream.result !== "function") { return undefined; @@ -843,16 +118,16 @@ function createObservedResultFunction( const result = resultFn.apply(stream, args); if (isPromiseLike(result)) { return result.then( - (resolved) => observeModelCallFinalResult(resolved, eventBase, startedAt, state), + (resolved) => observeModelCallFinalResult(resolved, lifecycle), (err: unknown) => { - emitModelCallError(eventBase, startedAt, state, err); + lifecycle.emitError(err); throw err; }, ); } - return observeModelCallFinalResult(result, eventBase, startedAt, state); + return observeModelCallFinalResult(result, lifecycle); } catch (err) { - emitModelCallError(eventBase, startedAt, state, err); + lifecycle.emitError(err); throw err; } }; @@ -861,13 +136,11 @@ function createObservedResultFunction( function observeModelCallStream>( stream: T, createIterator: () => AsyncIterator, - eventBase: ModelCallEventBase, - startedAt: number, - state: ModelCallObservationState, + lifecycle: ModelCallLifecycle, ): T { const observedIterator = () => - observeModelCallIterator(createIterator(), eventBase, startedAt, state)[Symbol.asyncIterator](); - const observedResult = createObservedResultFunction(stream, eventBase, startedAt, state); + observeModelCallIterator(createIterator(), lifecycle)[Symbol.asyncIterator](); + const observedResult = createObservedResultFunction(stream, lifecycle); let hasNonConfigurableIterator; try { hasNonConfigurableIterator = @@ -895,23 +168,12 @@ function observeModelCallStream>( }); } -function observeModelCallResult( - result: unknown, - eventBase: ModelCallEventBase, - startedAt: number, - state: ModelCallObservationState, -): unknown { +function observeModelCallResult(result: unknown, lifecycle: ModelCallLifecycle): unknown { const createIterator = asyncIteratorFactory(result); if (createIterator) { - return observeModelCallStream( - result as AsyncIterable, - createIterator, - eventBase, - startedAt, - state, - ); + return observeModelCallStream(result as AsyncIterable, createIterator, lifecycle); } - emitModelCallCompleted(eventBase, startedAt, state); + lifecycle.emitCompleted(); return result; } @@ -925,46 +187,33 @@ export function wrapStreamFnWithDiagnosticModelCallEvents( ctx: ModelCallDiagnosticContext, ): StreamFn { return ((model, streamContext, options) => { - const callId = ctx.nextCallId(); - const trace = freezeDiagnosticTraceContext(createChildDiagnosticTraceContext(ctx.trace)); - // Prompt stats JSON-stringify the input messages and tool definitions; only - // the diagnostic events consume them (plugin hooks never receive prompt - // stats), so skip the work when diagnostics are disabled and those events - // would be dropped. - const promptStats = areDiagnosticsEnabledForProcess() - ? streamContextModelPromptStats(streamContext) - : undefined; - const eventBase = baseModelCallEvent(ctx, callId, trace, promptStats); - const modelContent = streamContextModelContentFields(ctx.contentCapture, streamContext); - emitModelCallStarted(eventBase, modelContent, ctx.suppressPluginHooks === true); - ctx.onStarted?.(); - const startedAt = Date.now(); - const state: ModelCallObservationState = { - responseStreamBytes: 0, - modelContent, - contentCapture: ctx.contentCapture, - suppressPluginHooks: ctx.suppressPluginHooks, - }; - // Provider wrappers consume this same call id for transport correlation, - // keeping external request evidence joined to the emitted diagnostics. - const propagatedOptions = withDiagnosticRequestContext(options, trace, state, callId); + const lifecycle = createModelLifecycle({ + ctx, + options, + createObserver: (capturePromptStats) => + createModelObserver({ + streamContext, + contentCapture: ctx.contentCapture, + suppressPluginHooks: ctx.suppressPluginHooks, + capturePromptStats, + }), + }); try { - const result = streamFn(model, streamContext, propagatedOptions); + const result = streamFn(model, streamContext, lifecycle.propagatedOptions); if (isPromiseLike(result)) { return result.then( - (resolved) => observeModelCallResult(resolved, eventBase, startedAt, state), + (resolved) => observeModelCallResult(resolved, lifecycle), (err: unknown) => { - emitModelCallError(eventBase, startedAt, state, err); + lifecycle.emitError(err); throw err; }, ); } - return observeModelCallResult(result, eventBase, startedAt, state); + return observeModelCallResult(result, lifecycle); } catch (err) { - emitModelCallError(eventBase, startedAt, state, err); + lifecycle.emitError(err); throw err; } }) as StreamFn; } -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-lifecycle.test.ts b/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-lifecycle.test.ts new file mode 100644 index 000000000000..73f0476de98e --- /dev/null +++ b/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-lifecycle.test.ts @@ -0,0 +1,611 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +// Coverage for model-call diagnostic events around attempt stream functions. +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import type { StreamFn } from "openclaw/plugin-sdk/agent-core"; +import { createRequireRecord } from "openclaw/plugin-sdk/test-fixtures"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../../../test/helpers/temp-dir.js"; +import { + onInternalDiagnosticEvent, + resetDiagnosticEventsForTest, + type DiagnosticEventPayload, +} from "../../../infra/diagnostic-events.js"; +import { createDiagnosticTraceContext } from "../../../infra/diagnostic-trace-context.js"; +import { registerDiagnosticTracePropagationBridge } from "../../../infra/diagnostic-trace-propagation.js"; +import { + resetDiagnosticRunActivityForTest, + startDiagnosticRunActivityTracking, +} from "../../../logging/diagnostic-run-activity.js"; +import { + initializeGlobalHookRunner, + resetGlobalHookRunner, +} from "../../../plugins/hook-runner-global.js"; +import { createHookRunnerWithRegistry } from "../../../plugins/hooks.test-fixtures.js"; +import { withEnvAsync } from "../../../test-utils/env.js"; +import { wrapStreamFnWithDiagnosticModelCallEvents } from "./attempt.model-diagnostic-events.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +async function collectModelCallEvents(run: () => Promise): Promise { + // Diagnostics are emitted asynchronously; collect only public model-call + // events and flush one tick after the stream completes. + const events: DiagnosticEventPayload[] = []; + const stop = onInternalDiagnosticEvent((event) => { + if (event.type.startsWith("model.call.")) { + events.push(event); + } + }); + try { + await run(); + await new Promise((resolve) => { + setImmediate(resolve); + }); + return events; + } finally { + stop(); + } +} + +async function drain(stream: AsyncIterable): Promise { + // Force stream iteration so completion events include response byte and timing + // accounting. + for await (const _ of stream) { + // drain + } +} + +const requireRecord = createRequireRecord("record", "expected-label-object-capitalized"); + +function readRecordField(record: Record, key: string, label: string) { + const value = record[key]; + if (!isRecord(value)) { + throw new Error(`Expected ${label} to be an object`); + } + return value; +} + +function expectNumberField(record: Record, key: string) { + expect(typeof record[key]).toBe("number"); +} + +function getEvent(events: readonly DiagnosticEventPayload[], index: number) { + return requireRecord(events[index], `event ${index}`); +} + +function requireMockRecordArg( + mock: ReturnType, + callIndex: number, + argIndex: number, + label: string, +) { + return requireRecord(mock.mock.calls[callIndex]?.[argIndex], label); +} + +async function collectProviderTimelineEvents(run: () => Promise) { + const root = tempDirs.make("openclaw-provider-timeline-"); + const timelinePath = join(root, "timeline.jsonl"); + await withEnvAsync( + { + OPENCLAW_DIAGNOSTICS: "1", + OPENCLAW_DIAGNOSTICS_TIMELINE_PATH: timelinePath, + }, + run, + ); + return readFileSync(timelinePath, "utf8") + .trim() + .split("\n") + .filter(Boolean) + .map((line) => requireRecord(JSON.parse(line), "provider timeline event")) + .filter((event) => event.type === "provider.request"); +} + +describe("wrapStreamFnWithDiagnosticModelCallEvents lifecycle", () => { + beforeEach(() => { + resetDiagnosticEventsForTest(); + resetDiagnosticRunActivityForTest(); + startDiagnosticRunActivityTracking(); + resetGlobalHookRunner(); + }); + + afterEach(() => { + resetDiagnosticEventsForTest(); + resetGlobalHookRunner(); + resetDiagnosticRunActivityForTest(); + vi.restoreAllMocks(); + vi.useRealTimers(); + }); + + it("emits one successful provider timeline event for result and iterator completion", async () => { + let now = Date.parse("2026-07-09T18:30:00.000Z"); + vi.spyOn(Date, "now").mockImplementation(() => now); + async function* stream() { + yield { type: "text", text: "ok" }; + } + const originalStream = stream() as unknown as AsyncIterable & { + result: () => Promise; + }; + originalStream.result = async () => { + now += 125; + return "kept"; + }; + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (() => originalStream) as unknown as StreamFn, + { + runId: "run-timeline-success", + provider: "openai", + model: "gpt-5.5", + api: "openai-responses", + transport: "http", + trace: createDiagnosticTraceContext(), + nextCallId: () => "call-timeline-success", + }, + ); + + const events = await collectProviderTimelineEvents(async () => { + const returned = wrapped( + {} as never, + {} as never, + {} as never, + ) as unknown as typeof originalStream; + await returned.result(); + await drain(returned); + }); + + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + type: "provider.request", + name: "provider.request", + timestamp: "2026-07-09T18:30:00.000Z", + runId: "run-timeline-success", + spanId: "call-timeline-success", + durationMs: 125, + provider: "openai", + operation: "openai-responses", + ok: true, + attributes: { + model: "gpt-5.5", + api: "openai-responses", + transport: "http", + }, + }); + expect(events[0]?.status).toBeUndefined(); + }); + + it("records provider response status and preserves the original response callback", async () => { + const originalOnResponse = vi.fn(async () => undefined); + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (( + model: Parameters[0], + _context: Parameters[1], + options: Parameters[2], + ) => { + return options?.onResponse?.({ status: 200, headers: { "x-request-id": "req-1" } }, model); + }) as unknown as StreamFn, + { + runId: "run-timeline-status", + provider: "openai", + model: "gpt-5.6", + api: "openai-responses", + transport: "http", + trace: createDiagnosticTraceContext(), + nextCallId: () => "call-timeline-status", + }, + ); + + const events = await collectProviderTimelineEvents(async () => { + await wrapped( + { id: "gpt-5.6" } as never, + {} as never, + { + onResponse: originalOnResponse, + } as never, + ); + }); + + expect(originalOnResponse).toHaveBeenCalledWith( + { status: 200, headers: { "x-request-id": "req-1" } }, + { id: "gpt-5.6" }, + ); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + type: "provider.request", + ok: true, + status: 200, + }); + }); + + it("writes Unicode-safe bounded attributes to the provider timeline JSONL", async () => { + const modelPrefix = "m".repeat(255); + const exactBoundary = "b".repeat(256); + const events = await collectProviderTimelineEvents(async () => { + const cases: Array<{ callId: string; model: string }> = [ + { callId: "call-timeline-unicode-boundary", model: `${modelPrefix}😀tail` }, + { callId: "call-timeline-exact-boundary", model: exactBoundary }, + ]; + for (const { callId, model } of cases) { + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (() => undefined) as unknown as StreamFn, + { + runId: "run-timeline-unicode-boundary", + provider: "openai", + model, + trace: createDiagnosticTraceContext(), + nextCallId: () => callId, + }, + ); + await wrapped({} as never, {} as never, {} as never); + } + }); + + expect(events).toHaveLength(2); + const splitBoundaryModel = readRecordField(events[0]!, "attributes", "attributes").model; + expect(splitBoundaryModel).toBe(modelPrefix); + expect(splitBoundaryModel).toHaveLength(255); + expect(splitBoundaryModel).not.toContain("�"); + expect(splitBoundaryModel).not.toMatch(/[\uD800-\uDFFF]/u); + const exactBoundaryModel = readRecordField(events[1]!, "attributes", "attributes").model; + expect(exactBoundaryModel).toBe(exactBoundary); + expect(exactBoundaryModel).toHaveLength(256); + }); + + it("emits one failed provider timeline event for a thrown model call", async () => { + let now = Date.parse("2026-07-09T18:31:00.000Z"); + vi.spyOn(Date, "now").mockImplementation(() => now); + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (() => { + now += 75; + throw new Error("provider failed"); + }) as unknown as StreamFn, + { + runId: "run-timeline-error", + provider: "anthropic", + model: "claude-sonnet-4-6", + transport: "sse", + trace: createDiagnosticTraceContext(), + nextCallId: () => "call-timeline-error", + }, + ); + + const events = await collectProviderTimelineEvents(async () => { + expect(() => wrapped({} as never, {} as never, {} as never)).toThrow("provider failed"); + }); + + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + type: "provider.request", + name: "provider.request", + timestamp: "2026-07-09T18:31:00.000Z", + runId: "run-timeline-error", + spanId: "call-timeline-error", + durationMs: 75, + provider: "anthropic", + operation: "sse", + ok: false, + attributes: { + model: "claude-sonnet-4-6", + transport: "sse", + }, + }); + }); + + it("records a non-2xx provider response on a failed model call", async () => { + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (() => { + throw Object.assign(new Error("rate limited"), { status: 429 }); + }) as unknown as StreamFn, + { + runId: "run-timeline-http-error", + provider: "openai", + model: "gpt-5.6", + api: "openai-responses", + transport: "http", + trace: createDiagnosticTraceContext(), + nextCallId: () => "call-timeline-http-error", + }, + ); + + const events = await collectProviderTimelineEvents(async () => { + expect(() => wrapped({} as never, {} as never, {} as never)).toThrow("rate limited"); + }); + + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + type: "provider.request", + ok: false, + status: 429, + }); + }); + + it("keeps an observed response status when the terminal error has another status", async () => { + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (( + model: Parameters[0], + _context: Parameters[1], + options: Parameters[2], + ) => { + void options?.onResponse?.({ status: 503, headers: {} }, model); + throw Object.assign(new Error("retry failed"), { status: 429 }); + }) as unknown as StreamFn, + { + runId: "run-timeline-observed-http-error", + provider: "openai", + model: "gpt-5.6", + api: "openai-responses", + transport: "http", + trace: createDiagnosticTraceContext(), + nextCallId: () => "call-timeline-observed-http-error", + }, + ); + + const events = await collectProviderTimelineEvents(async () => { + expect(() => wrapped({} as never, {} as never, {} as never)).toThrow("retry failed"); + }); + + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + type: "provider.request", + ok: false, + status: 503, + }); + }); + + it("propagates the trusted model-call traceparent without mutating caller headers", async () => { + async function* stream() { + yield { type: "text", text: "ok" }; + } + const capturedOptions: Array[2]> = []; + const callerOptions = { + headers: { + "X-Custom": "kept", + TraceParent: "00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-01", + }, + sessionId: "provider-session", + }; + const exportedTrace = createDiagnosticTraceContext({ + traceId: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + spanId: "bbbbbbbbbbbbbbbb", + traceFlags: "01", + }); + registerDiagnosticTracePropagationBridge({ + resolveTraceContext: () => exportedTrace, + }); + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (( + _model: Parameters[0], + _context: Parameters[1], + options: Parameters[2], + ) => { + capturedOptions.push(options); + return stream(); + }) as unknown as StreamFn, + { + runId: "run-1", + provider: "openai", + model: "gpt-5.4", + trace: createDiagnosticTraceContext({ + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + spanId: "00f067aa0ba902b7", + traceFlags: "01", + }), + nextCallId: () => "call-traceparent", + }, + ); + + await drain( + wrapped({} as never, {} as never, callerOptions) as unknown as AsyncIterable, + ); + + expect(capturedOptions).toHaveLength(1); + expect(capturedOptions[0]).not.toBe(callerOptions); + const capturedOption = requireRecord(capturedOptions[0], "captured stream options"); + expect(capturedOption.sessionId).toBe("provider-session"); + expect(capturedOption.requestId).toBe("call-traceparent"); + const headers = readRecordField(capturedOption, "headers", "captured stream headers"); + expect(headers["X-Custom"]).toBe("kept"); + expect(headers.traceparent).toBe(`00-${exportedTrace.traceId}-${exportedTrace.spanId}-01`); + expect(capturedOptions[0]?.headers).not.toHaveProperty("TraceParent"); + expect(callerOptions.headers).toEqual({ + "X-Custom": "kept", + TraceParent: "00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-01", + }); + }); + + it("removes caller traceparent when the active exporter cannot resolve a span", async () => { + async function* stream() { + yield { type: "text", text: "ok" }; + } + const capturedOptions: Array[2]> = []; + registerDiagnosticTracePropagationBridge({ + resolveTraceContext: () => undefined, + }); + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (( + _model: Parameters[0], + _context: Parameters[1], + options: Parameters[2], + ) => { + capturedOptions.push(options); + return stream(); + }) as unknown as StreamFn, + { + runId: "run-1", + provider: "openai", + model: "gpt-5.4", + trace: createDiagnosticTraceContext(), + nextCallId: () => "call-no-exported-span", + }, + ); + + await drain( + wrapped({} as never, {} as never, { + headers: { + "X-Custom": "kept", + TraceParent: "00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-01", + }, + }) as unknown as AsyncIterable, + ); + + expect(capturedOptions[0]?.headers).toEqual({ "X-Custom": "kept" }); + }); + + it("adds failure kind and memory diagnostics for terminated model calls", async () => { + const stream = { + [Symbol.asyncIterator]() { + return { + async next(): Promise> { + throw new Error("terminated"); + }, + }; + }, + }; + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (() => stream) as unknown as StreamFn, + { + runId: "run-1", + provider: "lmstudio", + model: "qwen/qwen3.5-9b", + trace: createDiagnosticTraceContext(), + nextCallId: () => "call-terminated", + }, + ); + + const events = await collectModelCallEvents(async () => { + await expect( + drain(wrapped({} as never, {} as never, {} as never) as AsyncIterable), + ).rejects.toThrow("terminated"); + }); + + expect(events.map((event) => event.type)).toEqual(["model.call.started", "model.call.error"]); + const errorEvent = getEvent(events, 1); + expect(errorEvent.type).toBe("model.call.error"); + expect(errorEvent.callId).toBe("call-terminated"); + expect(errorEvent.errorCategory).toBe("Error"); + expect(errorEvent.failureKind).toBe("terminated"); + const memory = readRecordField(errorEvent, "memory", "error event memory"); + expectNumberField(memory, "rssBytes"); + expectNumberField(memory, "heapTotalBytes"); + expectNumberField(memory, "heapUsedBytes"); + expectNumberField(memory, "externalBytes"); + expectNumberField(memory, "arrayBuffersBytes"); + }); + + it("fires frozen sanitized model-call plugin hooks", async () => { + const started = vi.fn(); + const ended = vi.fn(); + const { registry } = createHookRunnerWithRegistry([ + { hookName: "model_call_started", handler: started }, + { hookName: "model_call_ended", handler: ended }, + ]); + initializeGlobalHookRunner(registry); + const secretChunk = "secret response with Bearer sk-test-secret-value"; + + async function* stream() { + yield { type: "text", text: secretChunk }; + } + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (() => stream()) as unknown as StreamFn, + { + runId: "run-1", + sessionKey: "session-key", + sessionId: "session-id", + provider: "openai", + model: "gpt-5.4", + api: "openai-responses", + transport: "http", + contextTokenBudget: 150_000, + contextWindowSource: "agentContextTokens", + contextWindowReferenceTokens: 200_000, + trace: createDiagnosticTraceContext(), + nextCallId: () => "call-hook", + }, + ); + + const events = await collectModelCallEvents(async () => { + await drain(wrapped({} as never, {} as never, {} as never) as AsyncIterable); + }); + await new Promise((resolve) => { + setImmediate(resolve); + }); + + expect(events.map((event) => event.type)).toEqual([ + "model.call.started", + "model.call.completed", + ]); + const startedEvent = requireMockRecordArg(started, 0, 0, "started hook event"); + expect(startedEvent.runId).toBe("run-1"); + expect(startedEvent.callId).toBe("call-hook"); + expect(startedEvent.sessionKey).toBe("session-key"); + expect(startedEvent.sessionId).toBe("session-id"); + expect(startedEvent.provider).toBe("openai"); + expect(startedEvent.model).toBe("gpt-5.4"); + expect(startedEvent.api).toBe("openai-responses"); + expect(startedEvent.transport).toBe("http"); + expect(startedEvent.contextTokenBudget).toBe(150_000); + expect(startedEvent.contextWindowSource).toBe("agentContextTokens"); + expect(startedEvent.contextWindowReferenceTokens).toBe(200_000); + const startedCtx = requireMockRecordArg(started, 0, 1, "started hook context"); + expect(startedCtx.runId).toBe("run-1"); + expect(startedCtx.sessionKey).toBe("session-key"); + expect(startedCtx.sessionId).toBe("session-id"); + expect(startedCtx.modelProviderId).toBe("openai"); + expect(startedCtx.modelId).toBe("gpt-5.4"); + expect(startedCtx.contextTokenBudget).toBe(150_000); + expect(startedCtx.contextWindowSource).toBe("agentContextTokens"); + expect(startedCtx.contextWindowReferenceTokens).toBe(200_000); + const endedEvent = requireMockRecordArg(ended, 0, 0, "ended hook event"); + expect(endedEvent.runId).toBe("run-1"); + expect(endedEvent.callId).toBe("call-hook"); + expect(endedEvent.outcome).toBe("completed"); + expect(endedEvent.contextTokenBudget).toBe(150_000); + expect(endedEvent.contextWindowSource).toBe("agentContextTokens"); + expect(endedEvent.contextWindowReferenceTokens).toBe(200_000); + expectNumberField(endedEvent, "durationMs"); + expectNumberField(endedEvent, "responseStreamBytes"); + expectNumberField(endedEvent, "timeToFirstByteMs"); + const endedCtx = requireMockRecordArg(ended, 0, 1, "ended hook context"); + expect(endedCtx.runId).toBe("run-1"); + expect(Object.isFrozen(startedEvent)).toBe(true); + expect(Object.isFrozen(startedCtx)).toBe(true); + expect(Object.isFrozen(startedCtx.trace)).toBe(true); + expect(JSON.stringify([started.mock.calls, ended.mock.calls])).not.toContain(secretChunk); + }); + + it("keeps core model-call diagnostics while suppressing finalization plugin hooks", async () => { + const started = vi.fn(); + const ended = vi.fn(); + const { registry } = createHookRunnerWithRegistry([ + { hookName: "model_call_started", handler: started }, + { hookName: "model_call_ended", handler: ended }, + ]); + initializeGlobalHookRunner(registry); + async function* stream() { + yield { type: "text", text: "final answer" }; + } + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (() => stream()) as unknown as StreamFn, + { + runId: "run-finalization", + provider: "openai", + model: "gpt-5.4", + trace: createDiagnosticTraceContext(), + nextCallId: () => "call-finalization", + suppressPluginHooks: true, + }, + ); + + const events = await collectModelCallEvents(async () => { + await drain(wrapped({} as never, {} as never, {} as never) as AsyncIterable); + }); + await new Promise((resolve) => { + setImmediate(resolve); + }); + + expect(events.map((event) => event.type)).toEqual([ + "model.call.started", + "model.call.completed", + ]); + expect(started).not.toHaveBeenCalled(); + expect(ended).not.toHaveBeenCalled(); + }); +}); diff --git a/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-lifecycle.ts b/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-lifecycle.ts new file mode 100644 index 000000000000..9d5201149299 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-lifecycle.ts @@ -0,0 +1,440 @@ +import { isPromiseLike } from "@openclaw/normalization-core/promise-like"; +import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; +import { fireAndForgetBoundedHook } from "../../../hooks/fire-and-forget.js"; +import { + diagnosticErrorCategory, + diagnosticErrorFailureKind, + diagnosticHttpStatusCode, + diagnosticProviderRequestIdHash, +} from "../../../infra/diagnostic-error-metadata.js"; +import { + areDiagnosticsEnabledForProcess, + type DiagnosticEventInput, + type DiagnosticModelCallContent, + type DiagnosticMemoryUsage, + emitTrustedDiagnosticEventWithPrivateData, +} from "../../../infra/diagnostic-events.js"; +import type { DiagnosticModelContentCapturePolicy } from "../../../infra/diagnostic-llm-content.js"; +import { emitCoreModelRequestStartedDiagnosticEvent } from "../../../infra/diagnostic-model-request.js"; +import { + createChildDiagnosticTraceContext, + freezeDiagnosticTraceContext, + type DiagnosticTraceContext, +} from "../../../infra/diagnostic-trace-context.js"; +import { formatPropagatedDiagnosticTraceparent } from "../../../infra/diagnostic-trace-propagation.js"; +import { emitDiagnosticsTimelineEvent } from "../../../infra/diagnostics-timeline.js"; +import { getGlobalHookRunner } from "../../../plugins/hook-runner-global.js"; +import type { + PluginHookAgentContext, + PluginHookContextWindowSource, + PluginHookModelCallEndedEvent, + PluginHookModelCallStartedEvent, +} from "../../../plugins/hook-types.js"; +import type { StreamFn } from "../../runtime/index.js"; + +export type ModelCallDiagnosticContext = { + runId: string; + sessionKey?: string; + sessionId?: string; + provider: string; + model: string; + api?: string; + transport?: string; + contextTokenBudget?: number; + contextWindowSource?: PluginHookContextWindowSource; + contextWindowReferenceTokens?: number; + trace: DiagnosticTraceContext; + contentCapture?: DiagnosticModelContentCapturePolicy; + nextCallId: () => string; + onStarted?: () => void; + suppressPluginHooks?: boolean; +}; + +export type ModelCallEventBase = Omit< + Extract, + "type" +>; +type ModelCallErrorFields = Pick< + Extract, + "errorCategory" | "failureKind" | "memory" | "upstreamRequestIdHash" +>; +type ModelCallEndedHookFields = Pick< + PluginHookModelCallEndedEvent, + | "durationMs" + | "outcome" + | "errorCategory" + | "requestPayloadBytes" + | "responseStreamBytes" + | "timeToFirstByteMs" + | "failureKind" + | "upstreamRequestIdHash" +>; +export type ModelCallSizeTimingFields = Pick< + Extract, + "requestPayloadBytes" | "responseStreamBytes" | "timeToFirstByteMs" +>; +export type ModelCallPromptStats = NonNullable< + Extract["promptStats"] +>; +export type ModelCallUsage = NonNullable< + Extract["usage"] +>; +export type ModelCallObservationState = { + requestPayloadBytes?: number; + responseStatus?: number; + responseStreamBytes: number; + timeToFirstByteMs?: number; + modelContent?: DiagnosticModelCallContent; + outputMessages?: unknown[]; + usage?: ModelCallUsage; + contentCapture?: DiagnosticModelContentCapturePolicy; + lastStreamProgressAt?: number; + semanticProgressEmitted?: boolean; + terminalEventEmitted?: boolean; + suppressPluginHooks?: boolean; +}; +export type ModelCallObserver = { + state: ModelCallObservationState; + promptStats?: ModelCallPromptStats; + modelContent?: DiagnosticModelCallContent; + assignRequestPayloadBytes: (payload: unknown) => void; + observeResponseChunk: (startedAt: number, chunk: unknown) => void; + observeFinalResult: (eventBase: ModelCallEventBase, startedAt: number, result: unknown) => void; + maybeEmitStreamProgress: (eventBase: ModelCallEventBase) => void; + sizeTimingFields: () => ModelCallSizeTimingFields; + completedContent: () => DiagnosticModelCallContent | undefined; + usageField: () => { usage?: ModelCallUsage }; +}; + +const TRACEPARENT_HEADER_NAME = "traceparent"; +const TIMELINE_ATTRIBUTE_MAX_LENGTH = 256; +type ModelCallStreamOptions = Parameters[2]; + +function baseModelCallEvent( + ctx: ModelCallDiagnosticContext, + callId: string, + trace: DiagnosticTraceContext, + promptStats: ModelCallPromptStats | undefined, +): ModelCallEventBase { + return { + runId: ctx.runId, + callId, + ...(ctx.sessionKey && { sessionKey: ctx.sessionKey }), + ...(ctx.sessionId && { sessionId: ctx.sessionId }), + provider: ctx.provider, + model: ctx.model, + ...(ctx.api && { api: ctx.api }), + ...(ctx.transport && { transport: ctx.transport }), + observationUnit: "request", + ...(ctx.contextTokenBudget ? { contextTokenBudget: ctx.contextTokenBudget } : {}), + ...(ctx.contextWindowSource ? { contextWindowSource: ctx.contextWindowSource } : {}), + ...(ctx.contextWindowReferenceTokens + ? { contextWindowReferenceTokens: ctx.contextWindowReferenceTokens } + : {}), + ...(promptStats ? { promptStats } : {}), + trace, + }; +} + +function modelContentPrivateData(modelContent: DiagnosticModelCallContent | undefined) { + return modelContent ? { modelContent } : undefined; +} + +function boundedTimelineAttribute(value: string | undefined): string | undefined { + return truncateUtf16Safe(value?.trim() ?? "", TIMELINE_ATTRIBUTE_MAX_LENGTH) || undefined; +} + +function emitProviderRequestTimelineEvent( + eventBase: ModelCallEventBase, + startedAt: number, + durationMs: number, + ok: boolean, + responseStatus: number | undefined, +): void { + const provider = boundedTimelineAttribute(eventBase.provider); + const model = boundedTimelineAttribute(eventBase.model); + const api = boundedTimelineAttribute(eventBase.api); + const transport = boundedTimelineAttribute(eventBase.transport); + emitDiagnosticsTimelineEvent({ + type: "provider.request", + name: "provider.request", + timestamp: new Date(startedAt).toISOString(), + runId: eventBase.runId, + spanId: eventBase.callId, + durationMs, + provider, + operation: api ?? transport ?? "model.call", + ok, + ...(responseStatus !== undefined ? { status: responseStatus } : {}), + attributes: { + ...(model ? { model } : {}), + ...(api ? { api } : {}), + ...(transport ? { transport } : {}), + }, + }); +} + +function modelCallErrorFields(err: unknown): ModelCallErrorFields { + const upstreamRequestIdHash = diagnosticProviderRequestIdHash(err); + const failureKind = diagnosticErrorFailureKind(err); + return { + errorCategory: diagnosticErrorCategory(err), + ...(failureKind ? { failureKind, memory: processMemoryUsageSnapshot() } : {}), + ...(upstreamRequestIdHash ? { upstreamRequestIdHash } : {}), + }; +} + +function processMemoryUsageSnapshot(): DiagnosticMemoryUsage | undefined { + try { + const memory = process.memoryUsage(); + return { + rssBytes: memory.rss, + heapTotalBytes: memory.heapTotal, + heapUsedBytes: memory.heapUsed, + externalBytes: memory.external, + arrayBuffersBytes: memory.arrayBuffers, + }; + } catch { + return undefined; + } +} + +function modelCallHookEventBase(eventBase: ModelCallEventBase): PluginHookModelCallStartedEvent { + return { + runId: eventBase.runId, + callId: eventBase.callId, + ...(eventBase.sessionKey ? { sessionKey: eventBase.sessionKey } : {}), + ...(eventBase.sessionId ? { sessionId: eventBase.sessionId } : {}), + provider: eventBase.provider, + model: eventBase.model, + ...(eventBase.api ? { api: eventBase.api } : {}), + ...(eventBase.transport ? { transport: eventBase.transport } : {}), + ...(eventBase.contextTokenBudget ? { contextTokenBudget: eventBase.contextTokenBudget } : {}), + ...(eventBase.contextWindowSource + ? { contextWindowSource: eventBase.contextWindowSource } + : {}), + ...(eventBase.contextWindowReferenceTokens + ? { contextWindowReferenceTokens: eventBase.contextWindowReferenceTokens } + : {}), + }; +} + +function modelCallHookContext(eventBase: ModelCallEventBase): PluginHookAgentContext { + return Object.freeze({ + runId: eventBase.runId, + trace: eventBase.trace, + ...(eventBase.sessionKey ? { sessionKey: eventBase.sessionKey } : {}), + ...(eventBase.sessionId ? { sessionId: eventBase.sessionId } : {}), + modelProviderId: eventBase.provider, + modelId: eventBase.model, + ...(eventBase.contextTokenBudget ? { contextTokenBudget: eventBase.contextTokenBudget } : {}), + ...(eventBase.contextWindowSource + ? { contextWindowSource: eventBase.contextWindowSource } + : {}), + ...(eventBase.contextWindowReferenceTokens + ? { contextWindowReferenceTokens: eventBase.contextWindowReferenceTokens } + : {}), + }) as PluginHookAgentContext; +} + +function dispatchModelCallStartedHook(eventBase: ModelCallEventBase): void { + const hookRunner = getGlobalHookRunner(); + if (!hookRunner?.hasHooks("model_call_started")) { + return; + } + const event = Object.freeze(modelCallHookEventBase(eventBase)) as PluginHookModelCallStartedEvent; + const hookCtx = modelCallHookContext(eventBase); + fireAndForgetBoundedHook( + () => hookRunner.runModelCallStarted(event, hookCtx), + "model_call_started plugin hook failed", + ); +} + +function dispatchModelCallEndedHook( + eventBase: ModelCallEventBase, + fields: ModelCallEndedHookFields, +): void { + const hookRunner = getGlobalHookRunner(); + if (!hookRunner?.hasHooks("model_call_ended")) { + return; + } + const event = Object.freeze({ + ...modelCallHookEventBase(eventBase), + ...fields, + }) as PluginHookModelCallEndedEvent; + const hookCtx = modelCallHookContext(eventBase); + fireAndForgetBoundedHook( + () => hookRunner.runModelCallEnded(event, hookCtx), + "model_call_ended plugin hook failed", + ); +} + +function emitModelCallStarted( + eventBase: ModelCallEventBase, + modelContent: DiagnosticModelCallContent | undefined, + suppressPluginHooks: boolean, +): void { + emitCoreModelRequestStartedDiagnosticEvent( + { + ...eventBase, + }, + modelContentPrivateData(modelContent), + ); + if (!suppressPluginHooks) { + dispatchModelCallStartedHook(eventBase); + } +} + +function emitModelCallCompleted( + eventBase: ModelCallEventBase, + startedAt: number, + observer: ModelCallObserver, +): void { + if (observer.state.terminalEventEmitted) { + return; + } + observer.state.terminalEventEmitted = true; + const durationMs = Date.now() - startedAt; + const sizeTimingFields = observer.sizeTimingFields(); + emitProviderRequestTimelineEvent( + eventBase, + startedAt, + durationMs, + true, + observer.state.responseStatus, + ); + emitTrustedDiagnosticEventWithPrivateData( + { + type: "model.call.completed", + ...eventBase, + durationMs, + ...sizeTimingFields, + ...observer.usageField(), + }, + modelContentPrivateData(observer.completedContent()), + ); + if (!observer.state.suppressPluginHooks) { + dispatchModelCallEndedHook(eventBase, { + durationMs, + outcome: "completed", + ...sizeTimingFields, + }); + } +} + +function emitModelCallError( + eventBase: ModelCallEventBase, + startedAt: number, + observer: ModelCallObserver, + err: unknown, +): void { + if (observer.state.terminalEventEmitted) { + return; + } + observer.state.terminalEventEmitted = true; + const durationMs = Date.now() - startedAt; + const sizeTimingFields = observer.sizeTimingFields(); + const fields = modelCallErrorFields(err); + const errorStatus = diagnosticHttpStatusCode(err); + const responseStatus = + observer.state.responseStatus ?? (errorStatus === undefined ? undefined : Number(errorStatus)); + emitProviderRequestTimelineEvent(eventBase, startedAt, durationMs, false, responseStatus); + emitTrustedDiagnosticEventWithPrivateData( + { + type: "model.call.error", + ...eventBase, + durationMs, + ...sizeTimingFields, + ...fields, + ...observer.usageField(), + }, + modelContentPrivateData(observer.completedContent()), + ); + if (!observer.state.suppressPluginHooks) { + dispatchModelCallEndedHook(eventBase, { + durationMs, + outcome: "error", + ...sizeTimingFields, + ...fields, + }); + } +} + +function withDiagnosticRequestContext( + options: ModelCallStreamOptions, + trace: DiagnosticTraceContext, + observer: ModelCallObserver, + callId: string, +): ModelCallStreamOptions { + const traceparent = formatPropagatedDiagnosticTraceparent(trace); + const originalOnPayload = options?.onPayload; + const originalOnResponse = options?.onResponse; + const onPayload: NonNullable["onPayload"] = (payload, model) => { + if (!originalOnPayload) { + observer.assignRequestPayloadBytes(payload); + return undefined; + } + const result = originalOnPayload(payload, model); + if (isPromiseLike(result)) { + return result.then((replacement) => { + observer.assignRequestPayloadBytes(replacement ?? payload); + return replacement; + }); + } + observer.assignRequestPayloadBytes(result ?? payload); + return result; + }; + const onResponse: NonNullable["onResponse"] = (response, model) => { + // Retrying providers can expose several responses; the terminal request status + // is the latest response observed before the model call completes or fails. + observer.state.responseStatus = response.status; + return originalOnResponse?.(response, model); + }; + + const headers: Record = {}; + for (const [key, value] of Object.entries(options?.headers ?? {})) { + if (key.toLowerCase() === TRACEPARENT_HEADER_NAME) { + continue; + } + headers[key] = value; + } + if (traceparent) { + headers[TRACEPARENT_HEADER_NAME] = traceparent; + } + return { + ...options, + requestId: callId, + ...((options?.headers || traceparent) && { headers }), + onPayload, + onResponse, + }; +} + +export function createModelLifecycle(params: { + ctx: ModelCallDiagnosticContext; + options: ModelCallStreamOptions; + createObserver: (capturePromptStats: boolean) => ModelCallObserver; +}) { + const callId = params.ctx.nextCallId(); + const trace = freezeDiagnosticTraceContext(createChildDiagnosticTraceContext(params.ctx.trace)); + const observer = params.createObserver(areDiagnosticsEnabledForProcess()); + const eventBase = baseModelCallEvent(params.ctx, callId, trace, observer.promptStats); + emitModelCallStarted(eventBase, observer.modelContent, params.ctx.suppressPluginHooks === true); + params.ctx.onStarted?.(); + const startedAt = Date.now(); + const propagatedOptions = withDiagnosticRequestContext(params.options, trace, observer, callId); + return { + eventBase, + observer, + propagatedOptions, + startedAt, + emitCompleted() { + emitModelCallCompleted(eventBase, startedAt, observer); + }, + emitError(err: unknown) { + emitModelCallError(eventBase, startedAt, observer, err); + }, + }; +} + +export type ModelCallLifecycle = ReturnType; diff --git a/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-observation.test.ts b/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-observation.test.ts new file mode 100644 index 000000000000..3773acb22764 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-observation.test.ts @@ -0,0 +1,732 @@ +// Coverage for model-call diagnostic events around attempt stream functions. +import type { StreamFn } from "openclaw/plugin-sdk/agent-core"; +import { createRequireRecord } from "openclaw/plugin-sdk/test-fixtures"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + onInternalDiagnosticEvent, + onTrustedInternalDiagnosticEvent, + resetDiagnosticEventsForTest, + setDiagnosticsEnabledForProcess, + type DiagnosticEventPrivateData, + type DiagnosticEventPayload, + waitForDiagnosticEventsDrained, +} from "../../../infra/diagnostic-events.js"; +import { isCoreSemanticRunProgressDiagnosticMetadata } from "../../../infra/diagnostic-semantic-run-progress.js"; +import { createDiagnosticTraceContext } from "../../../infra/diagnostic-trace-context.js"; +import { + getDiagnosticSessionActivitySnapshot, + markDiagnosticEmbeddedRunStarted, + resetDiagnosticRunActivityForTest, + startDiagnosticRunActivityTracking, +} from "../../../logging/diagnostic-run-activity.js"; +import { resetGlobalHookRunner } from "../../../plugins/hook-runner-global.js"; +import { wrapStreamFnWithDiagnosticModelCallEvents } from "./attempt.model-diagnostic-events.js"; + +async function collectModelCallEvents(run: () => Promise): Promise { + // Diagnostics are emitted asynchronously; collect only public model-call + // events and flush one tick after the stream completes. + const events: DiagnosticEventPayload[] = []; + const stop = onInternalDiagnosticEvent((event) => { + if (event.type.startsWith("model.call.")) { + events.push(event); + } + }); + try { + await run(); + await new Promise((resolve) => { + setImmediate(resolve); + }); + return events; + } finally { + stop(); + } +} + +async function collectTrustedModelCallEvents(run: () => Promise): Promise< + Array<{ + event: DiagnosticEventPayload; + privateData: DiagnosticEventPrivateData; + }> +> { + const events: Array<{ + event: DiagnosticEventPayload; + privateData: DiagnosticEventPrivateData; + }> = []; + const stop = onTrustedInternalDiagnosticEvent((event, _metadata, privateData) => { + if (event.type.startsWith("model.call.")) { + events.push({ event, privateData }); + } + }); + try { + await run(); + await new Promise((resolve) => { + setImmediate(resolve); + }); + return events; + } finally { + stop(); + } +} + +async function collectSemanticProgressEvents(run: () => Promise) { + const events: DiagnosticEventPayload[] = []; + const stop = onInternalDiagnosticEvent((event, metadata) => { + if ( + isCoreSemanticRunProgressDiagnosticMetadata(metadata) && + event.type === "run.progress" && + event.reason === "model_call:semantic_result" + ) { + events.push(event); + } + }); + try { + await run(); + await waitForDiagnosticEventsDrained(); + return events; + } finally { + stop(); + } +} + +function assistantResult(stopReason: string, content: unknown[]) { + return { role: "assistant", stopReason, content }; +} + +async function drain(stream: AsyncIterable): Promise { + // Force stream iteration so completion events include response byte and timing + // accounting. + for await (const _ of stream) { + // drain + } +} + +const requireRecord = createRequireRecord("record", "expected-label-object-capitalized"); + +function expectNumberField(record: Record, key: string) { + expect(typeof record[key]).toBe("number"); +} + +function getEvent(events: readonly DiagnosticEventPayload[], index: number) { + return requireRecord(events[index], `event ${index}`); +} + +describe("wrapStreamFnWithDiagnosticModelCallEvents observation", () => { + beforeEach(() => { + resetDiagnosticEventsForTest(); + resetDiagnosticRunActivityForTest(); + startDiagnosticRunActivityTracking(); + resetGlobalHookRunner(); + }); + + afterEach(() => { + resetDiagnosticEventsForTest(); + resetGlobalHookRunner(); + resetDiagnosticRunActivityForTest(); + vi.restoreAllMocks(); + vi.useRealTimers(); + }); + + it.each([ + { + name: "visible text", + result: assistantResult("stop", [{ type: "text", text: "done" }]), + expected: 1, + }, + { + name: "tool-use call", + result: assistantResult("toolUse", [ + { type: "toolCall", id: "call-1", name: "read", arguments: { path: "README.md" } }, + ]), + expected: 1, + }, + { + name: "error text", + result: assistantResult("error", [{ type: "text", text: "provider failed" }]), + expected: 0, + }, + { + name: "aborted text", + result: assistantResult("aborted", [{ type: "text", text: "partial" }]), + expected: 0, + }, + { + name: "reasoning only", + result: assistantResult("stop", [{ type: "thinking", thinking: "working" }]), + expected: 0, + }, + { + name: "blank text", + result: assistantResult("stop", [{ type: "text", text: " \n" }]), + expected: 0, + }, + { + name: "non-executable tool block", + result: assistantResult("stop", [ + { type: "toolCall", id: "call-1", name: "read", arguments: {} }, + ]), + expected: 0, + }, + { + name: "malformed tool-use call", + result: assistantResult("toolUse", [{ type: "toolCall", id: "", name: "read" }]), + expected: 0, + }, + ])("emits semantic progress once for $name final results", async ({ result, expected }) => { + const stream = { + async *[Symbol.asyncIterator]() {}, + result: async () => result, + }; + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (() => stream) as unknown as StreamFn, + { + runId: "run-semantic-result", + sessionId: "session-semantic-result", + provider: "openai", + model: "gpt-5.4", + trace: createDiagnosticTraceContext(), + nextCallId: () => "call-semantic-result", + }, + ); + + const events = await collectSemanticProgressEvents(async () => { + const observed = wrapped({} as never, {} as never, {} as never) as unknown as typeof stream; + await observed.result(); + await observed.result(); + }); + + expect(events).toHaveLength(expected); + }); + + it("orders semantic results between repeated request observations", async () => { + const ref = { + sessionId: "session-semantic-order", + sessionKey: "agent:main:semantic-order", + }; + const runId = "run-semantic-order"; + const results = [ + assistantResult("error", [{ type: "text", text: "retry one" }]), + assistantResult("error", [{ type: "text", text: "retry two" }]), + assistantResult("stop", [{ type: "text", text: "made progress" }]), + assistantResult("error", [{ type: "text", text: "retry after progress" }]), + ]; + let callSequence = 0; + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (() => { + const result = results.shift(); + return { + async *[Symbol.asyncIterator]() {}, + result: async () => result, + }; + }) as unknown as StreamFn, + { + ...ref, + runId, + provider: "openai", + model: "gpt-5.4", + trace: createDiagnosticTraceContext(), + nextCallId: () => `${runId}:${(callSequence += 1)}`, + }, + ); + markDiagnosticEmbeddedRunStarted({ ...ref, runId }); + + const repeatedRequestAges: Array = []; + for (let index = 0; index < 4; index += 1) { + const observed = wrapped({} as never, {} as never, {} as never) as unknown as { + result: () => Promise; + }; + await observed.result(); + await waitForDiagnosticEventsDrained(); + repeatedRequestAges.push( + getDiagnosticSessionActivitySnapshot(ref).repeatedRequestNoProgressAgeMs, + ); + } + + expect(repeatedRequestAges).toEqual([undefined, expect.any(Number), undefined, undefined]); + + expect(getDiagnosticSessionActivitySnapshot(ref)).toMatchObject({ + hasActiveEmbeddedRun: true, + repeatedRequestNoProgressAgeMs: undefined, + }); + }); + + it("updates diagnostic run activity from throttled stream chunks", async () => { + let now = 1_000_000; + vi.spyOn(Date, "now").mockImplementation(() => now); + async function* stream() { + yield { type: "text_delta", delta: "first" }; + yield { type: "text_delta", delta: "second" }; + yield { type: "text_delta", delta: "third" }; + } + const runProgressEvents: DiagnosticEventPayload[] = []; + const stop = onInternalDiagnosticEvent((event) => { + if (event.type === "run.progress") { + runProgressEvents.push(event); + } + }); + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (() => stream()) as unknown as StreamFn, + { + runId: "run-1", + sessionKey: "session-key", + sessionId: "session-id", + provider: "vllm", + model: "qwen/qwen3.5-9b", + trace: createDiagnosticTraceContext(), + nextCallId: () => "call-stream", + }, + ); + + const returned = wrapped({} as never, {} as never, {} as never) as AsyncIterable; + const iterator = returned[Symbol.asyncIterator](); + + try { + await iterator.next(); + await waitForDiagnosticEventsDrained(); + let snapshot = getDiagnosticSessionActivitySnapshot({ + sessionKey: "session-key", + sessionId: "session-id", + }); + expect(snapshot.activeWorkKind).toBe("model_call"); + expect(snapshot.lastProgressReason).toBe("model_call:stream_progress"); + expect(snapshot.lastProgressAgeMs).toBe(0); + expect(runProgressEvents).toHaveLength(1); + + now += 10_000; + await iterator.next(); + await waitForDiagnosticEventsDrained(); + snapshot = getDiagnosticSessionActivitySnapshot({ + sessionKey: "session-key", + sessionId: "session-id", + }); + expect(snapshot.lastProgressReason).toBe("model_call:stream_progress"); + expect(snapshot.lastProgressAgeMs).toBe(0); + expect(runProgressEvents).toHaveLength(1); + + now += 30_000; + await iterator.next(); + await waitForDiagnosticEventsDrained(); + snapshot = getDiagnosticSessionActivitySnapshot({ + sessionKey: "session-key", + sessionId: "session-id", + }); + expect(snapshot.lastProgressReason).toBe("model_call:stream_progress"); + expect(snapshot.lastProgressAgeMs).toBe(0); + expect(runProgressEvents).toHaveLength(2); + expect(runProgressEvents.every((event) => event.type === "run.progress")).toBe(true); + expect(runProgressEvents.every((event) => !("progressKind" in event))).toBe(true); + } finally { + await iterator.return?.(); + await waitForDiagnosticEventsDrained(); + stop(); + } + }); + + it("does not retain stream progress activity when diagnostics are disabled", async () => { + setDiagnosticsEnabledForProcess(false); + const runProgressEvents: DiagnosticEventPayload[] = []; + const stop = onInternalDiagnosticEvent((event) => { + if (event.type === "run.progress") { + runProgressEvents.push(event); + } + }); + async function* stream() { + yield { type: "text_delta", delta: "first" }; + yield { type: "text_delta", delta: "second" }; + } + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (() => stream()) as unknown as StreamFn, + { + runId: "run-1", + sessionKey: "session-key", + sessionId: "session-id", + provider: "vllm", + model: "qwen/qwen3.5-9b", + trace: createDiagnosticTraceContext(), + nextCallId: () => "call-disabled-diagnostics", + }, + ); + + try { + await drain(wrapped({} as never, {} as never, {} as never) as AsyncIterable); + await waitForDiagnosticEventsDrained(); + } finally { + stop(); + } + + expect( + getDiagnosticSessionActivitySnapshot({ + sessionKey: "session-key", + sessionId: "session-id", + }), + ).toEqual({}); + expect(runProgressEvents).toEqual([]); + }); + + it("counts async onPayload replacements instead of raw payload content", async () => { + async function* stream() { + yield { type: "text_delta", delta: "safe" }; + } + const originalPayload = { input: "secret sk-original-secret" }; + const replacementPayload = { input: "redacted" }; + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (async ( + model: Parameters[0], + _context: Parameters[1], + options: Parameters[2], + ) => { + await options?.onPayload?.(originalPayload, model); + return stream(); + }) as unknown as StreamFn, + { + runId: "run-1", + provider: "openai", + model: "gpt-5.4", + trace: createDiagnosticTraceContext(), + nextCallId: () => "call-payload", + }, + ); + + const events = await collectModelCallEvents(async () => { + const streamResult = await wrapped({} as never, {} as never, { + onPayload: async () => replacementPayload, + }); + await drain(streamResult as unknown as AsyncIterable); + }); + + const completedEvent = getEvent(events, 1); + expect(completedEvent.type).toBe("model.call.completed"); + expect(completedEvent.callId).toBe("call-payload"); + expect(completedEvent.requestPayloadBytes).toBe( + Buffer.byteLength(JSON.stringify(replacementPayload), "utf8"), + ); + expectNumberField(completedEvent, "responseStreamBytes"); + expectNumberField(completedEvent, "timeToFirstByteMs"); + expect(JSON.stringify(events)).not.toContain("sk-original-secret"); + }); + + it("counts text deltas without serializing full partial snapshots", async () => { + const serializedPartial = vi.fn(() => { + throw new Error("partial snapshot should not be serialized for text deltas"); + }); + async function* stream() { + yield { + type: "text_delta", + contentIndex: 0, + delta: "a", + partial: { + toJSON: serializedPartial, + role: "assistant", + content: [{ type: "text", text: "a".repeat(200_000) }], + }, + }; + yield { + type: "text_delta", + contentIndex: 0, + delta: "bc", + partial: { + toJSON: serializedPartial, + role: "assistant", + content: [{ type: "text", text: "abc".repeat(200_000) }], + }, + }; + } + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (() => stream()) as unknown as StreamFn, + { + runId: "run-1", + provider: "openai", + model: "gpt-5.4", + trace: createDiagnosticTraceContext(), + nextCallId: () => "call-delta-bytes", + }, + ); + + const events = await collectModelCallEvents(async () => { + await drain(wrapped({} as never, {} as never, {} as never) as AsyncIterable); + }); + + const completedEvent = getEvent(events, 1); + expect(completedEvent.type).toBe("model.call.completed"); + expect(completedEvent.responseStreamBytes).toBe(Buffer.byteLength("abc", "utf8")); + expect(serializedPartial).not.toHaveBeenCalled(); + }); + + it("keeps streams alive when diagnostic byte inspection cannot read a chunk", async () => { + const opaqueChunk = new Proxy( + {}, + { + get(_target, property) { + if (property === "then") { + return undefined; + } + throw new Error("chunk should not be inspected"); + }, + }, + ); + async function* stream() { + yield opaqueChunk; + yield { type: "text_delta", delta: "ok" }; + } + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (() => stream()) as unknown as StreamFn, + { + runId: "run-1", + provider: "openai", + model: "gpt-5.4", + trace: createDiagnosticTraceContext(), + nextCallId: () => "call-opaque-chunk", + }, + ); + + const chunks: unknown[] = []; + const events = await collectModelCallEvents(async () => { + for await (const chunk of wrapped( + {} as never, + {} as never, + {} as never, + ) as AsyncIterable) { + chunks.push(chunk); + } + }); + + expect(chunks).toHaveLength(2); + expect(chunks[0]).toBe(opaqueChunk); + expect(chunks[1]).toEqual({ type: "text_delta", delta: "ok" }); + const completedEvent = getEvent(events, 1); + expect(completedEvent.type).toBe("model.call.completed"); + expect(completedEvent.responseStreamBytes).toBe(Buffer.byteLength("ok", "utf8")); + }); + + it("captures model input, tools, and output only when content capture is enabled", async () => { + const assistant = { + role: "assistant", + content: [{ type: "text", text: "trace reply" }], + api: "openai-responses", + provider: "openai", + model: "gpt-5.4", + usage: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 2 }, + stopReason: "stop", + timestamp: 1, + }; + async function* stream() { + yield { type: "done", reason: "stop", message: assistant }; + } + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (() => stream()) as unknown as StreamFn, + { + runId: "run-1", + provider: "openai", + model: "gpt-5.4", + trace: createDiagnosticTraceContext(), + contentCapture: { + inputMessages: true, + outputMessages: true, + toolInputs: false, + toolOutputs: false, + systemPrompt: true, + toolDefinitions: true, + anyModelContent: true, + }, + nextCallId: () => "call-content", + }, + ); + + const inputMessages = [{ role: "user", content: "trace prompt", timestamp: 1 }]; + const tools = [{ name: "lookup", description: "Lookup data", parameters: { type: "object" } }]; + const events = await collectTrustedModelCallEvents(async () => { + const streamResult = wrapped( + {} as never, + { + systemPrompt: "trace system", + messages: inputMessages, + tools, + } as never, + {}, + ); + await drain(streamResult as unknown as AsyncIterable); + }); + + const startedEvent = getEvent( + events.map((entry) => entry.event), + 0, + ); + expect(startedEvent.type).toBe("model.call.started"); + expect(startedEvent.inputMessages).toBeUndefined(); + expect(startedEvent.systemPrompt).toBeUndefined(); + expect(startedEvent.toolDefinitions).toBeUndefined(); + expect(events[0]?.privateData.modelContent?.inputMessages).toEqual(inputMessages); + expect(events[0]?.privateData.modelContent?.systemPrompt).toBe("trace system"); + expect(events[0]?.privateData.modelContent?.toolDefinitions).toEqual(tools); + const completedEvent = getEvent( + events.map((entry) => entry.event), + 1, + ); + expect(completedEvent.type).toBe("model.call.completed"); + expect(completedEvent.outputMessages).toBeUndefined(); + expect(events[1]?.privateData.modelContent?.inputMessages).toEqual(inputMessages); + expect(events[1]?.privateData.modelContent?.outputMessages).toEqual([assistant]); + }); + + it("emits safe prompt stats and per-call usage without content capture", async () => { + const assistant = { + role: "assistant", + content: [{ type: "text", text: "trace reply" }], + usage: { + input: 11, + output: 7, + cacheRead: 3, + cacheWrite: 2, + reasoningTokens: 5, + totalTokens: 28, + }, + timestamp: 1, + }; + async function* stream() { + yield { type: "done", reason: "stop", message: assistant }; + } + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (() => stream()) as unknown as StreamFn, + { + runId: "run-1", + provider: "openai", + model: "gpt-5.4", + trace: createDiagnosticTraceContext(), + nextCallId: () => "call-stats", + }, + ); + + const inputMessages = [{ role: "user", content: "private prompt text", timestamp: 1 }]; + const tools = [ + { name: "lookup", description: "private tool description", parameters: { type: "object" } }, + ]; + const systemPrompt = "private system prompt"; + const events = await collectModelCallEvents(async () => { + const streamResult = wrapped( + {} as never, + { + systemPrompt, + messages: inputMessages, + tools, + } as never, + {}, + ); + await drain(streamResult as unknown as AsyncIterable); + }); + + const startedEvent = getEvent(events, 0); + const completedEvent = getEvent(events, 1); + const expectedPromptStats = { + inputMessagesCount: inputMessages.length, + inputMessagesChars: JSON.stringify(inputMessages).length, + systemPromptChars: systemPrompt.length, + toolDefinitionsCount: tools.length, + toolDefinitionsChars: JSON.stringify(tools).length, + totalChars: + JSON.stringify(inputMessages).length + systemPrompt.length + JSON.stringify(tools).length, + }; + expect(startedEvent.promptStats).toEqual(expectedPromptStats); + expect(completedEvent.promptStats).toEqual(expectedPromptStats); + expect(completedEvent.usage).toEqual({ + input: 11, + output: 7, + cacheRead: 3, + cacheWrite: 2, + reasoningTokens: 5, + total: 28, + promptTokens: 16, + }); + expect(JSON.stringify(events)).not.toContain("private prompt text"); + expect(JSON.stringify(events)).not.toContain("private system prompt"); + expect(JSON.stringify(events)).not.toContain("private tool description"); + }); + + it("captures per-call usage from terminal error events", async () => { + // Aborted/error streams terminate with an `error` event carrying the final + // AssistantMessage and its usage. Iterating to completion without awaiting + // result() must still surface per-call usage, matching the `done` path and + // the usage field already emitted on model.call.error and its OTel span. + const assistant = { + role: "assistant", + content: [{ type: "text", text: "partial reply" }], + usage: { + input: 11, + output: 7, + cacheRead: 3, + cacheWrite: 2, + reasoningTokens: 5, + totalTokens: 28, + }, + stopReason: "aborted", + timestamp: 1, + }; + async function* stream() { + yield { type: "error", reason: "aborted", error: assistant }; + } + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (() => stream()) as unknown as StreamFn, + { + runId: "run-1", + provider: "openrouter", + model: "openrouter/auto", + trace: createDiagnosticTraceContext(), + nextCallId: () => "call-error-usage", + }, + ); + + const events = await collectModelCallEvents(async () => { + await drain(wrapped({} as never, {} as never, {} as never) as AsyncIterable); + }); + + // An in-band error event is data, not a throw, so iteration completes + // normally; the per-call usage rides on the terminal completion event. + const completedEvent = getEvent(events, 1); + expect(completedEvent.type).toBe("model.call.completed"); + expect(completedEvent.usage).toEqual({ + input: 11, + output: 7, + cacheRead: 3, + cacheWrite: 2, + reasoningTokens: 5, + total: 28, + promptTokens: 16, + }); + }); + + it("skips prompt stat computation when diagnostics are disabled", async () => { + // Prompt stats are only attached to diagnostic events; when diagnostics are + // off those events are dropped, so the JSON.stringify of input messages and + // tool definitions must not run on the model-call hot path. + setDiagnosticsEnabledForProcess(false); + let promptInspected = false; + const streamContext = { + systemPrompt: "system", + get messages() { + promptInspected = true; + return [{ role: "user", content: "x", timestamp: 1 }]; + }, + get tools() { + promptInspected = true; + return [{ name: "lookup", description: "d", parameters: { type: "object" } }]; + }, + }; + async function* stream() { + yield { type: "text_delta", delta: "ok" }; + } + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (() => stream()) as unknown as StreamFn, + { + runId: "run-1", + provider: "openai", + model: "gpt-5.4", + trace: createDiagnosticTraceContext(), + nextCallId: () => "call-disabled-prompt-stats", + }, + ); + + await drain( + wrapped({} as never, streamContext as never, {} as never) as AsyncIterable, + ); + + expect(promptInspected).toBe(false); + }); +}); diff --git a/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-observation.ts b/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-observation.ts new file mode 100644 index 000000000000..ce0e0edb928a --- /dev/null +++ b/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-observation.ts @@ -0,0 +1,391 @@ +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { + areDiagnosticsEnabledForProcess, + emitTrustedDiagnosticEvent, + type DiagnosticModelCallContent, +} from "../../../infra/diagnostic-events.js"; +import { + cloneDiagnosticContentValue, + type DiagnosticModelContentCapturePolicy, +} from "../../../infra/diagnostic-llm-content.js"; +import { emitCoreSemanticRunProgressDiagnosticEvent } from "../../../infra/diagnostic-semantic-run-progress.js"; +import { markDiagnosticRunProgress } from "../../../logging/diagnostic-run-activity.js"; +import { derivePromptTokens, normalizeUsage, type UsageLike } from "../../usage.js"; +import type { + ModelCallEventBase, + ModelCallObservationState, + ModelCallObserver, + ModelCallPromptStats, + ModelCallSizeTimingFields, + ModelCallUsage, +} from "./attempt.model-diagnostic-lifecycle.js"; + +const MODEL_CALL_STREAM_PROGRESS_INTERVAL_MS = 30_000; +const MODEL_CALL_STREAM_PROGRESS_REASON = "model_call:stream_progress"; +const MODEL_CALL_SEMANTIC_PROGRESS_REASON = "model_call:semantic_result"; + +function utf8JsonByteLength(value: unknown): number | undefined { + try { + return Buffer.byteLength(JSON.stringify(value), "utf8"); + } catch { + return undefined; + } +} + +function assignRequestPayloadBytes(state: ModelCallObservationState, payload: unknown): void { + const bytes = utf8JsonByteLength(payload); + if (bytes !== undefined) { + state.requestPayloadBytes = bytes; + } +} + +function utf8StringByteLength(value: string): number { + return Buffer.byteLength(value, "utf8"); +} + +function jsonCharLength(value: unknown): number | undefined { + try { + return JSON.stringify(value)?.length; + } catch { + return undefined; + } +} + +function streamDeltaByteLength(chunk: Record): number | undefined { + const type = chunk.type; + if ( + (type === "text_delta" || type === "thinking_delta" || type === "toolcall_delta") && + typeof chunk.delta === "string" + ) { + return utf8StringByteLength(chunk.delta); + } + return undefined; +} + +function responseStreamChunkByteLengthUnchecked(chunk: unknown): number | undefined { + if (!isRecord(chunk)) { + return utf8JsonByteLength(chunk); + } + const deltaBytes = streamDeltaByteLength(chunk); + if (deltaBytes !== undefined) { + return deltaBytes; + } + if (!("partial" in chunk)) { + return utf8JsonByteLength(chunk); + } + // Plain stream deltas can carry an accumulated partial snapshot. Byte metrics + // count the new stream payload, not the answer-so-far replay. + const { partial: _partial, ...snapshotlessChunk } = chunk; + return utf8JsonByteLength(snapshotlessChunk); +} + +function responseStreamChunkByteLength(chunk: unknown): number | undefined { + try { + return responseStreamChunkByteLengthUnchecked(chunk); + } catch { + return undefined; + } +} + +function streamContextModelContentFields( + policy: DiagnosticModelContentCapturePolicy | undefined, + streamContext: unknown, +): DiagnosticModelCallContent | undefined { + if (!policy?.anyModelContent || !isRecord(streamContext)) { + return undefined; + } + const content = { + ...(policy.inputMessages && Array.isArray(streamContext.messages) + ? { inputMessages: cloneDiagnosticContentValue(streamContext.messages) } + : {}), + ...(policy.systemPrompt && typeof streamContext.systemPrompt === "string" + ? { systemPrompt: streamContext.systemPrompt } + : {}), + ...(policy.toolDefinitions && Array.isArray(streamContext.tools) + ? { toolDefinitions: cloneDiagnosticContentValue(streamContext.tools) } + : {}), + }; + return Object.keys(content).length > 0 ? content : undefined; +} + +function streamContextModelPromptStats(streamContext: unknown): ModelCallPromptStats | undefined { + if (!isRecord(streamContext)) { + return undefined; + } + const messages = Array.isArray(streamContext.messages) ? streamContext.messages : undefined; + const tools = Array.isArray(streamContext.tools) ? streamContext.tools : undefined; + const systemPrompt = + typeof streamContext.systemPrompt === "string" ? streamContext.systemPrompt : undefined; + const inputMessagesChars = messages ? jsonCharLength(messages) : undefined; + const toolDefinitionsChars = tools ? jsonCharLength(tools) : undefined; + const systemPromptChars = systemPrompt?.length; + if ( + messages === undefined && + tools === undefined && + systemPromptChars === undefined && + inputMessagesChars === undefined && + toolDefinitionsChars === undefined + ) { + return undefined; + } + const totalChars = + (inputMessagesChars ?? 0) + (systemPromptChars ?? 0) + (toolDefinitionsChars ?? 0); + return { + ...(messages ? { inputMessagesCount: messages.length } : {}), + ...(inputMessagesChars !== undefined ? { inputMessagesChars } : {}), + ...(systemPromptChars !== undefined ? { systemPromptChars } : {}), + ...(tools ? { toolDefinitionsCount: tools.length } : {}), + ...(toolDefinitionsChars !== undefined ? { toolDefinitionsChars } : {}), + totalChars, + }; +} + +function normalizedModelCallUsage(rawUsage: unknown): ModelCallUsage | undefined { + if (!isRecord(rawUsage)) { + return undefined; + } + const usage = normalizeUsage(rawUsage as UsageLike); + if (!usage) { + return undefined; + } + const promptTokens = derivePromptTokens(usage); + return { + ...usage, + ...(promptTokens !== undefined ? { promptTokens } : {}), + }; +} + +function observeModelCallUsage(state: ModelCallObservationState, value: unknown): void { + if (!isRecord(value)) { + return; + } + let rawUsage: unknown; + try { + rawUsage = value.usage; + } catch { + return; + } + const usage = normalizedModelCallUsage(rawUsage); + if (usage) { + state.usage = usage; + } +} + +function observeOutputMessageContent(state: ModelCallObservationState, chunk: unknown): void { + if (!isRecord(chunk)) { + return; + } + let type: unknown; + let message: unknown; + try { + type = chunk.type; + message = type === "done" ? chunk.message : type === "error" ? chunk.error : undefined; + } catch { + return; + } + // Terminal events carry the final AssistantMessage with usage — `done` for + // success, `error` for aborted/error streams. Capture usage from either so + // iterated error-terminated calls still report the per-call usage that the + // model.call.error event and its OTel span already expose. + if (message !== undefined) { + observeModelCallUsage(state, message); + if (state.contentCapture?.outputMessages) { + state.outputMessages = [cloneDiagnosticContentValue(message)]; + } + } +} + +function observeResultMessageContent( + state: ModelCallObservationState, + startedAt: number, + result: unknown, +): void { + state.timeToFirstByteMs ??= Math.max(0, Date.now() - startedAt); + observeModelCallUsage(state, result); + if (state.contentCapture?.outputMessages && state.outputMessages === undefined) { + state.outputMessages = [cloneDiagnosticContentValue(result)]; + } + if (state.responseStreamBytes === 0) { + const bytes = utf8JsonByteLength(result); + if (bytes !== undefined) { + state.responseStreamBytes = bytes; + } + } +} + +function isNormalizedToolCall(value: unknown): boolean { + if (!isRecord(value) || value.type !== "toolCall") { + return false; + } + return ( + typeof value.id === "string" && + value.id.trim().length > 0 && + typeof value.name === "string" && + value.name.trim().length > 0 && + isRecord(value.arguments) + ); +} + +function isSemanticModelCallResult(result: unknown): boolean { + try { + if ( + !isRecord(result) || + result.role !== "assistant" || + result.stopReason === "error" || + result.stopReason === "aborted" || + !Array.isArray(result.content) + ) { + return false; + } + const hasExecutableToolCall = + result.stopReason === "toolUse" && result.content.some(isNormalizedToolCall); + return ( + hasExecutableToolCall || + result.content.some( + (item) => + isRecord(item) && + item.type === "text" && + typeof item.text === "string" && + item.text.trim().length > 0, + ) + ); + } catch { + return false; + } +} + +function maybeEmitModelCallSemanticProgress( + eventBase: ModelCallEventBase, + state: ModelCallObservationState, + result: unknown, +): void { + if (state.semanticProgressEmitted || !isSemanticModelCallResult(result)) { + return; + } + state.semanticProgressEmitted = true; + emitCoreSemanticRunProgressDiagnosticEvent({ + runId: eventBase.runId, + ...(eventBase.sessionKey ? { sessionKey: eventBase.sessionKey } : {}), + ...(eventBase.sessionId ? { sessionId: eventBase.sessionId } : {}), + reason: MODEL_CALL_SEMANTIC_PROGRESS_REASON, + }); +} + +function observeResponseChunk( + state: ModelCallObservationState, + startedAt: number, + chunk: unknown, +): void { + state.timeToFirstByteMs ??= Math.max(0, Date.now() - startedAt); + observeOutputMessageContent(state, chunk); + const bytes = responseStreamChunkByteLength(chunk); + if (bytes !== undefined) { + state.responseStreamBytes += bytes; + } +} + +function maybeEmitModelCallStreamProgress( + eventBase: ModelCallEventBase, + state: ModelCallObservationState, +): void { + if (!areDiagnosticsEnabledForProcess()) { + return; + } + const now = Date.now(); + const progressFields = { + runId: eventBase.runId, + ...(eventBase.sessionKey ? { sessionKey: eventBase.sessionKey } : {}), + ...(eventBase.sessionId ? { sessionId: eventBase.sessionId } : {}), + reason: MODEL_CALL_STREAM_PROGRESS_REASON, + }; + markDiagnosticRunProgress(progressFields); + if ( + state.lastStreamProgressAt !== undefined && + now - state.lastStreamProgressAt < MODEL_CALL_STREAM_PROGRESS_INTERVAL_MS + ) { + return; + } + state.lastStreamProgressAt = now; + // Streaming providers, local or remote, are expected to produce chunks or + // heartbeat-style progress. The in-memory freshness clock is refreshed for + // each chunk, while diagnostic events are throttled so token streams do not + // spam observers; silent/non-streaming calls remain recoverable after the + // configured stuck-session timeout. + emitTrustedDiagnosticEvent({ + type: "run.progress", + ...progressFields, + }); +} + +function modelCallSizeTimingFields(state: ModelCallObservationState): ModelCallSizeTimingFields { + return { + ...(state.requestPayloadBytes !== undefined + ? { requestPayloadBytes: state.requestPayloadBytes } + : {}), + ...(state.responseStreamBytes > 0 ? { responseStreamBytes: state.responseStreamBytes } : {}), + ...(state.timeToFirstByteMs !== undefined + ? { timeToFirstByteMs: state.timeToFirstByteMs } + : {}), + }; +} + +function modelCallCompletedContent(state: ModelCallObservationState) { + if (!state.modelContent && !state.outputMessages) { + return undefined; + } + return { + ...state.modelContent, + ...(state.outputMessages ? { outputMessages: state.outputMessages } : {}), + }; +} + +function modelCallUsageField(state: ModelCallObservationState) { + return state.usage ? { usage: state.usage } : {}; +} + +export function createModelObserver(params: { + streamContext: unknown; + contentCapture?: DiagnosticModelContentCapturePolicy; + suppressPluginHooks?: boolean; + capturePromptStats: boolean; +}): ModelCallObserver { + const modelContent = streamContextModelContentFields(params.contentCapture, params.streamContext); + const promptStats = params.capturePromptStats + ? streamContextModelPromptStats(params.streamContext) + : undefined; + const state: ModelCallObservationState = { + responseStreamBytes: 0, + modelContent, + contentCapture: params.contentCapture, + suppressPluginHooks: params.suppressPluginHooks, + }; + return { + state, + promptStats, + modelContent, + assignRequestPayloadBytes(payload) { + assignRequestPayloadBytes(state, payload); + }, + observeResponseChunk(startedAt, chunk) { + observeResponseChunk(state, startedAt, chunk); + }, + observeFinalResult(eventBase, startedAt, result) { + observeResultMessageContent(state, startedAt, result); + // Queue semantic progress beside model lifecycle events so request starts, + // progress, and the next request retain their authoritative FIFO ordering. + maybeEmitModelCallSemanticProgress(eventBase, state, result); + }, + maybeEmitStreamProgress(eventBase) { + maybeEmitModelCallStreamProgress(eventBase, state); + }, + sizeTimingFields() { + return modelCallSizeTimingFields(state); + }, + completedContent() { + return modelCallCompletedContent(state); + }, + usageField() { + return modelCallUsageField(state); + }, + }; +} diff --git a/src/agents/embedded-agent-runner/run/payloads.errors.test.ts b/src/agents/embedded-agent-runner/run/payloads.errors.test.ts index 79d4c8308c5e..6d11a86d93b7 100644 --- a/src/agents/embedded-agent-runner/run/payloads.errors.test.ts +++ b/src/agents/embedded-agent-runner/run/payloads.errors.test.ts @@ -66,22 +66,6 @@ describe("buildEmbeddedRunPayloads", () => { expect(payloads[0]?.isError).toBe(expected.isError); } - function expectNoPayloads(params: Parameters[0]) { - const payloads = buildPayloads(params); - expect(payloads).toHaveLength(0); - } - - function expectNoSyntheticCompletionForSession(sessionKey: string) { - expectNoPayloads({ - sessionKey, - lastAssistant: makeAssistant({ - stopReason: "stop", - errorMessage: undefined, - content: [], - }), - }); - } - it("suppresses raw API error JSON when the assistant errored", () => { const payloads = buildPayloads({ assistantTexts: [errorJson], @@ -488,677 +472,4 @@ describe("buildEmbeddedRunPayloads", () => { expectSinglePayloadText(payloads, errorJsonPretty.trim()); }); - - it("adds a fallback error when a tool fails and no assistant output exists", () => { - const payloads = buildPayloads({ - lastToolError: { toolName: "browser", error: "tab not found" }, - }); - - expectSingleToolErrorPayload(payloads, { - title: "Browser", - absentDetail: "tab not found", - }); - }); - - it("does not add tool error fallback when assistant output exists", () => { - const payloads = buildPayloads({ - assistantTexts: ["All good"], - lastAssistant: makeStoppedAssistant(), - lastToolError: { toolName: "browser", error: "tab not found" }, - }); - - expectSinglePayloadText(payloads, "All good"); - }); - - it("does not add synthetic completion text for channel sessions", () => { - expectNoSyntheticCompletionForSession("agent:main:discord:channel:c123"); - }); - - it("does not add synthetic completion text for group sessions", () => { - expectNoSyntheticCompletionForSession("agent:main:telegram:group:g123"); - }); - - it("does not add synthetic completion text when messaging tool already delivered output", () => { - expectNoPayloads({ - sessionKey: "agent:main:discord:direct:u123", - didSendViaMessagingTool: true, - lastAssistant: makeAssistant({ - stopReason: "stop", - errorMessage: undefined, - content: [], - }), - }); - }); - - it("does not add synthetic completion text when the run still has a tool error", () => { - expectNoPayloads({ - lastToolError: { toolName: "browser", error: "url required" }, - }); - }); - - it("does not add synthetic completion text when no tools ran", () => { - expectNoPayloads({ - lastAssistant: makeStoppedAssistant(), - }); - }); - - it("adds compact tool error fallback when the assistant only invoked tools and verbose mode is on", () => { - const payloads = buildPayloads({ - lastAssistant: makeAssistant({ - stopReason: "toolUse", - errorMessage: undefined, - content: [ - { - type: "toolCall", - id: "toolu_01", - name: "exec", - arguments: { command: "echo hi" }, - }, - ], - }), - lastToolError: { toolName: "exec", error: "Command exited with code 1" }, - verboseLevel: "on", - }); - - expectSingleToolErrorPayload(payloads, { - title: "Exec", - absentDetail: "code 1", - }); - }); - - it("does not add tool error fallback when assistant text exists after tool calls", () => { - const payloads = buildPayloads({ - assistantTexts: ["Checked the page and recovered with final answer."], - lastAssistant: makeAssistant({ - stopReason: "toolUse", - errorMessage: undefined, - content: [ - { - type: "toolCall", - id: "toolu_01", - name: "browser", - arguments: { action: "search", query: "openclaw docs" }, - }, - ], - }), - lastToolError: { toolName: "browser", error: "connection timeout" }, - }); - - expectSinglePayloadSummary(payloads, { - text: "Checked the page and recovered with final answer.", - }); - }); - - it.each(["url required", "url missing", "invalid parameter: url"])( - "suppresses recoverable non-mutating tool error: %s", - (error) => { - expectNoPayloads({ - lastToolError: { toolName: "browser", error }, - }); - }, - ); - - it("suppresses non-mutating non-recoverable tool errors when messages.suppressToolErrors is enabled", () => { - expectNoPayloads({ - lastToolError: { toolName: "browser", error: "connection timeout" }, - config: { messages: { suppressToolErrors: true } }, - }); - }); - - it("suppresses mutating tool errors when suppressToolErrorWarnings is enabled", () => { - expectNoPayloads({ - lastToolError: { toolName: "exec", error: "command not found" }, - suppressToolErrorWarnings: true, - }); - }); - - it.each([ - { - name: "suppresses mutating tool errors when messages.suppressToolErrors is enabled", - payload: { - lastToolError: { toolName: "write", error: "connection timeout" }, - config: { messages: { suppressToolErrors: true } }, - }, - title: "Write", - absentDetail: "connection timeout", - suppressed: true, - }, - { - name: "shows recoverable tool errors for mutating tools", - payload: { - lastToolError: { toolName: "message", meta: "reply", error: "text required" }, - }, - title: "Message", - absentDetail: "required", - }, - { - name: "shows non-recoverable tool failure summaries to the user", - payload: { - lastToolError: { toolName: "browser", error: "connection timeout" }, - }, - title: "Browser", - absentDetail: "connection timeout", - }, - ])("$name", ({ payload, title, absentDetail, suppressed }) => { - const payloads = buildPayloads(payload); - if (suppressed) { - expect(payloads).toEqual([]); - return; - } - expectSingleToolErrorPayload(payloads, { title, absentDetail }); - }); - - it("shows mutating tool errors when assistant output claims success", () => { - const payloads = buildPayloads({ - assistantTexts: ["Done."], - lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, - lastToolError: { toolName: "write", error: "file missing" }, - }); - - expect(payloads).toHaveLength(2); - expect(payloads[0]?.text).toBe("Done."); - expect(payloads[1]?.isError).toBe(true); - expect(payloads[1]?.text).toContain("Write"); - expect(payloads[1]?.text).not.toContain("missing"); - expect(getReplyPayloadMetadata(payloads[1] as object)?.nonTerminalToolErrorWarning).toBe( - undefined, - ); - }); - - it("still shows write tool errors when timedOut is true but no fileTarget was recorded", () => { - // Without `fileTarget` we cannot distinguish a confirmed file write from - // an unrelated mutating-tool timeout, so the default-visible warning is - // preserved to avoid hiding real failures. - const payloads = buildPayloads({ - assistantTexts: ["Done."], - lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, - lastToolError: { - toolName: "write", - error: "invoke timed out", - timedOut: true, - mutatingAction: true, - }, - }); - - expect(payloads).toHaveLength(2); - expect(payloads[1]?.isError).toBe(true); - expect(payloads[1]?.text).toContain("Write"); - }); - - it("still shows write tool errors when timedOut and fileTarget only prove the attempted path", () => { - const payloads = buildPayloads({ - assistantTexts: ["Done."], - lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, - lastToolError: { - toolName: "write", - error: "invoke timed out", - timedOut: true, - mutatingAction: true, - fileTarget: { path: "/tmp/openclaw/output.md" }, - }, - }); - - expect(payloads).toHaveLength(2); - expect(payloads[1]?.isError).toBe(true); - expect(payloads[1]?.text).toContain("Write"); - }); - - it("does not warn for timed-out exec errors when a successful user-facing reply exists", () => { - // Exec/bash use the generic recovery rule, not the mutating-tool branch: - // a successful final reply is proof the agent recovered (#103574). - const payloads = buildPayloads({ - assistantTexts: ["The script is ready."], - lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, - lastToolError: { - toolName: "exec", - error: "command timed out", - timedOut: true, - mutatingAction: true, - }, - }); - - expectSinglePayloadSummary(payloads, { text: "The script is ready." }); - }); - - it("does not warn for exec-like tool errors when a successful user-facing reply exists", () => { - // Production repro: mid-run bash/exec failure recovered with a correct final answer. - const payloads = buildPayloads({ - assistantTexts: ["The script is ready to use and saved in your workspace."], - lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, - lastToolError: { - toolName: "exec", - error: "/bin/bash: line 1: python: command not found", - mutatingAction: true, - }, - }); - - expectSinglePayloadSummary(payloads, { - text: "The script is ready to use and saved in your workspace.", - }); - }); - - it("does not warn for bash tool errors when a successful user-facing reply exists", () => { - const payloads = buildPayloads({ - assistantTexts: ["Recovered after the command failed."], - lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, - lastToolError: { - toolName: "bash", - error: "exit code 1", - mutatingAction: true, - }, - }); - - expectSinglePayloadSummary(payloads, { text: "Recovered after the command failed." }); - }); - - it("keeps exec-like tool error warnings when there is no user-facing reply", () => { - const payloads = buildPayloads({ - lastToolError: { - toolName: "exec", - error: "/bin/bash: line 1: python: command not found", - mutatingAction: true, - }, - }); - - expectSingleToolErrorPayload(payloads, { - title: "Exec", - absentDetail: "python: command not found", - }); - }); - - it("keeps exec-like tool error warnings for recoverable-looking errors when there is no reply", () => { - const payloads = buildPayloads({ - lastToolError: { - toolName: "bash", - error: "invalid argument: missing required flag --agent", - mutatingAction: true, - }, - }); - - expectSingleToolErrorPayload(payloads, { - title: "Bash", - absentDetail: "missing required flag", - }); - }); - - it("suppresses exec-like tool errors when messages.suppressToolErrors is enabled", () => { - expectNoPayloads({ - lastToolError: { - toolName: "bash", - error: "command not found", - mutatingAction: true, - }, - config: { messages: { suppressToolErrors: true } }, - }); - }); - - it("shows mutating tool errors when assistant output does not acknowledge the failure", () => { - const payloads = buildPayloads({ - assistantTexts: ["No issues found. The update is complete."], - lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, - lastToolError: { toolName: "edit", error: "file missing" }, - }); - - expect(payloads).toHaveLength(2); - expect(payloads[0]?.text).toBe("No issues found. The update is complete."); - expect(payloads[1]?.isError).toBe(true); - expect(payloads[1]?.text).toContain("Edit"); - expect(payloads[1]?.text).not.toContain("missing"); - }); - - it("shows mutating tool errors when assistant says it did not find issues in the file", () => { - const text = "I did not find any issues in the file. The update is complete."; - const payloads = buildPayloads({ - assistantTexts: [text], - lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, - lastToolError: { toolName: "edit", error: "file missing" }, - }); - - expect(payloads).toHaveLength(2); - expect(payloads[0]?.text).toBe(text); - expect(payloads[1]?.isError).toBe(true); - expect(payloads[1]?.text).toContain("Edit"); - expect(payloads[1]?.text).not.toContain("missing"); - }); - - it.each([ - "I did not need to update the file; it is already correct.", - "I did not have to edit the file because it was already correct.", - ])("shows mutating tool errors when assistant output uses no-op phrasing: %s", (text) => { - const payloads = buildPayloads({ - assistantTexts: [text], - lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, - lastToolError: { toolName: "edit", error: "file missing" }, - }); - - expect(payloads).toHaveLength(2); - expect(payloads[0]?.text).toBe(text); - expect(payloads[1]?.isError).toBe(true); - expect(payloads[1]?.text).toContain("Edit"); - expect(payloads[1]?.text).not.toContain("missing"); - }); - - it("suppresses mutating tool errors when assistant output explicitly acknowledges the failed action", () => { - const text = "I couldn't update the file, so no changes were applied."; - const payloads = buildPayloads({ - assistantTexts: [text], - lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, - lastToolError: { toolName: "edit", error: "file missing" }, - }); - - expectSinglePayloadSummary(payloads, { text }); - }); - - it("suppresses exec warnings when assistant output explicitly acknowledges the command failure", () => { - const text = "I couldn't run the command because python was not found."; - const payloads = buildPayloads({ - assistantTexts: [text], - lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, - lastToolError: { toolName: "exec", error: "/bin/bash: line 1: python: command not found" }, - }); - - expectSinglePayloadSummary(payloads, { text }); - }); - - it("does not treat session_status read failures as mutating when explicitly flagged", () => { - const payloads = buildPayloads({ - assistantTexts: ["Status loaded."], - lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, - lastToolError: { - toolName: "session_status", - error: "model required", - mutatingAction: false, - }, - }); - - expectSinglePayloadSummary(payloads, { text: "Status loaded." }); - }); - - it("dedupes identical tool warning text already present in assistant output", () => { - const seed = buildPayloads({ - lastToolError: { - toolName: "write", - error: "file missing", - mutatingAction: true, - }, - }); - const warningText = seed[0]?.text; - expect(warningText).toBe("⚠️ ✍️ Write failed"); - - const payloads = buildPayloads({ - assistantTexts: [warningText ?? ""], - lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, - lastToolError: { - toolName: "write", - error: "file missing", - mutatingAction: true, - }, - }); - - expectSinglePayloadSummary(payloads, { text: warningText ?? "" }); - }); - - it("hides exec command and cwd metadata without full verbosity", () => { - const payloads = buildPayloads({ - lastToolError: { - toolName: "exec", - meta: "run python3 /path/to/daily-cost-audit.py (in /private/workspace)", - error: "Command exited with code 1", - mutatingAction: true, - }, - toolResultFormat: "markdown", - verboseLevel: "off", - }); - - expectSinglePayloadSummary(payloads, { - text: "⚠️ 🛠️ Exec failed (exit 1)", - isError: true, - }); - }); - - it("keeps full-verbose exec failure labels outside markdown command text", () => { - const payloads = buildPayloads({ - lastToolError: { - toolName: "exec", - meta: "run python3 /path/to/daily-cost-audit.py", - error: "Command exited with code 1", - mutatingAction: true, - }, - toolResultFormat: "markdown", - verboseLevel: "full", - }); - - expectSinglePayloadSummary(payloads, { - text: "⚠️ 🛠️ Exec failed: `python3 /path/to/daily-cost-audit.py`: Command exited with code 1", - isError: true, - }); - expect(payloads[0]?.text).not.toContain("`run python3"); - }); - - it.each([ - { - title: "prefers raw exec metadata when tool progress detail includes it", - meta: "run python3 /tmp/audit.py · `python3 /tmp/audit.py`", - toolResultFormat: "markdown", - expected: "⚠️ 🛠️ Exec failed: `python3 /tmp/audit.py`: Command exited with code 1", - }, - { - title: "prefers raw exec metadata when the literal command contains backticks", - meta: "run node inline script, `node -e 'console.log(1, `x`)'`", - toolResultFormat: "markdown", - expected: "⚠️ 🛠️ Exec failed: ``node -e 'console.log(1, `x`)'``: Command exited with code 1", - }, - { - title: "leaves exec metadata unwrapped for plain tool results", - meta: "run node inline script, `node -e 'console.log(1, `x`)'`", - toolResultFormat: "plain", - expected: "⚠️ 🛠️ Exec failed: node -e 'console.log(1, `x`)': Command exited with code 1", - }, - { - title: "preserves raw exec context before trailing raw command metadata", - meta: "run python3 /tmp/audit.py, node: mac-1, `python3 /tmp/audit.py`", - toolResultFormat: "markdown", - expected: - "⚠️ 🛠️ Exec failed: `node: mac-1 · python3 /tmp/audit.py`: Command exited with code 1", - }, - { - title: "does not promote display-summary commas into raw exec context", - meta: 'search "foo,bar" in src, `rg "foo,bar" src`', - toolResultFormat: "markdown", - expected: '⚠️ 🛠️ Exec failed: `rg "foo,bar" src`: Command exited with code 1', - }, - { - title: "does not treat parenthesized raw command arguments as cwd context", - meta: 'list files in (in progress) · `ls "(in progress)"`', - toolResultFormat: "markdown", - expected: '⚠️ 🛠️ Exec failed: `ls "(in progress)"`: Command exited with code 1', - }, - { - title: "does not duplicate compact cwd labels already present in raw command arguments", - meta: 'print text (repo) · `printf "%s" "(repo)"`', - toolResultFormat: "markdown", - expected: '⚠️ 🛠️ Exec failed: `printf "%s" "(repo)"`: Command exited with code 1', - }, - { - title: "keeps arbitrary exec cwd suffixes inside markdown command text", - meta: "run python3 /tmp/audit.py (in /tmp/build @everyone)", - toolResultFormat: "markdown", - expected: - "⚠️ 🛠️ Exec failed: `python3 /tmp/audit.py (in /tmp/build @everyone)`: Command exited with code 1", - }, - ] as const)("$title", ({ meta, toolResultFormat, expected }) => { - const payloads = buildPayloads({ - lastToolError: { - toolName: "exec", - meta, - error: "Command exited with code 1", - mutatingAction: true, - }, - toolResultFormat, - verboseLevel: "full", - }); - - expectSinglePayloadSummary(payloads, { - text: expected, - isError: true, - }); - }); - - it("preserves raw exec cwd context before trailing raw command metadata", () => { - const cwdPayloads = buildPayloads({ - lastToolError: { - toolName: "exec", - meta: "run python3 audit.py (in /tmp/build) · `python3 audit.py`", - error: "Command exited with code 1", - mutatingAction: true, - }, - toolResultFormat: "markdown", - verboseLevel: "full", - }); - const workspaceNodePayloads = buildPayloads({ - lastToolError: { - toolName: "exec", - meta: "run python3 audit.py (workspace), node: mac-1, `python3 audit.py`", - error: "Command exited with code 1", - mutatingAction: true, - }, - toolResultFormat: "markdown", - verboseLevel: "full", - }); - const semanticCompactPayloads = buildPayloads({ - lastToolError: { - toolName: "exec", - meta: "check git status (repo), `git status`", - error: "Command exited with code 1", - mutatingAction: true, - }, - toolResultFormat: "markdown", - verboseLevel: "full", - }); - - expectSinglePayloadSummary(cwdPayloads, { - text: "⚠️ 🛠️ Exec failed: `python3 audit.py (in /tmp/build)`: Command exited with code 1", - isError: true, - }); - expectSinglePayloadSummary(workspaceNodePayloads, { - text: "⚠️ 🛠️ Exec failed: `node: mac-1 · python3 audit.py (workspace)`: Command exited with code 1", - isError: true, - }); - expectSinglePayloadSummary(semanticCompactPayloads, { - text: "⚠️ 🛠️ Exec failed: `git status (repo)`: Command exited with code 1", - isError: true, - }); - }); - - it.each([ - { - name: "strips a literal synthetic run prefix", - meta: "run make build", - error: "Command failed with exit code 2", - expected: "⚠️ 🛠️ Exec failed: `make build`: Command failed with exit code 2", - }, - { - name: "preserves a semantic test summary", - meta: "run tests", - error: "Command failed with exit code 1", - expected: "⚠️ 🛠️ Exec failed: `run tests`: Command failed with exit code 1", - }, - { - name: "preserves a semantic deploy summary", - meta: "run deploy", - error: "Command failed with exit code 1", - expected: "⚠️ 🛠️ Exec failed: `run deploy`: Command failed with exit code 1", - }, - { - name: "preserves a compound summary", - meta: "run tests → install dependencies", - error: "Command failed with exit code 1", - expected: - "⚠️ 🛠️ Exec failed: `run tests → install dependencies`: Command failed with exit code 1", - }, - { - name: "preserves an inline-script summary", - meta: "run node inline script", - error: "Command failed with exit code 1", - expected: "⚠️ 🛠️ Exec failed: `run node inline script`: Command failed with exit code 1", - }, - { - name: "preserves a heredoc summary", - meta: "run python3 inline script (heredoc)", - error: "Command failed with exit code 1", - expected: - "⚠️ 🛠️ Exec failed: `run python3 inline script (heredoc)`: Command failed with exit code 1", - }, - { - name: "preserves a sed summary", - meta: "run sed on file", - error: "Command failed with exit code 1", - expected: "⚠️ 🛠️ Exec failed: `run sed on file`: Command failed with exit code 1", - }, - { - name: "preserves a pipeline summary", - meta: "run tests -> show first 3 lines", - error: "Command failed with exit code 1", - expected: - "⚠️ 🛠️ Exec failed: `run tests -> show first 3 lines`: Command failed with exit code 1", - }, - ])("formats exec metadata: $name", ({ meta, error, expected }) => { - const payloads = buildPayloads({ - lastToolError: { - toolName: "exec", - meta, - error, - mutatingAction: true, - }, - toolResultFormat: "markdown", - verboseLevel: "full", - }); - - expectSinglePayloadSummary(payloads, { text: expected, isError: true }); - }); - - it("wraps markdown-capable mutating tool warnings so mention-looking names stay inert", () => { - // Non-recoverable error so the generic exec-like rule still surfaces a warning - // for this no-reply formatting case (recoverable keywords would suppress it). - const payloads = buildPayloads({ - lastToolError: { - toolName: "bash", - meta: "show matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt (workspace)", - error: "Command exited with code 1", - mutatingAction: true, - }, - toolResultFormat: "markdown", - verboseLevel: "full", - }); - - expectSinglePayloadSummary(payloads, { - text: "⚠️ 🛠️ Bash failed: `show matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt` (workspace): Command exited with code 1", - isError: true, - }); - }); - - it("keeps non-recoverable tool errors compact when verbose mode is on", () => { - const payloads = buildPayloads({ - lastToolError: { toolName: "browser", error: "connection timeout" }, - verboseLevel: "on", - }); - - expectSingleToolErrorPayload(payloads, { - title: "Browser", - absentDetail: "connection timeout", - }); - }); - - it("includes non-recoverable tool error details when verbose mode is full", () => { - const payloads = buildPayloads({ - lastToolError: { toolName: "browser", error: "connection timeout" }, - verboseLevel: "full", - }); - - expectSingleToolErrorPayload(payloads, { - title: "Browser", - detail: "connection timeout", - }); - }); }); -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/agents/embedded-agent-runner/run/payloads.ts b/src/agents/embedded-agent-runner/run/payloads.ts index 9fab026c5291..a38abc5f42e1 100644 --- a/src/agents/embedded-agent-runner/run/payloads.ts +++ b/src/agents/embedded-agent-runner/run/payloads.ts @@ -1,10 +1,7 @@ /** * Builds embedded-agent payload objects from attempt inputs and outcomes. */ -import { - normalizeOptionalLowercaseString, - normalizeOptionalString, -} from "@openclaw/normalization-core/string-coerce"; +import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; import type { SourceReplyDeliveryMode } from "../../../auto-reply/get-reply-options.types.js"; import { createHeartbeatToolResponsePayload, @@ -24,7 +21,6 @@ import { isSilentReplyPayloadText, SILENT_REPLY_TOKEN, } from "../../../auto-reply/tokens.js"; -import { formatToolAggregate } from "../../../auto-reply/tool-meta.js"; import type { OpenClawConfig } from "../../../config/types.openclaw.js"; import { hasReplyPayloadContent } from "../../../interactive/payload.js"; import type { AssistantMessage } from "../../../llm/types.js"; @@ -32,7 +28,6 @@ import { extractAssistantTextForPhase, parseAssistantTextSignature, } from "../../../shared/chat-message-content.js"; -import { formatInlineCodeSpan } from "../../../shared/markdown-code.js"; import { sanitizeAssistantFinalAnswerText, sanitizeAssistantVisibleText, @@ -57,39 +52,14 @@ import { sanitizeAssistantVisibleStreamText, } from "../../embedded-agent-utils.js"; import type { PreparedProviderFailoverOwner } from "../../failover/provider-patterns.js"; -import { isExecLikeToolName, type ToolErrorSummary } from "../../tool-error-summary.js"; -import { isLikelyMutatingToolName } from "../../tool-mutation.js"; +import type { ToolErrorSummary } from "../../tool-error-summary.js"; import { buildSourceReplyPayloadState } from "./source-reply-payloads.js"; +import { buildFailureWarning } from "./tool-error-warning.js"; import { hasExplicitMutatingToolFailureAcknowledgement } from "./tool-failure-acknowledgement.js"; -type ToolErrorWarningPolicy = { - showWarning: boolean; - includeDetails: boolean; -}; - -const RECOVERABLE_TOOL_ERROR_KEYWORDS = [ - "required", - "missing", - "invalid", - "must be", - "must have", - "needs", - "requires", -] as const; - -function isRecoverableToolError(error: string | undefined): boolean { - const errorLower = normalizeOptionalLowercaseString(error) ?? ""; - return RECOVERABLE_TOOL_ERROR_KEYWORDS.some((keyword) => errorLower.includes(keyword)); -} - -function isVerboseToolDetailEnabled(level?: VerboseLevel): boolean { - return level === "full"; -} - function isAssistantTextContentBlockType(value: unknown): boolean { return value === "text" || value === "input_text" || value === "output_text"; } - function resolveRawAssistantAnswerText(lastAssistant: AssistantMessage | undefined): string { if (!lastAssistant) { return ""; @@ -150,341 +120,6 @@ function normalizeReplyTextForComparison(text: string): string { return normalizeTextForComparison(parseReplyDirectives(text).text ?? ""); } -function shouldMarkNonTerminalToolErrorWarning(lastToolError: ToolErrorSummary): boolean { - return lastToolError.middlewareError === true; -} - -function formatToolErrorWarningText(params: { - lastToolError: ToolErrorSummary; - includeDetails: boolean; - useMarkdown: boolean; -}): string { - const terminalDiagnostic = params.lastToolError.terminalDiagnostic; - if (terminalDiagnostic?.kind === "process") { - const toolLabel = formatToolAggregate( - "process", - params.includeDetails ? [terminalDiagnostic.sessionId] : undefined, - { markdown: params.useMarkdown }, - ); - const reason = - terminalDiagnostic.reason.kind === "exit" - ? `exit ${terminalDiagnostic.reason.exitCode}` - : terminalDiagnostic.reason.kind === "signal" - ? `signal ${terminalDiagnostic.reason.signal}` - : terminalDiagnostic.reason.timeoutKind === "no-output-timeout" - ? "timed out waiting for output" - : "timed out"; - const errorSuffix = - params.includeDetails && params.lastToolError.error ? `: ${params.lastToolError.error}` : ""; - const recoveryHint = params.includeDetails ? "" : ". Use /verbose full for complete output"; - return `⚠️ ${toolLabel} failed (${reason})${errorSuffix}${recoveryHint}.`; - } - - if (isExecLikeToolName(params.lastToolError.toolName)) { - const toolLabel = formatToolAggregate(params.lastToolError.toolName, undefined, { - markdown: params.useMarkdown, - }); - const subject = params.includeDetails - ? formatExecLikeFailureSubject(params.lastToolError.meta, params.useMarkdown) - : ""; - const conciseExitSuffix = params.includeDetails - ? "" - : formatConciseExecExitSuffix(params.lastToolError.error); - const errorSuffix = - params.includeDetails && params.lastToolError.error ? `: ${params.lastToolError.error}` : ""; - return subject - ? `⚠️ ${toolLabel} failed: ${subject}${conciseExitSuffix}${errorSuffix}` - : `⚠️ ${toolLabel} failed${conciseExitSuffix}${errorSuffix}`; - } - - const toolSummary = formatToolAggregate( - params.lastToolError.toolName, - params.includeDetails && params.lastToolError.meta ? [params.lastToolError.meta] : undefined, - { markdown: params.useMarkdown }, - ); - const errorSuffix = - params.includeDetails && params.lastToolError.error ? `: ${params.lastToolError.error}` : ""; - return `⚠️ ${toolSummary} failed${errorSuffix}`; -} - -function formatExecLikeFailureSubject(meta: string | undefined, markdown: boolean): string { - const normalized = normalizeOptionalString(meta); - if (!normalized) { - return ""; - } - - const { flags, body } = splitExecLikeFailureMeta(normalized); - if (!body) { - return flags.join(" · "); - } - - const { text, suffix } = splitDisplayContextSuffix(body); - const literalCommand = extractLiteralExecCommand(text); - const subject = `${maybeWrapInlineCode(literalCommand ?? text, markdown)}${suffix}`; - return flags.length > 0 ? `${flags.join(" · ")} · ${subject}` : subject; -} - -function splitExecLikeFailureMeta(meta: string): { flags: string[]; body: string } { - const flags: string[] = []; - const bodyParts: string[] = []; - for (const part of meta - .split(" · ") - .map((candidate) => candidate.trim()) - .filter(Boolean)) { - if (part === "elevated" || part === "pty") { - flags.push(part); - continue; - } - bodyParts.push(part); - } - return { flags, body: bodyParts.join(" · ") }; -} - -const SEMANTIC_RUN_SUMMARIES = new Set(["tests", "build", "lint", "script", "command"]); -const LITERAL_RUN_SUMMARY_PREFIXES = new Set([ - "python", - "python3", - "ruby", - "php", - "git", - "npm", - "pnpm", - "yarn", - "bun", - "openclaw", - "make", - "cargo", - "go", - "docker", - "npx", - "uv", - "poetry", - "pytest", - "vitest", - "jest", - "deno", -]); - -function extractLiteralExecCommand(body: string): string | undefined { - const rawCommand = extractRawExecCommand(body); - if (rawCommand) { - return rawCommand; - } - - const nodeScript = body.match(/^run node script (.+)$/u); - if (nodeScript?.[1]) { - return `node ${nodeScript[1]}`; - } - - const runSubject = body.match(/^run (.+)$/u)?.[1]; - if (runSubject && isKnownLiteralRunSummary(runSubject)) { - return runSubject; - } - - return undefined; -} - -type RawExecContext = { - leading: string[]; - trailing: string[]; -}; - -function extractRawExecCommand(body: string): string | undefined { - const codeSpan = extractTrailingMarkdownCodeSpan(body); - if (!codeSpan) { - return undefined; - } - const context = extractRawExecContext(codeSpan.prefix, codeSpan.value); - const command = context.trailing.reduce((value, suffix) => `${value} ${suffix}`, codeSpan.value); - return context.leading.length > 0 ? `${context.leading.join(" · ")} · ${command}` : command; -} - -function extractTrailingMarkdownCodeSpan( - body: string, -): { prefix: string | undefined; value: string } | undefined { - const trimmed = body.trimEnd(); - if (!trimmed.endsWith("`")) { - return undefined; - } - let delimiterLength = 0; - for (let index = trimmed.length - 1; index >= 0 && trimmed[index] === "`"; index -= 1) { - delimiterLength += 1; - } - const delimiter = "`".repeat(delimiterLength); - const valueEnd = trimmed.length - delimiterLength; - let searchIndex = 0; - while (searchIndex < valueEnd) { - const openIndex = trimmed.indexOf(delimiter, searchIndex); - if (openIndex < 0 || openIndex >= valueEnd) { - return undefined; - } - const prefixMatch = trimmed.slice(0, openIndex).match(/^(?:(.*)(?:,\s*| · ))?$/u); - if (prefixMatch) { - return { - prefix: prefixMatch[1], - value: unwrapMarkdownInlineCodePadding( - trimmed.slice(openIndex + delimiterLength, valueEnd), - ), - }; - } - searchIndex = openIndex + delimiterLength; - } - return undefined; -} - -function unwrapMarkdownInlineCodePadding(value: string): string { - if (value.length < 2 || !value.startsWith(" ") || !value.endsWith(" ")) { - return value; - } - const unwrapped = value.slice(1, -1); - return /\S/u.test(unwrapped) ? unwrapped : value; -} -function extractRawExecContext(prefix: string | undefined, inlineCode: string): RawExecContext { - const value = prefix ?? ""; - const leading = [...value.matchAll(/(?:^|,\s*| · )(node:\s*[^,·]+)(?=,\s*| · |$)/gu)] - .map((match) => match[1]?.trim()) - .filter((part): part is string => Boolean(part)); - const trailing = [ - ...value.matchAll( - /(\((?:agent|repo|sandbox|workspace)\)|\(in [^)\r\n]+\))(?=\s*(?:,\s*| · |$))/gu, - ), - ] - .filter((match) => shouldKeepRawExecTrailingContext(value, match, inlineCode)) - .map((match) => match[1]?.trim()) - .filter((part): part is string => Boolean(part)); - return { leading, trailing }; -} -function shouldKeepRawExecTrailingContext( - prefix: string, - match: RegExpMatchArray, - inlineCode: string, -): boolean { - const suffix = match[1]?.trim(); - if (!suffix || inlineCode.includes(suffix)) { - return false; - } - const segment = prefix - .slice(0, match.index ?? 0) - .trimEnd() - .split(/,\s*| · /u) - .at(-1) - ?.trim(); - const segmentCommand = segment ? extractLiteralExecCommand(segment) : undefined; - if (segmentCommand === inlineCode || segment === inlineCode) { - return true; - } - if (isCompactCwdSuffix(suffix)) { - return true; - } - return isPathLikeCwdSuffix(suffix); -} -function isCompactCwdSuffix(suffix: string): boolean { - return /^\((?:agent|repo|workspace)\)$/u.test(suffix); -} -function isPathLikeCwdSuffix(suffix: string): boolean { - const cwd = suffix.match(/^\(in ([^)\r\n]+)\)$/u)?.[1]?.trim(); - return Boolean( - cwd && (/^(?:\/|~|\.{1,2}(?:\/|$)|[A-Za-z]:[\\/]|\\\\)/u.test(cwd) || cwd.includes("/")), - ); -} -function isKnownLiteralRunSummary(subject: string): boolean { - if ( - SEMANTIC_RUN_SUMMARIES.has(subject) || - subject.includes("→") || - subject.includes("->") || - /^(?:node|python3?|ruby|php) inline script(?: \(heredoc\))?$/u.test(subject) - ) { - return false; - } - const match = subject.match(/^(\S+)\s+(.+)$/u); - const command = match?.[1]; - const remainder = match?.[2]; - if (!command || !remainder || remainder === "command") { - return false; - } - return LITERAL_RUN_SUMMARY_PREFIXES.has(command); -} -function splitDisplayContextSuffix(value: string): { text: string; suffix: string } { - const match = /^(.*?)( \((?:agent|repo|workspace|sandbox)\))$/u.exec(value); - if (!match) { - return { text: value, suffix: "" }; - } - return { text: match[1] ?? value, suffix: match[2] ?? "" }; -} -function formatConciseExecExitSuffix(error: string | undefined): string { - const normalized = normalizeOptionalString(error); - const code = normalized?.match( - /\b(?:command\s+)?(?:failed\s+with\s+exit\s+code|exited\s+with\s+code|exit(?:ed)?\s+code|exit\s+status)\s+(-?\d+)\b/iu, - )?.[1]; - return code ? ` (exit ${code})` : ""; -} -function maybeWrapInlineCode(value: string, markdown: boolean): string { - return markdown ? formatInlineCodeSpan(value) : value; -} -/** - * Chooses whether a tool failure needs a separate user-visible warning and - * whether to include raw details. Mutating failures are stricter because a - * silent failed write/send/delete can make the assistant look successful. - */ -function resolveToolErrorWarningPolicy(params: { - lastToolError: ToolErrorSummary; - hasUserFacingReply: boolean; - hasUserFacingErrorReply: boolean; - hasUserFacingFailureAcknowledgement: boolean; - suppressToolErrors: boolean; - suppressToolErrorWarnings?: boolean | (() => boolean | undefined); - verboseLevel?: VerboseLevel; -}): ToolErrorWarningPolicy { - const normalizedToolName = normalizeOptionalLowercaseString(params.lastToolError.toolName) ?? ""; - let toolErrorWarningOverride: boolean | undefined; - let dynamicToolErrorWarningsDisabled = false; - if (typeof params.suppressToolErrorWarnings === "function") { - toolErrorWarningOverride = params.suppressToolErrorWarnings(); - dynamicToolErrorWarningsDisabled = toolErrorWarningOverride === false; - } else { - toolErrorWarningOverride = params.suppressToolErrorWarnings; - } - const includeDetails = - !dynamicToolErrorWarningsDisabled && isVerboseToolDetailEnabled(params.verboseLevel); - const suppressToolErrorWarnings = toolErrorWarningOverride === true; - if (suppressToolErrorWarnings) { - return { showWarning: false, includeDetails }; - } - // sessions_send timeouts and errors are transient inter-session communication - // issues — the message may still have been delivered. Suppress warnings to - // prevent raw error text from leaking into the chat surface (#23989). - if (normalizedToolName === "sessions_send") { - return { showWarning: false, includeDetails }; - } - if (params.suppressToolErrors) { - return { showWarning: false, includeDetails }; - } - // Mutating branch protects "assistant claims success while a user-visible mutation - // silently failed". Shell/exec are the agent's own workspace actions: the model sees - // the exit code in-context, and a successful final reply is recovery proof (#103574). - // Deliberately ignores mutatingAction for exec: codex marks every commandExecution - // mutating fail-closed (replay metadata, not display signal). - if (isExecLikeToolName(params.lastToolError.toolName)) { - // No recoverable-keyword suppression here: with no reply at all, the exec - // warning may be the run's only failure signal. - return { showWarning: !params.hasUserFacingReply, includeDetails }; - } - if (params.lastToolError.terminalDiagnostic?.kind === "process") { - return { showWarning: !params.hasUserFacingReply, includeDetails }; - } - const isMutatingToolError = - params.lastToolError.mutatingAction ?? isLikelyMutatingToolName(params.lastToolError.toolName); - if (isMutatingToolError) { - return { - showWarning: !params.hasUserFacingErrorReply && !params.hasUserFacingFailureAcknowledgement, - includeDetails, - }; - } - return { - showWarning: !params.hasUserFacingReply && !isRecoverableToolError(params.lastToolError.error), - includeDetails, - }; -} /** * Converts a completed embedded attempt into reply payloads for channels. This * is the boundary that suppresses duplicate source replies, filters raw API @@ -758,7 +393,9 @@ export function buildEmbeddedRunPayloads(params: { } } if (params.lastToolError) { - const warningPolicy = resolveToolErrorWarningPolicy({ + // Surface mutating failures unless the assistant explicitly acknowledged the failed action. + // Otherwise, keep the previous behavior and only surface non-recoverable failures when no reply exists. + const failureWarning = buildFailureWarning({ lastToolError: params.lastToolError, hasUserFacingReply: hasUserFacingAssistantReply, hasUserFacingErrorReply, @@ -766,16 +403,10 @@ export function buildEmbeddedRunPayloads(params: { suppressToolErrors: Boolean(params.config?.messages?.suppressToolErrors), suppressToolErrorWarnings: params.suppressToolErrorWarnings, verboseLevel: params.verboseLevel, + useMarkdown, }); - // Surface mutating failures unless the assistant explicitly acknowledged the failed action. - // Otherwise, keep the previous behavior and only surface non-recoverable failures when no reply exists. - if (warningPolicy.showWarning) { - const warningText = formatToolErrorWarningText({ - lastToolError: params.lastToolError, - includeDetails: warningPolicy.includeDetails, - useMarkdown, - }); - const normalizedWarning = normalizeTextForComparison(warningText); + if (failureWarning) { + const normalizedWarning = normalizeTextForComparison(failureWarning.text); const duplicateWarning = normalizedWarning ? replyItems.some((item) => { if (!item.text) { @@ -787,11 +418,10 @@ export function buildEmbeddedRunPayloads(params: { : false; if (!duplicateWarning) { replyItems.push({ - text: warningText, + text: failureWarning.text, isError: true, nonTerminalToolErrorWarning: - hasUserFacingAssistantReply && - shouldMarkNonTerminalToolErrorWarning(params.lastToolError), + hasUserFacingAssistantReply && failureWarning.nonTerminalToolErrorWarning, }); } } @@ -920,4 +550,3 @@ export function buildEmbeddedRunPayloads(params: { return true; }); } -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/agents/embedded-agent-runner/run/tool-error-warning.test.ts b/src/agents/embedded-agent-runner/run/tool-error-warning.test.ts new file mode 100644 index 000000000000..1bfe7b061b1a --- /dev/null +++ b/src/agents/embedded-agent-runner/run/tool-error-warning.test.ts @@ -0,0 +1,730 @@ +// Tool warning tests ensure failed actions remain visible without exposing +// verbose execution details unless the operator explicitly requests them. +import type { AssistantMessage } from "openclaw/plugin-sdk/llm"; +import { describe, expect, it } from "vitest"; +import { getReplyPayloadMetadata } from "../../../auto-reply/reply-payload.js"; +import { makeAssistantMessageFixture } from "../../test-helpers/assistant-message-fixtures.js"; +import { + buildPayloads, + expectSinglePayloadText, + expectSingleToolErrorPayload, +} from "./payloads.test-helpers.js"; + +describe("buildEmbeddedRunPayloads tool warnings", () => { + const errorJson = + '{"type":"error","error":{"details":null,"type":"overloaded_error","message":"Overloaded"},"request_id":"req_011CX7DwS7tSvggaNHmefwWg"}'; + const makeAssistant = (overrides: Partial): AssistantMessage => + // Default to an overloaded provider error so each test can override only + // the assistant fields relevant to user-visible payload sanitization. + makeAssistantMessageFixture({ + errorMessage: errorJson, + content: [{ type: "text", text: errorJson }], + ...overrides, + }); + const makeStoppedAssistant = () => + makeAssistant({ + stopReason: "stop", + errorMessage: undefined, + content: [], + }); + + function expectSinglePayloadSummary( + payloads: ReturnType, + expected: { text: string; isError?: boolean }, + ) { + expectSinglePayloadText(payloads, expected.text); + if (expected.isError === undefined) { + expect(payloads[0]?.isError).toBeUndefined(); + return; + } + expect(payloads[0]?.isError).toBe(expected.isError); + } + + function expectNoPayloads(params: Parameters[0]) { + const payloads = buildPayloads(params); + expect(payloads).toHaveLength(0); + } + + function expectNoSyntheticCompletionForSession(sessionKey: string) { + expectNoPayloads({ + sessionKey, + lastAssistant: makeAssistant({ + stopReason: "stop", + errorMessage: undefined, + content: [], + }), + }); + } + + it("adds a fallback error when a tool fails and no assistant output exists", () => { + const payloads = buildPayloads({ + lastToolError: { toolName: "browser", error: "tab not found" }, + }); + + expectSingleToolErrorPayload(payloads, { + title: "Browser", + absentDetail: "tab not found", + }); + }); + + it("does not add tool error fallback when assistant output exists", () => { + const payloads = buildPayloads({ + assistantTexts: ["All good"], + lastAssistant: makeStoppedAssistant(), + lastToolError: { toolName: "browser", error: "tab not found" }, + }); + + expectSinglePayloadText(payloads, "All good"); + }); + + it("does not add synthetic completion text for channel sessions", () => { + expectNoSyntheticCompletionForSession("agent:main:discord:channel:c123"); + }); + + it("does not add synthetic completion text for group sessions", () => { + expectNoSyntheticCompletionForSession("agent:main:telegram:group:g123"); + }); + + it("does not add synthetic completion text when messaging tool already delivered output", () => { + expectNoPayloads({ + sessionKey: "agent:main:discord:direct:u123", + didSendViaMessagingTool: true, + lastAssistant: makeAssistant({ + stopReason: "stop", + errorMessage: undefined, + content: [], + }), + }); + }); + + it("does not add synthetic completion text when the run still has a tool error", () => { + expectNoPayloads({ + lastToolError: { toolName: "browser", error: "url required" }, + }); + }); + + it("does not add synthetic completion text when no tools ran", () => { + expectNoPayloads({ + lastAssistant: makeStoppedAssistant(), + }); + }); + + it("adds compact tool error fallback when the assistant only invoked tools and verbose mode is on", () => { + const payloads = buildPayloads({ + lastAssistant: makeAssistant({ + stopReason: "toolUse", + errorMessage: undefined, + content: [ + { + type: "toolCall", + id: "toolu_01", + name: "exec", + arguments: { command: "echo hi" }, + }, + ], + }), + lastToolError: { toolName: "exec", error: "Command exited with code 1" }, + verboseLevel: "on", + }); + + expectSingleToolErrorPayload(payloads, { + title: "Exec", + absentDetail: "code 1", + }); + }); + + it("does not add tool error fallback when assistant text exists after tool calls", () => { + const payloads = buildPayloads({ + assistantTexts: ["Checked the page and recovered with final answer."], + lastAssistant: makeAssistant({ + stopReason: "toolUse", + errorMessage: undefined, + content: [ + { + type: "toolCall", + id: "toolu_01", + name: "browser", + arguments: { action: "search", query: "openclaw docs" }, + }, + ], + }), + lastToolError: { toolName: "browser", error: "connection timeout" }, + }); + + expectSinglePayloadSummary(payloads, { + text: "Checked the page and recovered with final answer.", + }); + }); + + it.each(["url required", "url missing", "invalid parameter: url"])( + "suppresses recoverable non-mutating tool error: %s", + (error) => { + expectNoPayloads({ + lastToolError: { toolName: "browser", error }, + }); + }, + ); + + it("suppresses non-mutating non-recoverable tool errors when messages.suppressToolErrors is enabled", () => { + expectNoPayloads({ + lastToolError: { toolName: "browser", error: "connection timeout" }, + config: { messages: { suppressToolErrors: true } }, + }); + }); + + it("suppresses mutating tool errors when suppressToolErrorWarnings is enabled", () => { + expectNoPayloads({ + lastToolError: { toolName: "exec", error: "command not found" }, + suppressToolErrorWarnings: true, + }); + }); + + it.each([ + { + name: "suppresses mutating tool errors when messages.suppressToolErrors is enabled", + payload: { + lastToolError: { toolName: "write", error: "connection timeout" }, + config: { messages: { suppressToolErrors: true } }, + }, + title: "Write", + absentDetail: "connection timeout", + suppressed: true, + }, + { + name: "shows recoverable tool errors for mutating tools", + payload: { + lastToolError: { toolName: "message", meta: "reply", error: "text required" }, + }, + title: "Message", + absentDetail: "required", + }, + { + name: "shows non-recoverable tool failure summaries to the user", + payload: { + lastToolError: { toolName: "browser", error: "connection timeout" }, + }, + title: "Browser", + absentDetail: "connection timeout", + }, + ])("$name", ({ payload, title, absentDetail, suppressed }) => { + const payloads = buildPayloads(payload); + if (suppressed) { + expect(payloads).toEqual([]); + return; + } + expectSingleToolErrorPayload(payloads, { title, absentDetail }); + }); + + it("shows mutating tool errors when assistant output claims success", () => { + const payloads = buildPayloads({ + assistantTexts: ["Done."], + lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, + lastToolError: { toolName: "write", error: "file missing" }, + }); + + expect(payloads).toHaveLength(2); + expect(payloads[0]?.text).toBe("Done."); + expect(payloads[1]?.isError).toBe(true); + expect(payloads[1]?.text).toContain("Write"); + expect(payloads[1]?.text).not.toContain("missing"); + expect(getReplyPayloadMetadata(payloads[1] as object)?.nonTerminalToolErrorWarning).toBe( + undefined, + ); + }); + + it("still shows write tool errors when timedOut is true but no fileTarget was recorded", () => { + // Without `fileTarget` we cannot distinguish a confirmed file write from + // an unrelated mutating-tool timeout, so the default-visible warning is + // preserved to avoid hiding real failures. + const payloads = buildPayloads({ + assistantTexts: ["Done."], + lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, + lastToolError: { + toolName: "write", + error: "invoke timed out", + timedOut: true, + mutatingAction: true, + }, + }); + + expect(payloads).toHaveLength(2); + expect(payloads[1]?.isError).toBe(true); + expect(payloads[1]?.text).toContain("Write"); + }); + + it("still shows write tool errors when timedOut and fileTarget only prove the attempted path", () => { + const payloads = buildPayloads({ + assistantTexts: ["Done."], + lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, + lastToolError: { + toolName: "write", + error: "invoke timed out", + timedOut: true, + mutatingAction: true, + fileTarget: { path: "/tmp/openclaw/output.md" }, + }, + }); + + expect(payloads).toHaveLength(2); + expect(payloads[1]?.isError).toBe(true); + expect(payloads[1]?.text).toContain("Write"); + }); + + it("does not warn for timed-out exec errors when a successful user-facing reply exists", () => { + // Exec/bash use the generic recovery rule, not the mutating-tool branch: + // a successful final reply is proof the agent recovered (#103574). + const payloads = buildPayloads({ + assistantTexts: ["The script is ready."], + lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, + lastToolError: { + toolName: "exec", + error: "command timed out", + timedOut: true, + mutatingAction: true, + }, + }); + + expectSinglePayloadSummary(payloads, { text: "The script is ready." }); + }); + + it("does not warn for exec-like tool errors when a successful user-facing reply exists", () => { + // Production repro: mid-run bash/exec failure recovered with a correct final answer. + const payloads = buildPayloads({ + assistantTexts: ["The script is ready to use and saved in your workspace."], + lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, + lastToolError: { + toolName: "exec", + error: "/bin/bash: line 1: python: command not found", + mutatingAction: true, + }, + }); + + expectSinglePayloadSummary(payloads, { + text: "The script is ready to use and saved in your workspace.", + }); + }); + + it("does not warn for bash tool errors when a successful user-facing reply exists", () => { + const payloads = buildPayloads({ + assistantTexts: ["Recovered after the command failed."], + lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, + lastToolError: { + toolName: "bash", + error: "exit code 1", + mutatingAction: true, + }, + }); + + expectSinglePayloadSummary(payloads, { text: "Recovered after the command failed." }); + }); + + it("keeps exec-like tool error warnings when there is no user-facing reply", () => { + const payloads = buildPayloads({ + lastToolError: { + toolName: "exec", + error: "/bin/bash: line 1: python: command not found", + mutatingAction: true, + }, + }); + + expectSingleToolErrorPayload(payloads, { + title: "Exec", + absentDetail: "python: command not found", + }); + }); + + it("keeps exec-like tool error warnings for recoverable-looking errors when there is no reply", () => { + const payloads = buildPayloads({ + lastToolError: { + toolName: "bash", + error: "invalid argument: missing required flag --agent", + mutatingAction: true, + }, + }); + + expectSingleToolErrorPayload(payloads, { + title: "Bash", + absentDetail: "missing required flag", + }); + }); + + it("suppresses exec-like tool errors when messages.suppressToolErrors is enabled", () => { + expectNoPayloads({ + lastToolError: { + toolName: "bash", + error: "command not found", + mutatingAction: true, + }, + config: { messages: { suppressToolErrors: true } }, + }); + }); + + it("shows mutating tool errors when assistant output does not acknowledge the failure", () => { + const payloads = buildPayloads({ + assistantTexts: ["No issues found. The update is complete."], + lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, + lastToolError: { toolName: "edit", error: "file missing" }, + }); + + expect(payloads).toHaveLength(2); + expect(payloads[0]?.text).toBe("No issues found. The update is complete."); + expect(payloads[1]?.isError).toBe(true); + expect(payloads[1]?.text).toContain("Edit"); + expect(payloads[1]?.text).not.toContain("missing"); + }); + + it("shows mutating tool errors when assistant says it did not find issues in the file", () => { + const text = "I did not find any issues in the file. The update is complete."; + const payloads = buildPayloads({ + assistantTexts: [text], + lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, + lastToolError: { toolName: "edit", error: "file missing" }, + }); + + expect(payloads).toHaveLength(2); + expect(payloads[0]?.text).toBe(text); + expect(payloads[1]?.isError).toBe(true); + expect(payloads[1]?.text).toContain("Edit"); + expect(payloads[1]?.text).not.toContain("missing"); + }); + + it.each([ + "I did not need to update the file; it is already correct.", + "I did not have to edit the file because it was already correct.", + ])("shows mutating tool errors when assistant output uses no-op phrasing: %s", (text) => { + const payloads = buildPayloads({ + assistantTexts: [text], + lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, + lastToolError: { toolName: "edit", error: "file missing" }, + }); + + expect(payloads).toHaveLength(2); + expect(payloads[0]?.text).toBe(text); + expect(payloads[1]?.isError).toBe(true); + expect(payloads[1]?.text).toContain("Edit"); + expect(payloads[1]?.text).not.toContain("missing"); + }); + + it("suppresses mutating tool errors when assistant output explicitly acknowledges the failed action", () => { + const text = "I couldn't update the file, so no changes were applied."; + const payloads = buildPayloads({ + assistantTexts: [text], + lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, + lastToolError: { toolName: "edit", error: "file missing" }, + }); + + expectSinglePayloadSummary(payloads, { text }); + }); + + it("suppresses exec warnings when assistant output explicitly acknowledges the command failure", () => { + const text = "I couldn't run the command because python was not found."; + const payloads = buildPayloads({ + assistantTexts: [text], + lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, + lastToolError: { toolName: "exec", error: "/bin/bash: line 1: python: command not found" }, + }); + + expectSinglePayloadSummary(payloads, { text }); + }); + + it("does not treat session_status read failures as mutating when explicitly flagged", () => { + const payloads = buildPayloads({ + assistantTexts: ["Status loaded."], + lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, + lastToolError: { + toolName: "session_status", + error: "model required", + mutatingAction: false, + }, + }); + + expectSinglePayloadSummary(payloads, { text: "Status loaded." }); + }); + + it("dedupes identical tool warning text already present in assistant output", () => { + const seed = buildPayloads({ + lastToolError: { + toolName: "write", + error: "file missing", + mutatingAction: true, + }, + }); + const warningText = seed[0]?.text; + expect(warningText).toBe("⚠️ ✍️ Write failed"); + + const payloads = buildPayloads({ + assistantTexts: [warningText ?? ""], + lastAssistant: { stopReason: "end_turn" } as unknown as AssistantMessage, + lastToolError: { + toolName: "write", + error: "file missing", + mutatingAction: true, + }, + }); + + expectSinglePayloadSummary(payloads, { text: warningText ?? "" }); + }); + + it("hides exec command and cwd metadata without full verbosity", () => { + const payloads = buildPayloads({ + lastToolError: { + toolName: "exec", + meta: "run python3 /path/to/daily-cost-audit.py (in /private/workspace)", + error: "Command exited with code 1", + mutatingAction: true, + }, + toolResultFormat: "markdown", + verboseLevel: "off", + }); + + expectSinglePayloadSummary(payloads, { + text: "⚠️ 🛠️ Exec failed (exit 1)", + isError: true, + }); + }); + + it("keeps full-verbose exec failure labels outside markdown command text", () => { + const payloads = buildPayloads({ + lastToolError: { + toolName: "exec", + meta: "run python3 /path/to/daily-cost-audit.py", + error: "Command exited with code 1", + mutatingAction: true, + }, + toolResultFormat: "markdown", + verboseLevel: "full", + }); + + expectSinglePayloadSummary(payloads, { + text: "⚠️ 🛠️ Exec failed: `python3 /path/to/daily-cost-audit.py`: Command exited with code 1", + isError: true, + }); + expect(payloads[0]?.text).not.toContain("`run python3"); + }); + + it.each([ + { + title: "prefers raw exec metadata when tool progress detail includes it", + meta: "run python3 /tmp/audit.py · `python3 /tmp/audit.py`", + toolResultFormat: "markdown", + expected: "⚠️ 🛠️ Exec failed: `python3 /tmp/audit.py`: Command exited with code 1", + }, + { + title: "prefers raw exec metadata when the literal command contains backticks", + meta: "run node inline script, `node -e 'console.log(1, `x`)'`", + toolResultFormat: "markdown", + expected: "⚠️ 🛠️ Exec failed: ``node -e 'console.log(1, `x`)'``: Command exited with code 1", + }, + { + title: "leaves exec metadata unwrapped for plain tool results", + meta: "run node inline script, `node -e 'console.log(1, `x`)'`", + toolResultFormat: "plain", + expected: "⚠️ 🛠️ Exec failed: node -e 'console.log(1, `x`)': Command exited with code 1", + }, + { + title: "preserves raw exec context before trailing raw command metadata", + meta: "run python3 /tmp/audit.py, node: mac-1, `python3 /tmp/audit.py`", + toolResultFormat: "markdown", + expected: + "⚠️ 🛠️ Exec failed: `node: mac-1 · python3 /tmp/audit.py`: Command exited with code 1", + }, + { + title: "does not promote display-summary commas into raw exec context", + meta: 'search "foo,bar" in src, `rg "foo,bar" src`', + toolResultFormat: "markdown", + expected: '⚠️ 🛠️ Exec failed: `rg "foo,bar" src`: Command exited with code 1', + }, + { + title: "does not treat parenthesized raw command arguments as cwd context", + meta: 'list files in (in progress) · `ls "(in progress)"`', + toolResultFormat: "markdown", + expected: '⚠️ 🛠️ Exec failed: `ls "(in progress)"`: Command exited with code 1', + }, + { + title: "does not duplicate compact cwd labels already present in raw command arguments", + meta: 'print text (repo) · `printf "%s" "(repo)"`', + toolResultFormat: "markdown", + expected: '⚠️ 🛠️ Exec failed: `printf "%s" "(repo)"`: Command exited with code 1', + }, + { + title: "keeps arbitrary exec cwd suffixes inside markdown command text", + meta: "run python3 /tmp/audit.py (in /tmp/build @everyone)", + toolResultFormat: "markdown", + expected: + "⚠️ 🛠️ Exec failed: `python3 /tmp/audit.py (in /tmp/build @everyone)`: Command exited with code 1", + }, + ] as const)("$title", ({ meta, toolResultFormat, expected }) => { + const payloads = buildPayloads({ + lastToolError: { + toolName: "exec", + meta, + error: "Command exited with code 1", + mutatingAction: true, + }, + toolResultFormat, + verboseLevel: "full", + }); + + expectSinglePayloadSummary(payloads, { + text: expected, + isError: true, + }); + }); + + it("preserves raw exec cwd context before trailing raw command metadata", () => { + const cwdPayloads = buildPayloads({ + lastToolError: { + toolName: "exec", + meta: "run python3 audit.py (in /tmp/build) · `python3 audit.py`", + error: "Command exited with code 1", + mutatingAction: true, + }, + toolResultFormat: "markdown", + verboseLevel: "full", + }); + const workspaceNodePayloads = buildPayloads({ + lastToolError: { + toolName: "exec", + meta: "run python3 audit.py (workspace), node: mac-1, `python3 audit.py`", + error: "Command exited with code 1", + mutatingAction: true, + }, + toolResultFormat: "markdown", + verboseLevel: "full", + }); + const semanticCompactPayloads = buildPayloads({ + lastToolError: { + toolName: "exec", + meta: "check git status (repo), `git status`", + error: "Command exited with code 1", + mutatingAction: true, + }, + toolResultFormat: "markdown", + verboseLevel: "full", + }); + + expectSinglePayloadSummary(cwdPayloads, { + text: "⚠️ 🛠️ Exec failed: `python3 audit.py (in /tmp/build)`: Command exited with code 1", + isError: true, + }); + expectSinglePayloadSummary(workspaceNodePayloads, { + text: "⚠️ 🛠️ Exec failed: `node: mac-1 · python3 audit.py (workspace)`: Command exited with code 1", + isError: true, + }); + expectSinglePayloadSummary(semanticCompactPayloads, { + text: "⚠️ 🛠️ Exec failed: `git status (repo)`: Command exited with code 1", + isError: true, + }); + }); + + it.each([ + { + name: "strips a literal synthetic run prefix", + meta: "run make build", + error: "Command failed with exit code 2", + expected: "⚠️ 🛠️ Exec failed: `make build`: Command failed with exit code 2", + }, + { + name: "preserves a semantic test summary", + meta: "run tests", + error: "Command failed with exit code 1", + expected: "⚠️ 🛠️ Exec failed: `run tests`: Command failed with exit code 1", + }, + { + name: "preserves a semantic deploy summary", + meta: "run deploy", + error: "Command failed with exit code 1", + expected: "⚠️ 🛠️ Exec failed: `run deploy`: Command failed with exit code 1", + }, + { + name: "preserves a compound summary", + meta: "run tests → install dependencies", + error: "Command failed with exit code 1", + expected: + "⚠️ 🛠️ Exec failed: `run tests → install dependencies`: Command failed with exit code 1", + }, + { + name: "preserves an inline-script summary", + meta: "run node inline script", + error: "Command failed with exit code 1", + expected: "⚠️ 🛠️ Exec failed: `run node inline script`: Command failed with exit code 1", + }, + { + name: "preserves a heredoc summary", + meta: "run python3 inline script (heredoc)", + error: "Command failed with exit code 1", + expected: + "⚠️ 🛠️ Exec failed: `run python3 inline script (heredoc)`: Command failed with exit code 1", + }, + { + name: "preserves a sed summary", + meta: "run sed on file", + error: "Command failed with exit code 1", + expected: "⚠️ 🛠️ Exec failed: `run sed on file`: Command failed with exit code 1", + }, + { + name: "preserves a pipeline summary", + meta: "run tests -> show first 3 lines", + error: "Command failed with exit code 1", + expected: + "⚠️ 🛠️ Exec failed: `run tests -> show first 3 lines`: Command failed with exit code 1", + }, + ])("formats exec metadata: $name", ({ meta, error, expected }) => { + const payloads = buildPayloads({ + lastToolError: { + toolName: "exec", + meta, + error, + mutatingAction: true, + }, + toolResultFormat: "markdown", + verboseLevel: "full", + }); + + expectSinglePayloadSummary(payloads, { text: expected, isError: true }); + }); + + it("wraps markdown-capable mutating tool warnings so mention-looking names stay inert", () => { + // Non-recoverable error so the generic exec-like rule still surfaces a warning + // for this no-reply formatting case (recoverable keywords would suppress it). + const payloads = buildPayloads({ + lastToolError: { + toolName: "bash", + meta: "show matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt (workspace)", + error: "Command exited with code 1", + mutatingAction: true, + }, + toolResultFormat: "markdown", + verboseLevel: "full", + }); + + expectSinglePayloadSummary(payloads, { + text: "⚠️ 🛠️ Bash failed: `show matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt` (workspace): Command exited with code 1", + isError: true, + }); + }); + + it("keeps non-recoverable tool errors compact when verbose mode is on", () => { + const payloads = buildPayloads({ + lastToolError: { toolName: "browser", error: "connection timeout" }, + verboseLevel: "on", + }); + + expectSingleToolErrorPayload(payloads, { + title: "Browser", + absentDetail: "connection timeout", + }); + }); + + it("includes non-recoverable tool error details when verbose mode is full", () => { + const payloads = buildPayloads({ + lastToolError: { toolName: "browser", error: "connection timeout" }, + verboseLevel: "full", + }); + + expectSingleToolErrorPayload(payloads, { + title: "Browser", + detail: "connection timeout", + }); + }); +}); diff --git a/src/agents/embedded-agent-runner/run/tool-error-warning.ts b/src/agents/embedded-agent-runner/run/tool-error-warning.ts new file mode 100644 index 000000000000..ba4b9d54467c --- /dev/null +++ b/src/agents/embedded-agent-runner/run/tool-error-warning.ts @@ -0,0 +1,392 @@ +import { + normalizeOptionalLowercaseString, + normalizeOptionalString, +} from "@openclaw/normalization-core/string-coerce"; +import type { VerboseLevel } from "../../../auto-reply/thinking.js"; +import { formatToolAggregate } from "../../../auto-reply/tool-meta.js"; +import { formatInlineCodeSpan } from "../../../shared/markdown-code.js"; +import { isExecLikeToolName, type ToolErrorSummary } from "../../tool-error-summary.js"; +import { isLikelyMutatingToolName } from "../../tool-mutation.js"; + +type ToolErrorWarningPolicy = { + showWarning: boolean; + includeDetails: boolean; +}; + +const RECOVERABLE_TOOL_ERROR_KEYWORDS = [ + "required", + "missing", + "invalid", + "must be", + "must have", + "needs", + "requires", +] as const; + +function isRecoverableToolError(error: string | undefined): boolean { + const errorLower = normalizeOptionalLowercaseString(error) ?? ""; + return RECOVERABLE_TOOL_ERROR_KEYWORDS.some((keyword) => errorLower.includes(keyword)); +} +function isVerboseToolDetailEnabled(level?: VerboseLevel): boolean { + return level === "full"; +} + +function shouldMarkNonTerminalToolErrorWarning(lastToolError: ToolErrorSummary): boolean { + return lastToolError.middlewareError === true; +} + +function formatToolErrorWarningText(params: { + lastToolError: ToolErrorSummary; + includeDetails: boolean; + useMarkdown: boolean; +}): string { + const terminalDiagnostic = params.lastToolError.terminalDiagnostic; + if (terminalDiagnostic?.kind === "process") { + const toolLabel = formatToolAggregate( + "process", + params.includeDetails ? [terminalDiagnostic.sessionId] : undefined, + { markdown: params.useMarkdown }, + ); + const reason = + terminalDiagnostic.reason.kind === "exit" + ? `exit ${terminalDiagnostic.reason.exitCode}` + : terminalDiagnostic.reason.kind === "signal" + ? `signal ${terminalDiagnostic.reason.signal}` + : terminalDiagnostic.reason.timeoutKind === "no-output-timeout" + ? "timed out waiting for output" + : "timed out"; + const errorSuffix = + params.includeDetails && params.lastToolError.error ? `: ${params.lastToolError.error}` : ""; + const recoveryHint = params.includeDetails ? "" : ". Use /verbose full for complete output"; + return `⚠️ ${toolLabel} failed (${reason})${errorSuffix}${recoveryHint}.`; + } + + if (isExecLikeToolName(params.lastToolError.toolName)) { + const toolLabel = formatToolAggregate(params.lastToolError.toolName, undefined, { + markdown: params.useMarkdown, + }); + const subject = params.includeDetails + ? formatExecLikeFailureSubject(params.lastToolError.meta, params.useMarkdown) + : ""; + const conciseExitSuffix = params.includeDetails + ? "" + : formatConciseExecExitSuffix(params.lastToolError.error); + const errorSuffix = + params.includeDetails && params.lastToolError.error ? `: ${params.lastToolError.error}` : ""; + return subject + ? `⚠️ ${toolLabel} failed: ${subject}${conciseExitSuffix}${errorSuffix}` + : `⚠️ ${toolLabel} failed${conciseExitSuffix}${errorSuffix}`; + } + + const toolSummary = formatToolAggregate( + params.lastToolError.toolName, + params.includeDetails && params.lastToolError.meta ? [params.lastToolError.meta] : undefined, + { markdown: params.useMarkdown }, + ); + const errorSuffix = + params.includeDetails && params.lastToolError.error ? `: ${params.lastToolError.error}` : ""; + return `⚠️ ${toolSummary} failed${errorSuffix}`; +} + +function formatExecLikeFailureSubject(meta: string | undefined, markdown: boolean): string { + const normalized = normalizeOptionalString(meta); + if (!normalized) { + return ""; + } + + const { flags, body } = splitExecLikeFailureMeta(normalized); + if (!body) { + return flags.join(" · "); + } + + const { text, suffix } = splitDisplayContextSuffix(body); + const literalCommand = extractLiteralExecCommand(text); + const subject = `${maybeWrapInlineCode(literalCommand ?? text, markdown)}${suffix}`; + return flags.length > 0 ? `${flags.join(" · ")} · ${subject}` : subject; +} + +function splitExecLikeFailureMeta(meta: string): { flags: string[]; body: string } { + const flags: string[] = []; + const bodyParts: string[] = []; + for (const part of meta + .split(" · ") + .map((candidate) => candidate.trim()) + .filter(Boolean)) { + if (part === "elevated" || part === "pty") { + flags.push(part); + continue; + } + bodyParts.push(part); + } + return { flags, body: bodyParts.join(" · ") }; +} + +const SEMANTIC_RUN_SUMMARIES = new Set(["tests", "build", "lint", "script", "command"]); +const LITERAL_RUN_SUMMARY_PREFIXES = new Set([ + "python", + "python3", + "ruby", + "php", + "git", + "npm", + "pnpm", + "yarn", + "bun", + "openclaw", + "make", + "cargo", + "go", + "docker", + "npx", + "uv", + "poetry", + "pytest", + "vitest", + "jest", + "deno", +]); + +function extractLiteralExecCommand(body: string): string | undefined { + const rawCommand = extractRawExecCommand(body); + if (rawCommand) { + return rawCommand; + } + + const nodeScript = body.match(/^run node script (.+)$/u); + if (nodeScript?.[1]) { + return `node ${nodeScript[1]}`; + } + + const runSubject = body.match(/^run (.+)$/u)?.[1]; + if (runSubject && isKnownLiteralRunSummary(runSubject)) { + return runSubject; + } + + return undefined; +} + +type RawExecContext = { + leading: string[]; + trailing: string[]; +}; + +function extractRawExecCommand(body: string): string | undefined { + const codeSpan = extractTrailingMarkdownCodeSpan(body); + if (!codeSpan) { + return undefined; + } + const context = extractRawExecContext(codeSpan.prefix, codeSpan.value); + const command = context.trailing.reduce((value, suffix) => `${value} ${suffix}`, codeSpan.value); + return context.leading.length > 0 ? `${context.leading.join(" · ")} · ${command}` : command; +} + +function extractTrailingMarkdownCodeSpan( + body: string, +): { prefix: string | undefined; value: string } | undefined { + const trimmed = body.trimEnd(); + if (!trimmed.endsWith("`")) { + return undefined; + } + let delimiterLength = 0; + for (let index = trimmed.length - 1; index >= 0 && trimmed[index] === "`"; index -= 1) { + delimiterLength += 1; + } + const delimiter = "`".repeat(delimiterLength); + const valueEnd = trimmed.length - delimiterLength; + let searchIndex = 0; + while (searchIndex < valueEnd) { + const openIndex = trimmed.indexOf(delimiter, searchIndex); + if (openIndex < 0 || openIndex >= valueEnd) { + return undefined; + } + const prefixMatch = trimmed.slice(0, openIndex).match(/^(?:(.*)(?:,\s*| · ))?$/u); + if (prefixMatch) { + return { + prefix: prefixMatch[1], + value: unwrapMarkdownInlineCodePadding( + trimmed.slice(openIndex + delimiterLength, valueEnd), + ), + }; + } + searchIndex = openIndex + delimiterLength; + } + return undefined; +} + +function unwrapMarkdownInlineCodePadding(value: string): string { + if (value.length < 2 || !value.startsWith(" ") || !value.endsWith(" ")) { + return value; + } + const unwrapped = value.slice(1, -1); + return /\S/u.test(unwrapped) ? unwrapped : value; +} +function extractRawExecContext(prefix: string | undefined, inlineCode: string): RawExecContext { + const value = prefix ?? ""; + const leading = [...value.matchAll(/(?:^|,\s*| · )(node:\s*[^,·]+)(?=,\s*| · |$)/gu)] + .map((match) => match[1]?.trim()) + .filter((part): part is string => Boolean(part)); + const trailing = [ + ...value.matchAll( + /(\((?:agent|repo|sandbox|workspace)\)|\(in [^)\r\n]+\))(?=\s*(?:,\s*| · |$))/gu, + ), + ] + .filter((match) => shouldKeepRawExecTrailingContext(value, match, inlineCode)) + .map((match) => match[1]?.trim()) + .filter((part): part is string => Boolean(part)); + return { leading, trailing }; +} +function shouldKeepRawExecTrailingContext( + prefix: string, + match: RegExpMatchArray, + inlineCode: string, +): boolean { + const suffix = match[1]?.trim(); + if (!suffix || inlineCode.includes(suffix)) { + return false; + } + const segment = prefix + .slice(0, match.index ?? 0) + .trimEnd() + .split(/,\s*| · /u) + .at(-1) + ?.trim(); + const segmentCommand = segment ? extractLiteralExecCommand(segment) : undefined; + if (segmentCommand === inlineCode || segment === inlineCode) { + return true; + } + if (isCompactCwdSuffix(suffix)) { + return true; + } + return isPathLikeCwdSuffix(suffix); +} +function isCompactCwdSuffix(suffix: string): boolean { + return /^\((?:agent|repo|workspace)\)$/u.test(suffix); +} +function isPathLikeCwdSuffix(suffix: string): boolean { + const cwd = suffix.match(/^\(in ([^)\r\n]+)\)$/u)?.[1]?.trim(); + return Boolean( + cwd && (/^(?:\/|~|\.{1,2}(?:\/|$)|[A-Za-z]:[\\/]|\\\\)/u.test(cwd) || cwd.includes("/")), + ); +} +function isKnownLiteralRunSummary(subject: string): boolean { + if ( + SEMANTIC_RUN_SUMMARIES.has(subject) || + subject.includes("→") || + subject.includes("->") || + /^(?:node|python3?|ruby|php) inline script(?: \(heredoc\))?$/u.test(subject) + ) { + return false; + } + const match = subject.match(/^(\S+)\s+(.+)$/u); + const command = match?.[1]; + const remainder = match?.[2]; + if (!command || !remainder || remainder === "command") { + return false; + } + return LITERAL_RUN_SUMMARY_PREFIXES.has(command); +} +function splitDisplayContextSuffix(value: string): { text: string; suffix: string } { + const match = /^(.*?)( \((?:agent|repo|workspace|sandbox)\))$/u.exec(value); + if (!match) { + return { text: value, suffix: "" }; + } + return { text: match[1] ?? value, suffix: match[2] ?? "" }; +} +function formatConciseExecExitSuffix(error: string | undefined): string { + const normalized = normalizeOptionalString(error); + const code = normalized?.match( + /\b(?:command\s+)?(?:failed\s+with\s+exit\s+code|exited\s+with\s+code|exit(?:ed)?\s+code|exit\s+status)\s+(-?\d+)\b/iu, + )?.[1]; + return code ? ` (exit ${code})` : ""; +} +function maybeWrapInlineCode(value: string, markdown: boolean): string { + return markdown ? formatInlineCodeSpan(value) : value; +} +/** + * Chooses whether a tool failure needs a separate user-visible warning and + * whether to include raw details. Mutating failures are stricter because a + * silent failed write/send/delete can make the assistant look successful. + */ +function resolveToolErrorWarningPolicy(params: { + lastToolError: ToolErrorSummary; + hasUserFacingReply: boolean; + hasUserFacingErrorReply: boolean; + hasUserFacingFailureAcknowledgement: boolean; + suppressToolErrors: boolean; + suppressToolErrorWarnings?: boolean | (() => boolean | undefined); + verboseLevel?: VerboseLevel; +}): ToolErrorWarningPolicy { + const normalizedToolName = normalizeOptionalLowercaseString(params.lastToolError.toolName) ?? ""; + let toolErrorWarningOverride: boolean | undefined; + let dynamicToolErrorWarningsDisabled = false; + if (typeof params.suppressToolErrorWarnings === "function") { + toolErrorWarningOverride = params.suppressToolErrorWarnings(); + dynamicToolErrorWarningsDisabled = toolErrorWarningOverride === false; + } else { + toolErrorWarningOverride = params.suppressToolErrorWarnings; + } + const includeDetails = + !dynamicToolErrorWarningsDisabled && isVerboseToolDetailEnabled(params.verboseLevel); + const suppressToolErrorWarnings = toolErrorWarningOverride === true; + if (suppressToolErrorWarnings) { + return { showWarning: false, includeDetails }; + } + // sessions_send timeouts and errors are transient inter-session communication + // issues — the message may still have been delivered. Suppress warnings to + // prevent raw error text from leaking into the chat surface (#23989). + if (normalizedToolName === "sessions_send") { + return { showWarning: false, includeDetails }; + } + if (params.suppressToolErrors) { + return { showWarning: false, includeDetails }; + } + // Mutating branch protects "assistant claims success while a user-visible mutation + // silently failed". Shell/exec are the agent's own workspace actions: the model sees + // the exit code in-context, and a successful final reply is recovery proof (#103574). + // Deliberately ignores mutatingAction for exec: codex marks every commandExecution + // mutating fail-closed (replay metadata, not display signal). + if (isExecLikeToolName(params.lastToolError.toolName)) { + // No recoverable-keyword suppression here: with no reply at all, the exec + // warning may be the run's only failure signal. + return { showWarning: !params.hasUserFacingReply, includeDetails }; + } + if (params.lastToolError.terminalDiagnostic?.kind === "process") { + return { showWarning: !params.hasUserFacingReply, includeDetails }; + } + const isMutatingToolError = + params.lastToolError.mutatingAction ?? isLikelyMutatingToolName(params.lastToolError.toolName); + if (isMutatingToolError) { + return { + showWarning: !params.hasUserFacingErrorReply && !params.hasUserFacingFailureAcknowledgement, + includeDetails, + }; + } + return { + showWarning: !params.hasUserFacingReply && !isRecoverableToolError(params.lastToolError.error), + includeDetails, + }; +} + +export function buildFailureWarning(params: { + lastToolError: ToolErrorSummary; + hasUserFacingReply: boolean; + hasUserFacingErrorReply: boolean; + hasUserFacingFailureAcknowledgement: boolean; + suppressToolErrors: boolean; + suppressToolErrorWarnings?: boolean | (() => boolean | undefined); + verboseLevel?: VerboseLevel; + useMarkdown: boolean; +}): { text: string; nonTerminalToolErrorWarning: boolean } | undefined { + const warningPolicy = resolveToolErrorWarningPolicy(params); + if (!warningPolicy.showWarning) { + return undefined; + } + return { + text: formatToolErrorWarningText({ + lastToolError: params.lastToolError, + includeDetails: warningPolicy.includeDetails, + useMarkdown: params.useMarkdown, + }), + nonTerminalToolErrorWarning: shouldMarkNonTerminalToolErrorWarning(params.lastToolError), + }; +}