From cf4cb0ac852a1ffbe1620fc46347fe901ce1ac00 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 27 Jul 2026 23:48:01 -0400 Subject: [PATCH] feat(agents): report per-run stats (code-mode engagement, round trips, cost) in agent JSON envelopes (#114688) * feat(agents): add per-run stats to embedded agent run meta Adds codeModeEngaged, assistantTurns, bridgeCalls, and costUsd to EmbeddedAgentRunMeta.agentMeta and mirrors them on the agent exec --json envelope. Code-mode engagement is stamped from the tool-surface truth, round trips accumulate across attempts beside usage, bridge counts come from the run's tool-search catalog counters, and cost reuses the shared model pricing helpers (cache tiers included, omitted without cost data). * fix(agents): accumulate bridge call counts across run attempts Attempt cleanup clears the per-attempt tool-search catalog, so retries and fallbacks discarded earlier bridge counts. Fold each attempt's bridgeCalls into the run accumulator beside assistantTurns and stamp the cumulative totals into agentMeta, matching the documented per-run contract. --- docs/cli/agent.md | 13 ++ docs/tools/code-mode.md | 18 +++ .../run/attempt-normalization.ts | 6 +- .../run/attempt-result.test.ts | 1 + .../run/attempt-result.ts | 2 + .../attempt.spawn-workspace.test-support.ts | 1 + .../embedded-agent-runner/run/attempt.ts | 18 ++- .../run/settled-turn-finalization.ts | 6 +- .../run/terminal-preparation.test.ts | 135 ++++++++++++++++++ .../run/terminal-preparation.ts | 23 +++ src/agents/embedded-agent-runner/run/types.ts | 14 ++ src/agents/embedded-agent-runner/types.ts | 21 +++ .../usage-accumulator.test.ts | 29 ++++ .../usage-accumulator.ts | 39 +++++ ...-agent-subscribe.handlers.messages.test.ts | 29 ++++ ...edded-agent-subscribe.handlers.messages.ts | 3 + ...embedded-agent-subscribe.handlers.types.ts | 5 + src/agents/embedded-agent-subscribe.ts | 2 + src/commands/agent-exec.ts | 12 ++ 19 files changed, 374 insertions(+), 3 deletions(-) diff --git a/docs/cli/agent.md b/docs/cli/agent.md index 3713980d692b..3ea5a4a17e9f 100644 --- a/docs/cli/agent.md +++ b/docs/cli/agent.md @@ -52,6 +52,10 @@ Plain output writes only the final assistant text to stdout. Diagnostics use std "final": "The focused tests pass.", "payloads": [{ "text": "The focused tests pass." }], "usage": { "input": 120, "output": 8, "total": 128 }, + "costUsd": 0.0021, + "codeModeEngaged": false, + "assistantTurns": 2, + "bridgeCalls": { "search": 1, "describe": 0, "call": 3 }, "model": "gpt-5.6-sol", "provider": "openai", "sessionId": "019..." @@ -60,6 +64,15 @@ Plain output writes only the final assistant text to stdout. Diagnostics use std `status` is `ok`, `error`, or `timeout`. `usage` is omitted when unavailable. Failed envelopes add `error: { message, kind }`; `model` and `provider` are `null` when failure happens before model selection. +Run-stat fields are additive and may be absent: + +- `costUsd`: estimated USD cost of the run's accumulated usage, including cache read/write pricing; omitted when the model has no cost data. +- `codeModeEngaged`: `true` only when [code mode](/tools/code-mode) actually owned the model tool surface for the run. `tools.codeMode.enabled=true` alone does not guarantee engagement; models routed through a native harness surface can leave it `false`. +- `assistantTurns`: completed assistant/provider round trips in the run; omitted when none completed. +- `bridgeCalls`: inner tool-search/code-mode bridge call counts (`search`/`describe`/`call`). These are invisible to the provider; outer tool calls stay in `meta.toolSummary.calls` of the full run metadata. + +The same fields appear on `meta.agentMeta` in the `openclaw agent --json` response. + ### `agent exec` options - `[message]`: positional prompt text diff --git a/docs/tools/code-mode.md b/docs/tools/code-mode.md index ffff45e704e1..b5faa803afd0 100644 --- a/docs/tools/code-mode.md +++ b/docs/tools/code-mode.md @@ -971,6 +971,24 @@ breakdown (`openclaw`/`mcp`/`client` counts), cumulative search/describe/call counts for the run's catalog, and the model-visible tool names (`exec`, `wait`, and retained direct-only tools). +The run metadata (`meta.agentMeta` in `openclaw agent --json`, mirrored on the +`agent exec --json` envelope) adds per-run stats: + +- `codeModeEngaged`: `true` only when code mode actually owned the model tool + surface. This is the reliable engagement signal — do not infer engagement + from config or tool names: the shell tool is also named `exec`, the + `"auto"` tier engages per model capability, and a model routed through a + native harness surface (for example OpenAI-family models on their harness) + reports `codeModeEngaged: false` even with `tools.codeMode.enabled=true`, + making the silent no-op observable. +- `assistantTurns`: completed assistant/provider round trips across the run. +- `bridgeCalls`: the run's cumulative inner bridge counts + (`{ search, describe, call }`). These calls never reach the provider; + provider-visible outer tool calls remain in `meta.toolSummary.calls`. +- `costUsd`: estimated USD cost from the run's accumulated usage and the + model's cost config (cache read/write tiers included); omitted when the + model has no cost data. + Telemetry must not include secrets, raw environment values, or unredacted tool inputs beyond existing OpenClaw trajectory policy. diff --git a/src/agents/embedded-agent-runner/run/attempt-normalization.ts b/src/agents/embedded-agent-runner/run/attempt-normalization.ts index 50a1c8f0b2ce..31ee4be77636 100644 --- a/src/agents/embedded-agent-runner/run/attempt-normalization.ts +++ b/src/agents/embedded-agent-runner/run/attempt-normalization.ts @@ -7,7 +7,10 @@ import { log } from "../logger.js"; import { createEmbeddedRunReplayState, observeReplayMetadata } from "../replay-state.js"; import type { EmbeddedAgentRunResult } from "../types.js"; import type { createUsageAccumulator } from "../usage-accumulator.js"; -import { mergeUsageIntoAccumulator } from "../usage-accumulator.js"; +import { + mergeAttemptRunStatsIntoAccumulator, + mergeUsageIntoAccumulator, +} from "../usage-accumulator.js"; import { applyEmbeddedAttemptSessionIdentity } from "./attempt-session-identity.js"; import type { createEmbeddedRunContextRecoveryState } from "./context-recovery-state.js"; import type { PreparedEmbeddedRunInput } from "./execution-context.js"; @@ -165,6 +168,7 @@ export async function normalizeEmbeddedRunAttempt(input: { }); const attemptUsage = attempt.attemptUsage ?? callUsage.currentAttempt; mergeUsageIntoAccumulator(input.usageAccumulator, attemptUsage); + mergeAttemptRunStatsIntoAccumulator(input.usageAccumulator, attempt); const lastRunPromptUsage = callUsage.latest; const lastTurnTotal = callUsage.latest?.total; const breakerStep = stepIdleTimeoutBreaker(input.idleTimeoutBreakerState, { diff --git a/src/agents/embedded-agent-runner/run/attempt-result.test.ts b/src/agents/embedded-agent-runner/run/attempt-result.test.ts index 30f79e316be4..63118489e90b 100644 --- a/src/agents/embedded-agent-runner/run/attempt-result.test.ts +++ b/src/agents/embedded-agent-runner/run/attempt-result.test.ts @@ -34,6 +34,7 @@ function completeResult(params?: { didSendDeterministicApprovalPrompt: () => false, didSendViaMessagingTool: () => false, getAcceptedSessionSpawns: () => [], + getAssistantTurnCount: () => 0, getCompactionCount: () => 0, getHeartbeatToolResponse: () => undefined, getItemLifecycle: () => undefined, diff --git a/src/agents/embedded-agent-runner/run/attempt-result.ts b/src/agents/embedded-agent-runner/run/attempt-result.ts index 9dafbfede761..c812ca39abcf 100644 --- a/src/agents/embedded-agent-runner/run/attempt-result.ts +++ b/src/agents/embedded-agent-runner/run/attempt-result.ts @@ -150,6 +150,7 @@ export function completeEmbeddedAttemptResult( didSendDeterministicApprovalPrompt, didSendViaMessagingTool, getAcceptedSessionSpawns, + getAssistantTurnCount, getCompactionCount, getHeartbeatToolResponse, getItemLifecycle, @@ -376,6 +377,7 @@ export function completeEmbeddedAttemptResult( replayMetadata, currentAttemptReplayMetadata, itemLifecycle: getItemLifecycle(), + assistantTurns: getAssistantTurnCount(), setTerminalLifecycleMeta, bootstrapPromptWarningSignaturesSeen: input.bootstrapPromptWarning.warningSignaturesSeen, bootstrapPromptWarningSignature: input.bootstrapPromptWarning.signature, diff --git a/src/agents/embedded-agent-runner/run/attempt.spawn-workspace.test-support.ts b/src/agents/embedded-agent-runner/run/attempt.spawn-workspace.test-support.ts index 40ce6e35fa5d..42437608eece 100644 --- a/src/agents/embedded-agent-runner/run/attempt.spawn-workspace.test-support.ts +++ b/src/agents/embedded-agent-runner/run/attempt.spawn-workspace.test-support.ts @@ -158,6 +158,7 @@ function createSubscriptionMock(): SubscriptionMock { getLastToolError: () => undefined, getUsageTotals: () => undefined, getLastAssistantUsage: () => undefined, + getAssistantTurnCount: () => 0, getCompactionCount: () => 0, getLastCompactionTokensAfter: () => undefined, getItemLifecycle: () => ({ startedCount: 0, completedCount: 0, activeCount: 0 }), diff --git a/src/agents/embedded-agent-runner/run/attempt.ts b/src/agents/embedded-agent-runner/run/attempt.ts index afd8c6e0e0b3..02c1829306ae 100644 --- a/src/agents/embedded-agent-runner/run/attempt.ts +++ b/src/agents/embedded-agent-runner/run/attempt.ts @@ -406,7 +406,7 @@ export async function runEmbeddedAttempt( }, }, }); - return await runEmbeddedAttemptExecutionPhase({ + const executionResult = await runEmbeddedAttemptExecutionPhase({ attempt: params, ...(activeContextEngine ? { activeContextEngine } : {}), agentDir, @@ -447,6 +447,22 @@ export async function runEmbeddedAttempt( }, }, }); + // Read catalog counters before the finally-phase cleanup clears the + // run-scoped catalog session; afterwards the counts are gone. + const catalogSession = toolSearchCatalogRef?.current; + return { + ...executionResult, + codeModeEngaged: codeModeControlsEnabledForRun, + ...(catalogSession + ? { + bridgeCalls: { + search: catalogSession.searchCount, + describe: catalogSession.describeCount, + call: catalogSession.callCount, + }, + } + : {}), + }; } finally { const terminal = projectAgentRunAttemptTerminal(executionState.terminal); await cleanupEmbeddedAttemptSessionPhase({ diff --git a/src/agents/embedded-agent-runner/run/settled-turn-finalization.ts b/src/agents/embedded-agent-runner/run/settled-turn-finalization.ts index 269d9b2ec83a..73ca32277738 100644 --- a/src/agents/embedded-agent-runner/run/settled-turn-finalization.ts +++ b/src/agents/embedded-agent-runner/run/settled-turn-finalization.ts @@ -5,7 +5,10 @@ import type { AgentHarnessSettledTurnFinalizationResult, } from "../../harness/types.js"; import { log } from "../logger.js"; -import { mergeUsageIntoAccumulator } from "../usage-accumulator.js"; +import { + mergeAttemptRunStatsIntoAccumulator, + mergeUsageIntoAccumulator, +} from "../usage-accumulator.js"; import { runEmbeddedSettledTurnFinalizationWithBackend } from "./backend.js"; import { EMBEDDED_RUN_LANE_HEARTBEAT_MS } from "./lane-runtime.js"; import { @@ -106,6 +109,7 @@ export async function prepareTerminalWithSettledTurnFinalization(input: { noteLaneTaskProgress: input.finalization.noteLaneTaskProgress, }); mergeUsageIntoAccumulator(input.terminalBase.usageAccumulator, attempt.attemptUsage); + mergeAttemptRunStatsIntoAccumulator(input.terminalBase.usageAccumulator, attempt); lastRunPromptUsage = attempt.attemptUsage ?? lastRunPromptUsage; lastTurnTotal = attempt.attemptUsage?.total ?? lastTurnTotal; // Successful isolated finalization owns a fresh terminal, never the original abort signal. diff --git a/src/agents/embedded-agent-runner/run/terminal-preparation.test.ts b/src/agents/embedded-agent-runner/run/terminal-preparation.test.ts index 5375e6b4c310..57764e33a41f 100644 --- a/src/agents/embedded-agent-runner/run/terminal-preparation.test.ts +++ b/src/agents/embedded-agent-runner/run/terminal-preparation.test.ts @@ -109,3 +109,138 @@ describe("prepareEmbeddedRunTerminal", () => { }, ); }); + +describe("prepareEmbeddedRunTerminal run stats", () => { + type StatsInput = { + attempt?: Partial; + assistantTurns?: number; + bridgeCalls?: { search: number; describe: number; call: number }; + config?: unknown; + provider?: string; + model?: string; + usage?: Partial< + Pick< + ReturnType, + "input" | "output" | "cacheRead" | "cacheWrite" | "total" + > + >; + }; + + async function prepareStats(statsInput: StatsInput = {}) { + const { prepareEmbeddedRunTerminal } = await import("./terminal-preparation.js"); + const provider = statsInput.provider ?? "cost-test-provider"; + const model = statsInput.model ?? "cost-model"; + const assistant = { + ...assistantMessage("stop"), + provider, + model, + }; + const usageAccumulator = createUsageAccumulator(); + Object.assign(usageAccumulator, statsInput.usage); + usageAccumulator.assistantTurns = statsInput.assistantTurns ?? 0; + usageAccumulator.bridgeCalls = statsInput.bridgeCalls; + return prepareEmbeddedRunTerminal({ + runParams: { + sessionId: "session-1", + runId: "run-1", + workspaceDir: "/tmp/openclaw-test", + prompt: "hi", + trigger: "user", + timeoutMs: 60_000, + ...(statsInput.config ? { config: statsInput.config as never } : {}), + }, + attempt: attemptResult({ + lastAssistant: assistant, + currentAttemptAssistant: assistant, + currentAttemptCompletedAssistant: assistant, + ...statsInput.attempt, + }), + currentAttemptCompletedAssistant: assistant, + provider, + model, + activeErrorContext: { provider, model }, + authProfileStore: { version: 1, profiles: {} }, + sessionIdUsed: "session-1", + outerContextTokenMeta: {}, + usageAccumulator, + contextRecoveryState: createEmbeddedRunContextRecoveryState(), + resolvedToolResultFormat: "markdown", + terminalState: { + outcome: { reason: "completed", status: "ok", stopReason: "stop" }, + signalOwnedInterruption: false, + }, + }); + } + + const COST_CONFIG = { + models: { + providers: { + "cost-test-provider": { + models: [ + { + id: "cost-model", + cost: { input: 1, output: 2, cacheRead: 0.5, cacheWrite: 4 }, + }, + ], + }, + }, + }, + }; + + it.each([ + { name: "engaged", codeModeEngaged: true, expected: true }, + { name: "not engaged", codeModeEngaged: false, expected: false }, + { name: "unreported (harness route)", codeModeEngaged: undefined, expected: false }, + ])("stamps codeModeEngaged when $name", async ({ codeModeEngaged, expected }) => { + const prepared = await prepareStats({ attempt: { codeModeEngaged } }); + expect(prepared.agentMeta.codeModeEngaged).toBe(expected); + }); + + it("stamps assistantTurns from the run accumulator and omits zero", async () => { + const counted = await prepareStats({ assistantTurns: 3 }); + expect(counted.agentMeta.assistantTurns).toBe(3); + + const empty = await prepareStats({ assistantTurns: 0 }); + expect(empty.agentMeta).not.toHaveProperty("assistantTurns"); + }); + + it("stamps run-accumulated bridge call counts and omits them when absent", async () => { + const withBridge = await prepareStats({ + bridgeCalls: { search: 2, describe: 1, call: 5 }, + }); + expect(withBridge.agentMeta.bridgeCalls).toEqual({ search: 2, describe: 1, call: 5 }); + + const withoutBridge = await prepareStats({}); + expect(withoutBridge.agentMeta).not.toHaveProperty("bridgeCalls"); + }); + + it("computes costUsd from accumulated usage including cache pricing", async () => { + const prepared = await prepareStats({ + config: COST_CONFIG, + usage: { + input: 1_000_000, + output: 500_000, + cacheRead: 2_000_000, + cacheWrite: 250_000, + total: 3_750_000, + }, + }); + // (1M*$1 + 0.5M*$2 + 2M*$0.5 + 0.25M*$4) per million tokens. + expect(prepared.agentMeta.costUsd).toBeCloseTo(4, 10); + }); + + it("omits costUsd when the model has no cost data", async () => { + const prepared = await prepareStats({ + provider: "no-cost-provider", + model: "uncosted-model", + config: COST_CONFIG, + usage: { input: 1_000_000, output: 500_000, total: 1_500_000 }, + }); + expect(prepared.agentMeta).not.toHaveProperty("costUsd"); + }); + + it("omits costUsd when the run reported no usage", async () => { + const prepared = await prepareStats({ config: COST_CONFIG }); + expect(prepared.agentMeta).not.toHaveProperty("costUsd"); + }); +}); diff --git a/src/agents/embedded-agent-runner/run/terminal-preparation.ts b/src/agents/embedded-agent-runner/run/terminal-preparation.ts index b5f5a02ba1ee..44c3f3f7f0ef 100644 --- a/src/agents/embedded-agent-runner/run/terminal-preparation.ts +++ b/src/agents/embedded-agent-runner/run/terminal-preparation.ts @@ -1,5 +1,6 @@ import { copyReplyPayloadMetadata } from "../../../auto-reply/reply-payload.js"; import type { AssistantMessage } from "../../../llm/types.js"; +import { estimateUsageCost, resolveModelCostConfig } from "../../../utils/usage-format.js"; import { projectAgentRunAttemptTerminal } from "../../agent-run-terminal-outcome.js"; import type { AuthProfileStore } from "../../auth-profiles.js"; import type { NormalizedUsage, UsageLike } from "../../usage.js"; @@ -81,6 +82,20 @@ export function prepareEmbeddedRunTerminal(input: { const finalAssistantStopReason = (terminalAssistant?.stopReason ?? "").trim().toLowerCase(); const terminalAssistantCanOwnFinalText = finalAssistantStopReason !== "error" && finalAssistantStopReason !== "aborted"; + // Total-only usage (lastTurnTotal override) carries no token split, so cost + // math uses the accumulated input/output/cache fields untouched by it. + const costUsd = estimateUsageCost({ + usage: usageMeta.usage, + cost: resolveModelCostConfig({ + provider: reportedModelRef.provider, + model: reportedModelRef.model, + config: runParams.config, + agentDir: runParams.agentDir, + }), + }); + // Attempt normalization already folded every attempt (terminal included) + // into the accumulator, so read it directly instead of re-adding the attempt. + const runAssistantTurns = input.usageAccumulator.assistantTurns; const agentMeta: EmbeddedAgentMeta = { sessionId: input.sessionIdUsed, sessionFile: input.sessionFileUsed, @@ -99,6 +114,14 @@ export function prepareEmbeddedRunTerminal(input: { ? input.contextRecoveryState.autoCompactionCount : undefined, compactionTokensAfter: input.contextRecoveryState.lastCompactionTokensAfter, + // Absent attempt engagement (plugin harness routes) intentionally reads as + // false so config-enabled-but-unengaged code mode is visible to consumers. + codeModeEngaged: attempt.codeModeEngaged === true, + ...(runAssistantTurns > 0 ? { assistantTurns: runAssistantTurns } : {}), + ...(input.usageAccumulator.bridgeCalls + ? { bridgeCalls: { ...input.usageAccumulator.bridgeCalls } } + : {}), + ...(costUsd !== undefined ? { costUsd } : {}), }; const attemptFinalText = attempt.assistantTexts .toReversed() diff --git a/src/agents/embedded-agent-runner/run/types.ts b/src/agents/embedded-agent-runner/run/types.ts index 2db04c1a437f..e0c919bacb03 100644 --- a/src/agents/embedded-agent-runner/run/types.ts +++ b/src/agents/embedded-agent-runner/run/types.ts @@ -304,6 +304,20 @@ export type EmbeddedRunAttemptResult = { clientToolCalls?: Array<{ name: string; params: Record }>; /** True when sessions_yield tool was called during this attempt. */ yieldDetected?: boolean; + /** + * True when code mode owned this attempt's model tool surface. Absent means + * the harness did not report engagement (treated as not engaged), which is + * how config-enabled code mode stays visible as a no-op on harness routes. + */ + codeModeEngaged?: boolean; + /** Completed assistant round trips observed during this attempt. */ + assistantTurns?: number; + /** Inner bridge call counts from this attempt's tool-search/code-mode catalog. */ + bridgeCalls?: { + search: number; + describe: number; + call: number; + }; replayMetadata: EmbeddedRunReplayMetadata; /** * Replay metadata for this attempt before prior session state is accumulated. diff --git a/src/agents/embedded-agent-runner/types.ts b/src/agents/embedded-agent-runner/types.ts index c83287fc4f3c..e0f91ac99771 100644 --- a/src/agents/embedded-agent-runner/types.ts +++ b/src/agents/embedded-agent-runner/types.ts @@ -82,6 +82,27 @@ export type EmbeddedAgentMeta = { total?: number; }; contextBudgetStatus?: SessionContextBudgetStatus; + /** + * True when code mode owned the model tool surface for this run. Config + * alone is not proof: the "auto" tier engages per model capability, raw + * model runs and plugin-harness surfaces can decline engagement, and the + * shell tool is also named `exec`, so consumers must read this flag + * instead of config or tool names. + */ + codeModeEngaged?: boolean; + /** Completed assistant/provider round trips accumulated across run attempts. */ + assistantTurns?: number; + /** + * Code-mode/tool-search inner bridge calls for the run's catalog. These are + * invisible to the provider; `toolSummary.calls` stays the outer count. + */ + bridgeCalls?: { + search: number; + describe: number; + call: number; + }; + /** Estimated USD cost of the run's accumulated usage. Omitted when the model has no cost data. */ + costUsd?: number; }; export type TraceAttempt = { diff --git a/src/agents/embedded-agent-runner/usage-accumulator.test.ts b/src/agents/embedded-agent-runner/usage-accumulator.test.ts index 2416cc21fc43..cc8a30b40a70 100644 --- a/src/agents/embedded-agent-runner/usage-accumulator.test.ts +++ b/src/agents/embedded-agent-runner/usage-accumulator.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; import { createUsageAccumulator, + mergeAttemptRunStatsIntoAccumulator, mergeUsageIntoAccumulator, toNormalizedUsage, } from "./usage-accumulator.js"; @@ -79,6 +80,34 @@ describe("usage-accumulator", () => { }); }); + describe("mergeAttemptRunStatsIntoAccumulator", () => { + it("accumulates turns and bridge calls across retry/fallback attempts", () => { + const acc = createUsageAccumulator(); + + // First attempt makes bridge calls, then a retry/fallback attempt runs. + mergeAttemptRunStatsIntoAccumulator(acc, { + assistantTurns: 2, + bridgeCalls: { search: 1, describe: 2, call: 3 }, + }); + mergeAttemptRunStatsIntoAccumulator(acc, { + assistantTurns: 1, + bridgeCalls: { search: 0, describe: 1, call: 4 }, + }); + + expect(acc.assistantTurns).toBe(3); + expect(acc.bridgeCalls).toEqual({ search: 1, describe: 3, call: 7 }); + }); + + it("keeps bridgeCalls absent for catalog-less attempts", () => { + const acc = createUsageAccumulator(); + + mergeAttemptRunStatsIntoAccumulator(acc, { assistantTurns: 1 }); + + expect(acc.assistantTurns).toBe(1); + expect(acc.bridgeCalls).toBeUndefined(); + }); + }); + describe("toNormalizedUsage", () => { it("returns undefined for an empty accumulator", () => { expect(toNormalizedUsage(createUsageAccumulator())).toBeUndefined(); diff --git a/src/agents/embedded-agent-runner/usage-accumulator.ts b/src/agents/embedded-agent-runner/usage-accumulator.ts index bfc8e133f29e..071c4beb14a1 100644 --- a/src/agents/embedded-agent-runner/usage-accumulator.ts +++ b/src/agents/embedded-agent-runner/usage-accumulator.ts @@ -10,6 +10,21 @@ export type UsageAccumulator = { cacheWrite: number; reasoningTokens: number; total: number; + /** + * Completed assistant round trips across every model attempt of the run. + * Kept beside token totals so retried attempts stay counted like their usage. + */ + assistantTurns: number; + /** + * Cumulative inner bridge calls across attempts. Present only once an + * attempt reported a tool-search/code-mode catalog, so catalog-less runs + * omit the field instead of publishing zero sentinels. + */ + bridgeCalls?: { + search: number; + describe: number; + call: number; + }; }; export const createUsageAccumulator = (): UsageAccumulator => ({ @@ -19,6 +34,7 @@ export const createUsageAccumulator = (): UsageAccumulator => ({ cacheWrite: 0, reasoningTokens: 0, total: 0, + assistantTurns: 0, }); type MaybeUsage = NormalizedUsage | undefined; @@ -57,6 +73,29 @@ export const mergeUsageIntoAccumulator = (target: UsageAccumulator, usage: Maybe target.total += callTotal; }; +/** + * Folds one attempt's run stats into the accumulator. Attempt cleanup clears + * the per-attempt tool-search catalog, so retries would otherwise discard + * earlier bridge counts and undercount the documented cumulative run totals. + */ +export const mergeAttemptRunStatsIntoAccumulator = ( + target: UsageAccumulator, + attempt: { + assistantTurns?: number; + bridgeCalls?: { search: number; describe: number; call: number }; + }, +) => { + target.assistantTurns += attempt.assistantTurns ?? 0; + if (!attempt.bridgeCalls) { + return; + } + const bridgeCalls = target.bridgeCalls ?? { search: 0, describe: 0, call: 0 }; + bridgeCalls.search += attempt.bridgeCalls.search; + bridgeCalls.describe += attempt.bridgeCalls.describe; + bridgeCalls.call += attempt.bridgeCalls.call; + target.bridgeCalls = bridgeCalls; +}; + export const toNormalizedUsage = (usage: UsageAccumulator): NormalizedUsage | undefined => { const hasUsage = usage.input > 0 || diff --git a/src/agents/embedded-agent-subscribe.handlers.messages.test.ts b/src/agents/embedded-agent-subscribe.handlers.messages.test.ts index b3e80e7a5530..52aff7b241a8 100644 --- a/src/agents/embedded-agent-subscribe.handlers.messages.test.ts +++ b/src/agents/embedded-agent-subscribe.handlers.messages.test.ts @@ -1391,6 +1391,35 @@ describe("handleMessageUpdate commentary phase", () => { }); describe("handleMessageEnd", () => { + it.each([ + { + name: "counts a completed provider assistant message", + message: { role: "assistant", content: [{ type: "text", text: "Done." }] }, + expected: 1, + }, + { + name: "ignores transcript-only mirrored assistant messages", + message: { + role: "assistant", + provider: "openclaw", + model: "delivery-mirror", + content: [{ type: "text", text: "Done." }], + }, + expected: 0, + }, + { + name: "ignores non-assistant messages", + message: { role: "user", content: [{ type: "text", text: "hi" }] }, + expected: 0, + }, + ])("$name for assistantTurnCount", ({ message, expected }) => { + const ctx = createMessageEndContext({ state: { assistantTurnCount: 0 } }); + + void endMessage(ctx, { message }); + + expect(ctx.state.assistantTurnCount).toBe(expected); + }); + it("keeps duplicate-reply diagnostics free of lone surrogates", () => { const text = `${"a".repeat(49)}😀tail`; const ctx = createMessageEndContext({ diff --git a/src/agents/embedded-agent-subscribe.handlers.messages.ts b/src/agents/embedded-agent-subscribe.handlers.messages.ts index a69d18b2e022..3e82e995ae13 100644 --- a/src/agents/embedded-agent-subscribe.handlers.messages.ts +++ b/src/agents/embedded-agent-subscribe.handlers.messages.ts @@ -1153,6 +1153,9 @@ export function handleMessageEnd( return; } + // Transcript-only messages never reach the provider, so this counts exactly + // the completed model round trips consumers see as `assistantTurns`. + ctx.state.assistantTurnCount += 1; const assistantMessage = preservePendingAssistantUsage(msg, ctx.state.pendingAssistantUsage); const assistantPhase = resolveAssistantMessagePhase(assistantMessage); const suppressVisibleAssistantOutput = shouldSuppressAssistantVisibleOutput(assistantMessage); diff --git a/src/agents/embedded-agent-subscribe.handlers.types.ts b/src/agents/embedded-agent-subscribe.handlers.types.ts index 6bd5b3c3680f..60c1d6496885 100644 --- a/src/agents/embedded-agent-subscribe.handlers.types.ts +++ b/src/agents/embedded-agent-subscribe.handlers.types.ts @@ -85,6 +85,11 @@ export type EmbeddedAgentSubscribeState = { itemActiveIds: Set; itemStartedCount: number; itemCompletedCount: number; + /** + * Completed assistant round trips in this attempt. Survives compaction-retry + * presentation resets, matching how usage totals keep counting model calls. + */ + assistantTurnCount: number; lastToolError?: ToolErrorSummary; latestMcpAppChannelView?: McpAppChannelView; diff --git a/src/agents/embedded-agent-subscribe.ts b/src/agents/embedded-agent-subscribe.ts index 0a4e7c8118ab..1d62a6a39c1b 100644 --- a/src/agents/embedded-agent-subscribe.ts +++ b/src/agents/embedded-agent-subscribe.ts @@ -178,6 +178,7 @@ export function subscribeEmbeddedAgentSession(params: SubscribeEmbeddedAgentSess itemActiveIds: new Set(), itemStartedCount: 0, itemCompletedCount: 0, + assistantTurnCount: 0, lastToolError: undefined, blockReplyBreak: params.blockReplyBreak ?? "text_end", reasoningMode, @@ -1531,6 +1532,7 @@ export function subscribeEmbeddedAgentSession(params: SubscribeEmbeddedAgentSess getLastAssistantUsage, getCompactionCount: () => compactionCount, getLastCompactionTokensAfter: () => state.lastCompactionTokensAfter, + getAssistantTurnCount: () => state.assistantTurnCount, waitForPendingEvents: () => state.pendingEventChain ?? Promise.resolve(), getItemLifecycle: () => ({ startedCount: state.itemStartedCount, diff --git a/src/commands/agent-exec.ts b/src/commands/agent-exec.ts index ec015282c801..37afa64a2df5 100644 --- a/src/commands/agent-exec.ts +++ b/src/commands/agent-exec.ts @@ -51,6 +51,10 @@ export type AgentExecEnvelope = { final: string; payloads: AgentExecPayload[]; usage?: NonNullable["usage"]>; + costUsd?: number; + codeModeEngaged?: boolean; + assistantTurns?: number; + bridgeCalls?: NonNullable["bridgeCalls"]>; model: string | null; provider: string | null; sessionId: string; @@ -227,6 +231,14 @@ export function classifyAgentExecResult( final: finalTextFromResult(result, payloads, !hasErrorPayload), payloads, ...(agentMeta?.usage ? { usage: agentMeta.usage } : {}), + ...(agentMeta?.costUsd !== undefined ? { costUsd: agentMeta.costUsd } : {}), + ...(agentMeta?.codeModeEngaged !== undefined + ? { codeModeEngaged: agentMeta.codeModeEngaged } + : {}), + ...(agentMeta?.assistantTurns !== undefined + ? { assistantTurns: agentMeta.assistantTurns } + : {}), + ...(agentMeta?.bridgeCalls ? { bridgeCalls: agentMeta.bridgeCalls } : {}), model: agentMeta?.model ?? null, provider: agentMeta?.provider ?? null, sessionId: agentMeta?.sessionId ?? "",