mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-28 05:16:23 -06:00
feat(agents): report per-run stats (code-mode engagement, round trips, cost) in agent JSON envelopes (#114688)
* feat(agents): add per-run stats to embedded agent run meta Adds codeModeEngaged, assistantTurns, bridgeCalls, and costUsd to EmbeddedAgentRunMeta.agentMeta and mirrors them on the agent exec --json envelope. Code-mode engagement is stamped from the tool-surface truth, round trips accumulate across attempts beside usage, bridge counts come from the run's tool-search catalog counters, and cost reuses the shared model pricing helpers (cache tiers included, omitted without cost data). * fix(agents): accumulate bridge call counts across run attempts Attempt cleanup clears the per-attempt tool-search catalog, so retries and fallbacks discarded earlier bridge counts. Fold each attempt's bridgeCalls into the run accumulator beside assistantTurns and stamp the cumulative totals into agentMeta, matching the documented per-run contract.
This commit is contained in:
committed by
GitHub
parent
e1ced6de50
commit
cf4cb0ac85
@@ -52,6 +52,10 @@ Plain output writes only the final assistant text to stdout. Diagnostics use std
|
||||
"final": "The focused tests pass.",
|
||||
"payloads": [{ "text": "The focused tests pass." }],
|
||||
"usage": { "input": 120, "output": 8, "total": 128 },
|
||||
"costUsd": 0.0021,
|
||||
"codeModeEngaged": false,
|
||||
"assistantTurns": 2,
|
||||
"bridgeCalls": { "search": 1, "describe": 0, "call": 3 },
|
||||
"model": "gpt-5.6-sol",
|
||||
"provider": "openai",
|
||||
"sessionId": "019..."
|
||||
@@ -60,6 +64,15 @@ Plain output writes only the final assistant text to stdout. Diagnostics use std
|
||||
|
||||
`status` is `ok`, `error`, or `timeout`. `usage` is omitted when unavailable. Failed envelopes add `error: { message, kind }`; `model` and `provider` are `null` when failure happens before model selection.
|
||||
|
||||
Run-stat fields are additive and may be absent:
|
||||
|
||||
- `costUsd`: estimated USD cost of the run's accumulated usage, including cache read/write pricing; omitted when the model has no cost data.
|
||||
- `codeModeEngaged`: `true` only when [code mode](/tools/code-mode) actually owned the model tool surface for the run. `tools.codeMode.enabled=true` alone does not guarantee engagement; models routed through a native harness surface can leave it `false`.
|
||||
- `assistantTurns`: completed assistant/provider round trips in the run; omitted when none completed.
|
||||
- `bridgeCalls`: inner tool-search/code-mode bridge call counts (`search`/`describe`/`call`). These are invisible to the provider; outer tool calls stay in `meta.toolSummary.calls` of the full run metadata.
|
||||
|
||||
The same fields appear on `meta.agentMeta` in the `openclaw agent --json` response.
|
||||
|
||||
### `agent exec` options
|
||||
|
||||
- `[message]`: positional prompt text
|
||||
|
||||
@@ -971,6 +971,24 @@ breakdown (`openclaw`/`mcp`/`client` counts), cumulative search/describe/call
|
||||
counts for the run's catalog, and the model-visible tool names (`exec`,
|
||||
`wait`, and retained direct-only tools).
|
||||
|
||||
The run metadata (`meta.agentMeta` in `openclaw agent --json`, mirrored on the
|
||||
`agent exec --json` envelope) adds per-run stats:
|
||||
|
||||
- `codeModeEngaged`: `true` only when code mode actually owned the model tool
|
||||
surface. This is the reliable engagement signal — do not infer engagement
|
||||
from config or tool names: the shell tool is also named `exec`, the
|
||||
`"auto"` tier engages per model capability, and a model routed through a
|
||||
native harness surface (for example OpenAI-family models on their harness)
|
||||
reports `codeModeEngaged: false` even with `tools.codeMode.enabled=true`,
|
||||
making the silent no-op observable.
|
||||
- `assistantTurns`: completed assistant/provider round trips across the run.
|
||||
- `bridgeCalls`: the run's cumulative inner bridge counts
|
||||
(`{ search, describe, call }`). These calls never reach the provider;
|
||||
provider-visible outer tool calls remain in `meta.toolSummary.calls`.
|
||||
- `costUsd`: estimated USD cost from the run's accumulated usage and the
|
||||
model's cost config (cache read/write tiers included); omitted when the
|
||||
model has no cost data.
|
||||
|
||||
Telemetry must not include secrets, raw environment values, or unredacted
|
||||
tool inputs beyond existing OpenClaw trajectory policy.
|
||||
|
||||
|
||||
@@ -7,7 +7,10 @@ import { log } from "../logger.js";
|
||||
import { createEmbeddedRunReplayState, observeReplayMetadata } from "../replay-state.js";
|
||||
import type { EmbeddedAgentRunResult } from "../types.js";
|
||||
import type { createUsageAccumulator } from "../usage-accumulator.js";
|
||||
import { mergeUsageIntoAccumulator } from "../usage-accumulator.js";
|
||||
import {
|
||||
mergeAttemptRunStatsIntoAccumulator,
|
||||
mergeUsageIntoAccumulator,
|
||||
} from "../usage-accumulator.js";
|
||||
import { applyEmbeddedAttemptSessionIdentity } from "./attempt-session-identity.js";
|
||||
import type { createEmbeddedRunContextRecoveryState } from "./context-recovery-state.js";
|
||||
import type { PreparedEmbeddedRunInput } from "./execution-context.js";
|
||||
@@ -165,6 +168,7 @@ export async function normalizeEmbeddedRunAttempt(input: {
|
||||
});
|
||||
const attemptUsage = attempt.attemptUsage ?? callUsage.currentAttempt;
|
||||
mergeUsageIntoAccumulator(input.usageAccumulator, attemptUsage);
|
||||
mergeAttemptRunStatsIntoAccumulator(input.usageAccumulator, attempt);
|
||||
const lastRunPromptUsage = callUsage.latest;
|
||||
const lastTurnTotal = callUsage.latest?.total;
|
||||
const breakerStep = stepIdleTimeoutBreaker(input.idleTimeoutBreakerState, {
|
||||
|
||||
@@ -34,6 +34,7 @@ function completeResult(params?: {
|
||||
didSendDeterministicApprovalPrompt: () => false,
|
||||
didSendViaMessagingTool: () => false,
|
||||
getAcceptedSessionSpawns: () => [],
|
||||
getAssistantTurnCount: () => 0,
|
||||
getCompactionCount: () => 0,
|
||||
getHeartbeatToolResponse: () => undefined,
|
||||
getItemLifecycle: () => undefined,
|
||||
|
||||
@@ -150,6 +150,7 @@ export function completeEmbeddedAttemptResult(
|
||||
didSendDeterministicApprovalPrompt,
|
||||
didSendViaMessagingTool,
|
||||
getAcceptedSessionSpawns,
|
||||
getAssistantTurnCount,
|
||||
getCompactionCount,
|
||||
getHeartbeatToolResponse,
|
||||
getItemLifecycle,
|
||||
@@ -376,6 +377,7 @@ export function completeEmbeddedAttemptResult(
|
||||
replayMetadata,
|
||||
currentAttemptReplayMetadata,
|
||||
itemLifecycle: getItemLifecycle(),
|
||||
assistantTurns: getAssistantTurnCount(),
|
||||
setTerminalLifecycleMeta,
|
||||
bootstrapPromptWarningSignaturesSeen: input.bootstrapPromptWarning.warningSignaturesSeen,
|
||||
bootstrapPromptWarningSignature: input.bootstrapPromptWarning.signature,
|
||||
|
||||
@@ -158,6 +158,7 @@ function createSubscriptionMock(): SubscriptionMock {
|
||||
getLastToolError: () => undefined,
|
||||
getUsageTotals: () => undefined,
|
||||
getLastAssistantUsage: () => undefined,
|
||||
getAssistantTurnCount: () => 0,
|
||||
getCompactionCount: () => 0,
|
||||
getLastCompactionTokensAfter: () => undefined,
|
||||
getItemLifecycle: () => ({ startedCount: 0, completedCount: 0, activeCount: 0 }),
|
||||
|
||||
@@ -406,7 +406,7 @@ export async function runEmbeddedAttempt(
|
||||
},
|
||||
},
|
||||
});
|
||||
return await runEmbeddedAttemptExecutionPhase({
|
||||
const executionResult = await runEmbeddedAttemptExecutionPhase({
|
||||
attempt: params,
|
||||
...(activeContextEngine ? { activeContextEngine } : {}),
|
||||
agentDir,
|
||||
@@ -447,6 +447,22 @@ export async function runEmbeddedAttempt(
|
||||
},
|
||||
},
|
||||
});
|
||||
// Read catalog counters before the finally-phase cleanup clears the
|
||||
// run-scoped catalog session; afterwards the counts are gone.
|
||||
const catalogSession = toolSearchCatalogRef?.current;
|
||||
return {
|
||||
...executionResult,
|
||||
codeModeEngaged: codeModeControlsEnabledForRun,
|
||||
...(catalogSession
|
||||
? {
|
||||
bridgeCalls: {
|
||||
search: catalogSession.searchCount,
|
||||
describe: catalogSession.describeCount,
|
||||
call: catalogSession.callCount,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
} finally {
|
||||
const terminal = projectAgentRunAttemptTerminal(executionState.terminal);
|
||||
await cleanupEmbeddedAttemptSessionPhase({
|
||||
|
||||
@@ -5,7 +5,10 @@ import type {
|
||||
AgentHarnessSettledTurnFinalizationResult,
|
||||
} from "../../harness/types.js";
|
||||
import { log } from "../logger.js";
|
||||
import { mergeUsageIntoAccumulator } from "../usage-accumulator.js";
|
||||
import {
|
||||
mergeAttemptRunStatsIntoAccumulator,
|
||||
mergeUsageIntoAccumulator,
|
||||
} from "../usage-accumulator.js";
|
||||
import { runEmbeddedSettledTurnFinalizationWithBackend } from "./backend.js";
|
||||
import { EMBEDDED_RUN_LANE_HEARTBEAT_MS } from "./lane-runtime.js";
|
||||
import {
|
||||
@@ -106,6 +109,7 @@ export async function prepareTerminalWithSettledTurnFinalization(input: {
|
||||
noteLaneTaskProgress: input.finalization.noteLaneTaskProgress,
|
||||
});
|
||||
mergeUsageIntoAccumulator(input.terminalBase.usageAccumulator, attempt.attemptUsage);
|
||||
mergeAttemptRunStatsIntoAccumulator(input.terminalBase.usageAccumulator, attempt);
|
||||
lastRunPromptUsage = attempt.attemptUsage ?? lastRunPromptUsage;
|
||||
lastTurnTotal = attempt.attemptUsage?.total ?? lastTurnTotal;
|
||||
// Successful isolated finalization owns a fresh terminal, never the original abort signal.
|
||||
|
||||
@@ -109,3 +109,138 @@ describe("prepareEmbeddedRunTerminal", () => {
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("prepareEmbeddedRunTerminal run stats", () => {
|
||||
type StatsInput = {
|
||||
attempt?: Partial<EmbeddedRunAttemptResult>;
|
||||
assistantTurns?: number;
|
||||
bridgeCalls?: { search: number; describe: number; call: number };
|
||||
config?: unknown;
|
||||
provider?: string;
|
||||
model?: string;
|
||||
usage?: Partial<
|
||||
Pick<
|
||||
ReturnType<typeof createUsageAccumulator>,
|
||||
"input" | "output" | "cacheRead" | "cacheWrite" | "total"
|
||||
>
|
||||
>;
|
||||
};
|
||||
|
||||
async function prepareStats(statsInput: StatsInput = {}) {
|
||||
const { prepareEmbeddedRunTerminal } = await import("./terminal-preparation.js");
|
||||
const provider = statsInput.provider ?? "cost-test-provider";
|
||||
const model = statsInput.model ?? "cost-model";
|
||||
const assistant = {
|
||||
...assistantMessage("stop"),
|
||||
provider,
|
||||
model,
|
||||
};
|
||||
const usageAccumulator = createUsageAccumulator();
|
||||
Object.assign(usageAccumulator, statsInput.usage);
|
||||
usageAccumulator.assistantTurns = statsInput.assistantTurns ?? 0;
|
||||
usageAccumulator.bridgeCalls = statsInput.bridgeCalls;
|
||||
return prepareEmbeddedRunTerminal({
|
||||
runParams: {
|
||||
sessionId: "session-1",
|
||||
runId: "run-1",
|
||||
workspaceDir: "/tmp/openclaw-test",
|
||||
prompt: "hi",
|
||||
trigger: "user",
|
||||
timeoutMs: 60_000,
|
||||
...(statsInput.config ? { config: statsInput.config as never } : {}),
|
||||
},
|
||||
attempt: attemptResult({
|
||||
lastAssistant: assistant,
|
||||
currentAttemptAssistant: assistant,
|
||||
currentAttemptCompletedAssistant: assistant,
|
||||
...statsInput.attempt,
|
||||
}),
|
||||
currentAttemptCompletedAssistant: assistant,
|
||||
provider,
|
||||
model,
|
||||
activeErrorContext: { provider, model },
|
||||
authProfileStore: { version: 1, profiles: {} },
|
||||
sessionIdUsed: "session-1",
|
||||
outerContextTokenMeta: {},
|
||||
usageAccumulator,
|
||||
contextRecoveryState: createEmbeddedRunContextRecoveryState(),
|
||||
resolvedToolResultFormat: "markdown",
|
||||
terminalState: {
|
||||
outcome: { reason: "completed", status: "ok", stopReason: "stop" },
|
||||
signalOwnedInterruption: false,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const COST_CONFIG = {
|
||||
models: {
|
||||
providers: {
|
||||
"cost-test-provider": {
|
||||
models: [
|
||||
{
|
||||
id: "cost-model",
|
||||
cost: { input: 1, output: 2, cacheRead: 0.5, cacheWrite: 4 },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
it.each([
|
||||
{ name: "engaged", codeModeEngaged: true, expected: true },
|
||||
{ name: "not engaged", codeModeEngaged: false, expected: false },
|
||||
{ name: "unreported (harness route)", codeModeEngaged: undefined, expected: false },
|
||||
])("stamps codeModeEngaged when $name", async ({ codeModeEngaged, expected }) => {
|
||||
const prepared = await prepareStats({ attempt: { codeModeEngaged } });
|
||||
expect(prepared.agentMeta.codeModeEngaged).toBe(expected);
|
||||
});
|
||||
|
||||
it("stamps assistantTurns from the run accumulator and omits zero", async () => {
|
||||
const counted = await prepareStats({ assistantTurns: 3 });
|
||||
expect(counted.agentMeta.assistantTurns).toBe(3);
|
||||
|
||||
const empty = await prepareStats({ assistantTurns: 0 });
|
||||
expect(empty.agentMeta).not.toHaveProperty("assistantTurns");
|
||||
});
|
||||
|
||||
it("stamps run-accumulated bridge call counts and omits them when absent", async () => {
|
||||
const withBridge = await prepareStats({
|
||||
bridgeCalls: { search: 2, describe: 1, call: 5 },
|
||||
});
|
||||
expect(withBridge.agentMeta.bridgeCalls).toEqual({ search: 2, describe: 1, call: 5 });
|
||||
|
||||
const withoutBridge = await prepareStats({});
|
||||
expect(withoutBridge.agentMeta).not.toHaveProperty("bridgeCalls");
|
||||
});
|
||||
|
||||
it("computes costUsd from accumulated usage including cache pricing", async () => {
|
||||
const prepared = await prepareStats({
|
||||
config: COST_CONFIG,
|
||||
usage: {
|
||||
input: 1_000_000,
|
||||
output: 500_000,
|
||||
cacheRead: 2_000_000,
|
||||
cacheWrite: 250_000,
|
||||
total: 3_750_000,
|
||||
},
|
||||
});
|
||||
// (1M*$1 + 0.5M*$2 + 2M*$0.5 + 0.25M*$4) per million tokens.
|
||||
expect(prepared.agentMeta.costUsd).toBeCloseTo(4, 10);
|
||||
});
|
||||
|
||||
it("omits costUsd when the model has no cost data", async () => {
|
||||
const prepared = await prepareStats({
|
||||
provider: "no-cost-provider",
|
||||
model: "uncosted-model",
|
||||
config: COST_CONFIG,
|
||||
usage: { input: 1_000_000, output: 500_000, total: 1_500_000 },
|
||||
});
|
||||
expect(prepared.agentMeta).not.toHaveProperty("costUsd");
|
||||
});
|
||||
|
||||
it("omits costUsd when the run reported no usage", async () => {
|
||||
const prepared = await prepareStats({ config: COST_CONFIG });
|
||||
expect(prepared.agentMeta).not.toHaveProperty("costUsd");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { copyReplyPayloadMetadata } from "../../../auto-reply/reply-payload.js";
|
||||
import type { AssistantMessage } from "../../../llm/types.js";
|
||||
import { estimateUsageCost, resolveModelCostConfig } from "../../../utils/usage-format.js";
|
||||
import { projectAgentRunAttemptTerminal } from "../../agent-run-terminal-outcome.js";
|
||||
import type { AuthProfileStore } from "../../auth-profiles.js";
|
||||
import type { NormalizedUsage, UsageLike } from "../../usage.js";
|
||||
@@ -81,6 +82,20 @@ export function prepareEmbeddedRunTerminal(input: {
|
||||
const finalAssistantStopReason = (terminalAssistant?.stopReason ?? "").trim().toLowerCase();
|
||||
const terminalAssistantCanOwnFinalText =
|
||||
finalAssistantStopReason !== "error" && finalAssistantStopReason !== "aborted";
|
||||
// Total-only usage (lastTurnTotal override) carries no token split, so cost
|
||||
// math uses the accumulated input/output/cache fields untouched by it.
|
||||
const costUsd = estimateUsageCost({
|
||||
usage: usageMeta.usage,
|
||||
cost: resolveModelCostConfig({
|
||||
provider: reportedModelRef.provider,
|
||||
model: reportedModelRef.model,
|
||||
config: runParams.config,
|
||||
agentDir: runParams.agentDir,
|
||||
}),
|
||||
});
|
||||
// Attempt normalization already folded every attempt (terminal included)
|
||||
// into the accumulator, so read it directly instead of re-adding the attempt.
|
||||
const runAssistantTurns = input.usageAccumulator.assistantTurns;
|
||||
const agentMeta: EmbeddedAgentMeta = {
|
||||
sessionId: input.sessionIdUsed,
|
||||
sessionFile: input.sessionFileUsed,
|
||||
@@ -99,6 +114,14 @@ export function prepareEmbeddedRunTerminal(input: {
|
||||
? input.contextRecoveryState.autoCompactionCount
|
||||
: undefined,
|
||||
compactionTokensAfter: input.contextRecoveryState.lastCompactionTokensAfter,
|
||||
// Absent attempt engagement (plugin harness routes) intentionally reads as
|
||||
// false so config-enabled-but-unengaged code mode is visible to consumers.
|
||||
codeModeEngaged: attempt.codeModeEngaged === true,
|
||||
...(runAssistantTurns > 0 ? { assistantTurns: runAssistantTurns } : {}),
|
||||
...(input.usageAccumulator.bridgeCalls
|
||||
? { bridgeCalls: { ...input.usageAccumulator.bridgeCalls } }
|
||||
: {}),
|
||||
...(costUsd !== undefined ? { costUsd } : {}),
|
||||
};
|
||||
const attemptFinalText = attempt.assistantTexts
|
||||
.toReversed()
|
||||
|
||||
@@ -304,6 +304,20 @@ export type EmbeddedRunAttemptResult = {
|
||||
clientToolCalls?: Array<{ name: string; params: Record<string, unknown> }>;
|
||||
/** True when sessions_yield tool was called during this attempt. */
|
||||
yieldDetected?: boolean;
|
||||
/**
|
||||
* True when code mode owned this attempt's model tool surface. Absent means
|
||||
* the harness did not report engagement (treated as not engaged), which is
|
||||
* how config-enabled code mode stays visible as a no-op on harness routes.
|
||||
*/
|
||||
codeModeEngaged?: boolean;
|
||||
/** Completed assistant round trips observed during this attempt. */
|
||||
assistantTurns?: number;
|
||||
/** Inner bridge call counts from this attempt's tool-search/code-mode catalog. */
|
||||
bridgeCalls?: {
|
||||
search: number;
|
||||
describe: number;
|
||||
call: number;
|
||||
};
|
||||
replayMetadata: EmbeddedRunReplayMetadata;
|
||||
/**
|
||||
* Replay metadata for this attempt before prior session state is accumulated.
|
||||
|
||||
@@ -82,6 +82,27 @@ export type EmbeddedAgentMeta = {
|
||||
total?: number;
|
||||
};
|
||||
contextBudgetStatus?: SessionContextBudgetStatus;
|
||||
/**
|
||||
* True when code mode owned the model tool surface for this run. Config
|
||||
* alone is not proof: the "auto" tier engages per model capability, raw
|
||||
* model runs and plugin-harness surfaces can decline engagement, and the
|
||||
* shell tool is also named `exec`, so consumers must read this flag
|
||||
* instead of config or tool names.
|
||||
*/
|
||||
codeModeEngaged?: boolean;
|
||||
/** Completed assistant/provider round trips accumulated across run attempts. */
|
||||
assistantTurns?: number;
|
||||
/**
|
||||
* Code-mode/tool-search inner bridge calls for the run's catalog. These are
|
||||
* invisible to the provider; `toolSummary.calls` stays the outer count.
|
||||
*/
|
||||
bridgeCalls?: {
|
||||
search: number;
|
||||
describe: number;
|
||||
call: number;
|
||||
};
|
||||
/** Estimated USD cost of the run's accumulated usage. Omitted when the model has no cost data. */
|
||||
costUsd?: number;
|
||||
};
|
||||
|
||||
export type TraceAttempt = {
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
createUsageAccumulator,
|
||||
mergeAttemptRunStatsIntoAccumulator,
|
||||
mergeUsageIntoAccumulator,
|
||||
toNormalizedUsage,
|
||||
} from "./usage-accumulator.js";
|
||||
@@ -79,6 +80,34 @@ describe("usage-accumulator", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("mergeAttemptRunStatsIntoAccumulator", () => {
|
||||
it("accumulates turns and bridge calls across retry/fallback attempts", () => {
|
||||
const acc = createUsageAccumulator();
|
||||
|
||||
// First attempt makes bridge calls, then a retry/fallback attempt runs.
|
||||
mergeAttemptRunStatsIntoAccumulator(acc, {
|
||||
assistantTurns: 2,
|
||||
bridgeCalls: { search: 1, describe: 2, call: 3 },
|
||||
});
|
||||
mergeAttemptRunStatsIntoAccumulator(acc, {
|
||||
assistantTurns: 1,
|
||||
bridgeCalls: { search: 0, describe: 1, call: 4 },
|
||||
});
|
||||
|
||||
expect(acc.assistantTurns).toBe(3);
|
||||
expect(acc.bridgeCalls).toEqual({ search: 1, describe: 3, call: 7 });
|
||||
});
|
||||
|
||||
it("keeps bridgeCalls absent for catalog-less attempts", () => {
|
||||
const acc = createUsageAccumulator();
|
||||
|
||||
mergeAttemptRunStatsIntoAccumulator(acc, { assistantTurns: 1 });
|
||||
|
||||
expect(acc.assistantTurns).toBe(1);
|
||||
expect(acc.bridgeCalls).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("toNormalizedUsage", () => {
|
||||
it("returns undefined for an empty accumulator", () => {
|
||||
expect(toNormalizedUsage(createUsageAccumulator())).toBeUndefined();
|
||||
|
||||
@@ -10,6 +10,21 @@ export type UsageAccumulator = {
|
||||
cacheWrite: number;
|
||||
reasoningTokens: number;
|
||||
total: number;
|
||||
/**
|
||||
* Completed assistant round trips across every model attempt of the run.
|
||||
* Kept beside token totals so retried attempts stay counted like their usage.
|
||||
*/
|
||||
assistantTurns: number;
|
||||
/**
|
||||
* Cumulative inner bridge calls across attempts. Present only once an
|
||||
* attempt reported a tool-search/code-mode catalog, so catalog-less runs
|
||||
* omit the field instead of publishing zero sentinels.
|
||||
*/
|
||||
bridgeCalls?: {
|
||||
search: number;
|
||||
describe: number;
|
||||
call: number;
|
||||
};
|
||||
};
|
||||
|
||||
export const createUsageAccumulator = (): UsageAccumulator => ({
|
||||
@@ -19,6 +34,7 @@ export const createUsageAccumulator = (): UsageAccumulator => ({
|
||||
cacheWrite: 0,
|
||||
reasoningTokens: 0,
|
||||
total: 0,
|
||||
assistantTurns: 0,
|
||||
});
|
||||
|
||||
type MaybeUsage = NormalizedUsage | undefined;
|
||||
@@ -57,6 +73,29 @@ export const mergeUsageIntoAccumulator = (target: UsageAccumulator, usage: Maybe
|
||||
target.total += callTotal;
|
||||
};
|
||||
|
||||
/**
|
||||
* Folds one attempt's run stats into the accumulator. Attempt cleanup clears
|
||||
* the per-attempt tool-search catalog, so retries would otherwise discard
|
||||
* earlier bridge counts and undercount the documented cumulative run totals.
|
||||
*/
|
||||
export const mergeAttemptRunStatsIntoAccumulator = (
|
||||
target: UsageAccumulator,
|
||||
attempt: {
|
||||
assistantTurns?: number;
|
||||
bridgeCalls?: { search: number; describe: number; call: number };
|
||||
},
|
||||
) => {
|
||||
target.assistantTurns += attempt.assistantTurns ?? 0;
|
||||
if (!attempt.bridgeCalls) {
|
||||
return;
|
||||
}
|
||||
const bridgeCalls = target.bridgeCalls ?? { search: 0, describe: 0, call: 0 };
|
||||
bridgeCalls.search += attempt.bridgeCalls.search;
|
||||
bridgeCalls.describe += attempt.bridgeCalls.describe;
|
||||
bridgeCalls.call += attempt.bridgeCalls.call;
|
||||
target.bridgeCalls = bridgeCalls;
|
||||
};
|
||||
|
||||
export const toNormalizedUsage = (usage: UsageAccumulator): NormalizedUsage | undefined => {
|
||||
const hasUsage =
|
||||
usage.input > 0 ||
|
||||
|
||||
@@ -1391,6 +1391,35 @@ describe("handleMessageUpdate commentary phase", () => {
|
||||
});
|
||||
|
||||
describe("handleMessageEnd", () => {
|
||||
it.each([
|
||||
{
|
||||
name: "counts a completed provider assistant message",
|
||||
message: { role: "assistant", content: [{ type: "text", text: "Done." }] },
|
||||
expected: 1,
|
||||
},
|
||||
{
|
||||
name: "ignores transcript-only mirrored assistant messages",
|
||||
message: {
|
||||
role: "assistant",
|
||||
provider: "openclaw",
|
||||
model: "delivery-mirror",
|
||||
content: [{ type: "text", text: "Done." }],
|
||||
},
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "ignores non-assistant messages",
|
||||
message: { role: "user", content: [{ type: "text", text: "hi" }] },
|
||||
expected: 0,
|
||||
},
|
||||
])("$name for assistantTurnCount", ({ message, expected }) => {
|
||||
const ctx = createMessageEndContext({ state: { assistantTurnCount: 0 } });
|
||||
|
||||
void endMessage(ctx, { message });
|
||||
|
||||
expect(ctx.state.assistantTurnCount).toBe(expected);
|
||||
});
|
||||
|
||||
it("keeps duplicate-reply diagnostics free of lone surrogates", () => {
|
||||
const text = `${"a".repeat(49)}😀tail`;
|
||||
const ctx = createMessageEndContext({
|
||||
|
||||
@@ -1153,6 +1153,9 @@ export function handleMessageEnd(
|
||||
return;
|
||||
}
|
||||
|
||||
// Transcript-only messages never reach the provider, so this counts exactly
|
||||
// the completed model round trips consumers see as `assistantTurns`.
|
||||
ctx.state.assistantTurnCount += 1;
|
||||
const assistantMessage = preservePendingAssistantUsage(msg, ctx.state.pendingAssistantUsage);
|
||||
const assistantPhase = resolveAssistantMessagePhase(assistantMessage);
|
||||
const suppressVisibleAssistantOutput = shouldSuppressAssistantVisibleOutput(assistantMessage);
|
||||
|
||||
@@ -85,6 +85,11 @@ export type EmbeddedAgentSubscribeState = {
|
||||
itemActiveIds: Set<string>;
|
||||
itemStartedCount: number;
|
||||
itemCompletedCount: number;
|
||||
/**
|
||||
* Completed assistant round trips in this attempt. Survives compaction-retry
|
||||
* presentation resets, matching how usage totals keep counting model calls.
|
||||
*/
|
||||
assistantTurnCount: number;
|
||||
lastToolError?: ToolErrorSummary;
|
||||
latestMcpAppChannelView?: McpAppChannelView;
|
||||
|
||||
|
||||
@@ -178,6 +178,7 @@ export function subscribeEmbeddedAgentSession(params: SubscribeEmbeddedAgentSess
|
||||
itemActiveIds: new Set(),
|
||||
itemStartedCount: 0,
|
||||
itemCompletedCount: 0,
|
||||
assistantTurnCount: 0,
|
||||
lastToolError: undefined,
|
||||
blockReplyBreak: params.blockReplyBreak ?? "text_end",
|
||||
reasoningMode,
|
||||
@@ -1531,6 +1532,7 @@ export function subscribeEmbeddedAgentSession(params: SubscribeEmbeddedAgentSess
|
||||
getLastAssistantUsage,
|
||||
getCompactionCount: () => compactionCount,
|
||||
getLastCompactionTokensAfter: () => state.lastCompactionTokensAfter,
|
||||
getAssistantTurnCount: () => state.assistantTurnCount,
|
||||
waitForPendingEvents: () => state.pendingEventChain ?? Promise.resolve(),
|
||||
getItemLifecycle: () => ({
|
||||
startedCount: state.itemStartedCount,
|
||||
|
||||
@@ -51,6 +51,10 @@ export type AgentExecEnvelope = {
|
||||
final: string;
|
||||
payloads: AgentExecPayload[];
|
||||
usage?: NonNullable<NonNullable<EmbeddedAgentRunMeta["agentMeta"]>["usage"]>;
|
||||
costUsd?: number;
|
||||
codeModeEngaged?: boolean;
|
||||
assistantTurns?: number;
|
||||
bridgeCalls?: NonNullable<NonNullable<EmbeddedAgentRunMeta["agentMeta"]>["bridgeCalls"]>;
|
||||
model: string | null;
|
||||
provider: string | null;
|
||||
sessionId: string;
|
||||
@@ -227,6 +231,14 @@ export function classifyAgentExecResult(
|
||||
final: finalTextFromResult(result, payloads, !hasErrorPayload),
|
||||
payloads,
|
||||
...(agentMeta?.usage ? { usage: agentMeta.usage } : {}),
|
||||
...(agentMeta?.costUsd !== undefined ? { costUsd: agentMeta.costUsd } : {}),
|
||||
...(agentMeta?.codeModeEngaged !== undefined
|
||||
? { codeModeEngaged: agentMeta.codeModeEngaged }
|
||||
: {}),
|
||||
...(agentMeta?.assistantTurns !== undefined
|
||||
? { assistantTurns: agentMeta.assistantTurns }
|
||||
: {}),
|
||||
...(agentMeta?.bridgeCalls ? { bridgeCalls: agentMeta.bridgeCalls } : {}),
|
||||
model: agentMeta?.model ?? null,
|
||||
provider: agentMeta?.provider ?? null,
|
||||
sessionId: agentMeta?.sessionId ?? "",
|
||||
|
||||
Reference in New Issue
Block a user