From fa0fcef9f624f156e723ce10eeaf06fbfa3a7231 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 7 Aug 2026 23:06:30 -0700 Subject: [PATCH] fix(sessions): require provenance for fresh context-size facts (#120497) --- .../src/app-server/startup-binding.test.ts | 25 +- .../codex/src/app-server/startup-binding.ts | 3 +- .../src/harness/compaction/compaction.test.ts | 32 ++ .../src/harness/compaction/compaction.ts | 21 ++ .../src/schema/sessions-checkpoint.test.ts | 26 ++ .../gateway-protocol/src/schema/sessions.ts | 1 + .../command/attempt-execution.cli.test.ts | 68 +++- src/agents/command/attempt-execution.ts | 38 ++- src/agents/command/cli-compaction.test.ts | 16 +- src/agents/command/cli-compaction.ts | 6 +- src/agents/command/session-store.ts | 13 +- .../run.attempt-normalization.direct.test.ts | 60 +++- .../run/attempt-normalization.ts | 5 +- .../embedded-agent-runner/run/helpers.test.ts | 28 ++ .../embedded-agent-runner/run/helpers.ts | 21 +- .../run/session-bootstrap.ts | 3 +- .../usage-reporting.test-support.ts | 34 +- .../agent-session-loop-correctness.test.ts | 54 +++ src/agents/subagent-announce-output.ts | 5 +- src/agents/subagent-list.test.ts | 2 + src/agents/tools/goal-tools.test.ts | 1 + ...nner-memory.preflight-stale-tokens.test.ts | 2 + .../reply/agent-runner-memory.test.ts | 322 ++++++++++++++++++ src/auto-reply/reply/agent-runner-memory.ts | 58 +++- .../reply/agent-runner-result-accounting.ts | 1 - .../reply/agent-runner-session-reset.ts | 1 + .../agent-runner.runreplyagent.e2e.test.ts | 10 +- .../reply/commands-context-report.test.ts | 2 + .../reply/commands-context-report.ts | 4 +- src/auto-reply/reply/commands-goal.test.ts | 25 +- src/auto-reply/reply/commands-status.test.ts | 3 + .../reply/session-parent-fork-prepare.ts | 1 + src/auto-reply/reply/session-updates.ts | 4 +- src/auto-reply/reply/session-usage.ts | 18 +- src/auto-reply/reply/session.test.ts | 19 +- src/auto-reply/reply/session.ts | 2 + src/auto-reply/status.test.ts | 22 +- src/commands/sessions-table.ts | 2 + src/commands/sessions.test.ts | 2 + src/commands/sessions.ts | 9 +- src/commands/status.test.ts | 10 +- src/config/sessions/goals.test.ts | 13 +- src/config/sessions/goals.ts | 8 +- .../session-accessor.conformance.test.ts | 6 + .../session-accessor.parent-fork.test.ts | 49 +++ .../session-accessor.sqlite-checkpoint.ts | 12 +- .../session-accessor.sqlite-message-cut.ts | 1 + .../session-accessor.sqlite-parent-fork.ts | 25 +- .../session-accessor.sqlite-parent-session.ts | 3 + ...ession-accessor.sqlite-transcript-write.ts | 1 + src/config/sessions/types.ts | 17 +- src/cron/isolated-agent/run-finalize.ts | 3 + .../server-methods/sessions-compact.ts | 3 + .../server.sessions.compaction.test.ts | 3 +- src/gateway/server.sessions.create.test.ts | 2 + src/gateway/session-compaction-checkpoints.ts | 15 +- src/gateway/session-create-fork-entry.ts | 1 + src/gateway/session-reset-service.ts | 2 + .../session-transcript-derived-readers.ts | 21 +- src/gateway/session-utils-creators.test.ts | 1 + src/gateway/session-utils-row.ts | 2 + src/gateway/session-utils.fs.test.ts | 58 ++++ src/gateway/session-utils.fs.ts | 20 +- src/gateway/session-utils.search.test.ts | 2 + src/gateway/session-utils.test.ts | 109 ++++++ src/plugins/session-entry-slot-keys.ts | 1 + src/shared/subagents-format.test.ts | 32 +- src/shared/subagents-format.ts | 9 +- src/status/status-message.test.ts | 4 + src/status/status-message.ts | 10 +- src/status/summary.ts | 7 +- 71 files changed, 1253 insertions(+), 166 deletions(-) create mode 100644 packages/gateway-protocol/src/schema/sessions-checkpoint.test.ts diff --git a/extensions/codex/src/app-server/startup-binding.test.ts b/extensions/codex/src/app-server/startup-binding.test.ts index 98b6c82bf3a9..e90b61f98697 100644 --- a/extensions/codex/src/app-server/startup-binding.test.ts +++ b/extensions/codex/src/app-server/startup-binding.test.ts @@ -53,6 +53,11 @@ describe("Codex app-server startup binding", () => { JSON.stringify({ "agent:main:session-1": { sessionFile, + ...(typeof record.totalTokens === "number" && + record.totalTokensFresh !== false && + !Object.hasOwn(record, "totalTokensVersion") + ? { totalTokensFresh: true, totalTokensVersion: 1 } + : {}), ...record, }, }), @@ -741,15 +746,25 @@ describe("Codex app-server startup binding", () => { expect(savedBinding?.threadId).toBe("thread-existing"); }); - it("ignores stale session token totals for native rollout rotation", async () => { + it.each([ + { + name: "stale", + record: { totalTokens: 300_000, totalTokensFresh: false }, + }, + { + name: "unversioned", + record: { + totalTokens: 300_000, + totalTokensFresh: true, + totalTokensVersion: undefined, + }, + }, + ])("ignores $name session token totals for native rollout rotation", async ({ record }) => { const sessionFile = path.join(tempDir, "session.jsonl"); const workspaceDir = path.join(tempDir, "workspace"); const agentDir = path.join(tempDir, "agent"); await writeExistingBinding(sessionFile, workspaceDir, { dynamicToolsFingerprint: "[]" }); - await writeSessionRecord(sessionFile, { - totalTokens: 300_000, - totalTokensFresh: false, - }); + await writeSessionRecord(sessionFile, record); const rolloutDir = path.join(agentDir, "codex-home", "sessions"); await fs.mkdir(rolloutDir, { recursive: true }); await fs.writeFile( diff --git a/extensions/codex/src/app-server/startup-binding.ts b/extensions/codex/src/app-server/startup-binding.ts index daec59ee20a8..c0f71db82611 100644 --- a/extensions/codex/src/app-server/startup-binding.ts +++ b/extensions/codex/src/app-server/startup-binding.ts @@ -501,7 +501,8 @@ export async function rotateOversizedCodexAppServerStartupBinding(params: { projectedTurnTokens: params.projectedTurnTokens, }); const sessionTokens = - sessionRecord?.totalTokensFresh !== false && + sessionRecord?.totalTokensFresh === true && + sessionRecord.totalTokensVersion === 1 && typeof sessionRecord?.totalTokens === "number" && Number.isFinite(sessionRecord.totalTokens) ? sessionRecord.totalTokens diff --git a/packages/agent-core/src/harness/compaction/compaction.test.ts b/packages/agent-core/src/harness/compaction/compaction.test.ts index 2e0b0c04a7a1..26d51d05764a 100644 --- a/packages/agent-core/src/harness/compaction/compaction.test.ts +++ b/packages/agent-core/src/harness/compaction/compaction.test.ts @@ -178,6 +178,38 @@ describe("calculateContextTokens", () => { expect(estimate.lastUsageIndex).toBe(0); }); + it("does not scan past a zero unavailable context marker", () => { + const messages: AgentMessage[] = [ + createAssistant("old cumulative turn", createUsage(950), 0), + { + ...createAssistant("usage unavailable", createUsage(0), 1), + usage: { + ...createUsage(0), + contextUsage: { state: "unavailable" }, + }, + }, + ]; + const estimate = estimateContextTokens(messages); + + expect(estimate.usageTokens).toBe(0); + expect(estimate.lastUsageIndex).toBeNull(); + expect(estimate.tokens).toBeGreaterThan(0); + expect(estimate.tokens).toBeLessThan(950); + expect(getLastAssistantUsage(messages.map(createMessageEntry))).toBeUndefined(); + }); + + it("treats legacy CLI usage without context provenance as a barrier", () => { + const legacyCli = { + ...createAssistant("legacy CLI", createUsage(950), 1), + api: "cli", + usage: { ...createUsage(950), contextUsage: undefined }, + }; + const messages = [createAssistant("old", createUsage(900), 0), legacyCli]; + + expect(estimateContextTokens(messages).usageTokens).toBe(0); + expect(getLastAssistantUsage(messages.map(createMessageEntry))).toBeUndefined(); + }); + it("ignores an all-zero terminal usage block", () => { const validUsage = createUsage(20); const messages: AgentMessage[] = [ diff --git a/packages/agent-core/src/harness/compaction/compaction.ts b/packages/agent-core/src/harness/compaction/compaction.ts index 4d2ef0dd7659..3108acdb353f 100644 --- a/packages/agent-core/src/harness/compaction/compaction.ts +++ b/packages/agent-core/src/harness/compaction/compaction.ts @@ -130,10 +130,26 @@ function getAssistantUsage(msg: AgentMessage): Usage | undefined { return undefined; } +function isUnavailableContextBarrier(message: AgentMessage): boolean { + if (message.role !== "assistant") { + return false; + } + if (message.api === "cli" && message.usage.contextUsage === undefined) { + return true; + } + if (message.usage.contextUsage?.state !== "unavailable") { + return false; + } + return calculateContextTokens(message.usage) === 0; +} + /** Return usage from the last valid assistant message in session entries. */ export function getLastAssistantUsage(entries: SessionTreeEntry[]): Usage | undefined { for (const entry of entries.toReversed()) { if (entry.type === "message") { + if (isUnavailableContextBarrier(entry.message)) { + return undefined; + } const usage = getAssistantUsage(entry.message); if (usage) { return usage; @@ -163,6 +179,11 @@ function getLastAssistantUsageInfo( if (!message) { continue; } + if (isUnavailableContextBarrier(message)) { + // Synthetic CLI markers invalidate older usage without contributing a + // replacement. Estimate the whole transcript instead of scanning past it. + return undefined; + } const usage = getAssistantUsage(message); if (usage && usage.contextUsage?.state !== "unavailable") { return { usage, index: i }; diff --git a/packages/gateway-protocol/src/schema/sessions-checkpoint.test.ts b/packages/gateway-protocol/src/schema/sessions-checkpoint.test.ts new file mode 100644 index 000000000000..f7d4816bad94 --- /dev/null +++ b/packages/gateway-protocol/src/schema/sessions-checkpoint.test.ts @@ -0,0 +1,26 @@ +import { Value } from "typebox/value"; +import { describe, expect, it } from "vitest"; +import { SessionCompactionCheckpointSchema } from "./sessions.js"; + +describe("SessionCompactionCheckpointSchema", () => { + const checkpoint = { + checkpointId: "checkpoint-1", + sessionKey: "agent:main:main", + sessionId: "session-1", + createdAt: 1, + reason: "manual", + tokensBefore: 100, + tokensAfter: 40, + tokensVersion: 1, + preCompaction: { sessionId: "session-1", entryId: "before" }, + postCompaction: { sessionId: "session-1", entryId: "after" }, + } as const; + + it("round-trips v1 token provenance and rejects unknown versions", () => { + expect(Value.Check(SessionCompactionCheckpointSchema, checkpoint)).toBe(true); + expect(Value.Decode(SessionCompactionCheckpointSchema, checkpoint)).toEqual(checkpoint); + expect( + Value.Check(SessionCompactionCheckpointSchema, { ...checkpoint, tokensVersion: 2 }), + ).toBe(false); + }); +}); diff --git a/packages/gateway-protocol/src/schema/sessions.ts b/packages/gateway-protocol/src/schema/sessions.ts index 814aefe4e09a..f3f3e6217d4f 100644 --- a/packages/gateway-protocol/src/schema/sessions.ts +++ b/packages/gateway-protocol/src/schema/sessions.ts @@ -155,6 +155,7 @@ export const SessionCompactionCheckpointSchema = closedObject({ reason: SessionCompactionCheckpointReasonSchema, tokensBefore: Type.Optional(Type.Integer({ minimum: 0 })), tokensAfter: Type.Optional(Type.Integer({ minimum: 0 })), + tokensVersion: Type.Optional(Type.Literal(1)), summary: Type.Optional(Type.String()), firstKeptEntryId: Type.Optional(NonEmptyString), preCompaction: SessionCompactionTranscriptReferenceSchema, diff --git a/src/agents/command/attempt-execution.cli.test.ts b/src/agents/command/attempt-execution.cli.test.ts index 77b8efc1c6db..2f3be0c11ff4 100644 --- a/src/agents/command/attempt-execution.cli.test.ts +++ b/src/agents/command/attempt-execution.cli.test.ts @@ -435,6 +435,13 @@ function makeCliResult(text: string): EmbeddedAgentRunResult { cacheWrite: 0, total: 19, }, + lastCallUsage: { + input: 12, + output: 4, + cacheRead: 3, + cacheWrite: 0, + total: 19, + }, }, executionTrace: { winnerProvider: "claude-cli", @@ -469,7 +476,13 @@ async function readSessionMessages(target: TranscriptReadTarget) { .filter((entry) => entry.type === "message") .map( (entry) => - entry.message as { role?: string; content?: unknown; provider?: string; model?: string }, + entry.message as { + role?: string; + content?: unknown; + provider?: string; + model?: string; + usage?: unknown; + }, ); } @@ -1984,9 +1997,15 @@ describe("CLI attempt execution", () => { }); let updatedEntry: SessionEntry | undefined; try { + const result = makeCliResult("hello from cli"); + if (!result.meta.agentMeta) { + throw new Error("expected agent metadata"); + } + result.meta.agentMeta.usage = { input: 12, output: 4, cacheRead: 3, total: 19 }; + result.meta.agentMeta.lastCallUsage = { input: 7, output: 4, cacheRead: 2, total: 13 }; updatedEntry = await persistCliTranscriptEntry({ body: "persist this", - result: makeCliResult("hello from cli"), + result, sessionId: sessionEntry.sessionId, sessionKey, sessionEntry, @@ -2034,6 +2053,13 @@ describe("CLI attempt execution", () => { model: "opus", content: [{ type: "text", text: "hello from cli" }], }); + expectRecordFields(requireRecord(messages[1]?.usage, "assistant usage"), { + input: 7, + output: 4, + cacheRead: 2, + totalTokens: 13, + contextUsage: { state: "available", promptTokens: 9, totalTokens: 13 }, + }); const persisted = readSessionStore(); expect(persisted[sessionKey]).not.toHaveProperty("sessionFile"); @@ -2042,6 +2068,44 @@ describe("CLI attempt execution", () => { expect(sessionStore[sessionKey]?.updatedAt).toBe(persisted[sessionKey]?.updatedAt); }); + it("marks CLI transcript context unavailable when only cumulative usage exists", async () => { + const sessionKey = "agent:main:subagent:cli-cumulative-only"; + const sessionEntry = makeSessionEntry("session-cli-cumulative-only"); + const result = makeCliResult("cumulative reply"); + if (!result.meta.agentMeta) { + throw new Error("expected agent metadata"); + } + result.meta.agentMeta.lastCallUsage = undefined; + + await persistCliTurnTranscript({ + body: "run tools", + result, + sessionId: sessionEntry.sessionId, + sessionKey, + sessionEntry, + storePath, + sessionAgentId: "main", + sessionCwd: tmpDir, + config: {}, + }); + + const messages = await readSessionMessages({ + agentId: "main", + sessionId: sessionEntry.sessionId, + sessionKey, + storePath, + }); + const assistant = requireRecord(messages.at(-1), "assistant message"); + expectRecordFields(requireRecord(assistant.usage, "assistant usage"), { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + contextUsage: { state: "unavailable" }, + }); + }); + it("mirrors only the CLI reply when the shared recorder already persisted the user turn", async () => { const sessionKey = "agent:main:direct:cli-recorder-owned-user"; const sessionFile = path.join(tmpDir, "session-cli-recorder-owned-user.jsonl"); diff --git a/src/agents/command/attempt-execution.ts b/src/agents/command/attempt-execution.ts index 0acdc64bb150..d7d4c5ed47ed 100644 --- a/src/agents/command/attempt-execution.ts +++ b/src/agents/command/attempt-execution.ts @@ -101,6 +101,7 @@ import { } from "../subagent-announce-handoff.js"; import { isRuntimeToolAllowed, isToolAllowedByPolicies } from "../tool-policy-match.js"; import { DEFAULT_MAX_LIVE_TOOL_RESULT_CHARS } from "../tool-result-limits.js"; +import type { ContextUsage } from "../usage.js"; import { buildClaudeCliFallbackContextPrelude, claudeCliSessionTranscriptHasContent, @@ -158,6 +159,35 @@ const ACP_TRANSCRIPT_USAGE = { total: 0, }, } as const; +const CLI_TRANSCRIPT_UNAVAILABLE_USAGE = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + total: 0, + contextUsage: { state: "unavailable" }, +} as const; + +function resolveCliTranscriptUsage(usage: TranscriptUsage | undefined): TranscriptUsage { + if (!usage) { + return CLI_TRANSCRIPT_UNAVAILABLE_USAGE; + } + if (usage.contextUsage) { + return usage; + } + const promptTokens = (usage.input ?? 0) + (usage.cacheRead ?? 0) + (usage.cacheWrite ?? 0); + return { + ...usage, + contextUsage: + promptTokens > 0 + ? { + state: "available", + promptTokens, + totalTokens: promptTokens + (usage.output ?? 0), + } + : { state: "unavailable" }, + }; +} function shouldSuppressEmbeddedLiveStreamOutput(params: { opts: AgentCommandOpts }): boolean { return params.opts.sessionEffects === "internal" && params.opts.deliver !== true; } @@ -168,6 +198,7 @@ type TranscriptUsage = { cacheRead?: number; cacheWrite?: number; total?: number; + contextUsage?: ContextUsage; }; type PersistTextTurnTranscriptParams = { @@ -292,13 +323,14 @@ function resolveTranscriptUsage(usage: PersistTextTurnTranscriptParams["assistan if (!usage) { return ACP_TRANSCRIPT_USAGE; } - return buildUsageWithNoCost({ + const resolved = buildUsageWithNoCost({ input: usage.input, output: usage.output, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite, totalTokens: usage.total, }); + return usage.contextUsage ? { ...resolved, contextUsage: usage.contextUsage } : resolved; } async function persistTextTurnTranscript( @@ -478,7 +510,9 @@ export async function persistCliTurnTranscript(params: { api: "cli", provider, model, - usage: params.result.meta.agentMeta?.usage, + // The marker is terminal for fallback scans: without it, readers could + // skip this turn and revive an older cumulative usage record as fresh. + usage: resolveCliTranscriptUsage(params.result.meta.agentMeta?.lastCallUsage), }, skipAssistantTurn: params.skipAssistantTurn, }); diff --git a/src/agents/command/cli-compaction.test.ts b/src/agents/command/cli-compaction.test.ts index e9110fec1588..019724ae11e1 100644 --- a/src/agents/command/cli-compaction.test.ts +++ b/src/agents/command/cli-compaction.test.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { CURRENT_SESSION_VERSION } from "openclaw/plugin-sdk/agent-sessions"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { replaceSessionEntry } from "../../config/sessions/session-accessor.js"; -import type { SessionEntry } from "../../config/sessions/types.js"; +import { SESSION_TOTAL_TOKENS_VERSION, type SessionEntry } from "../../config/sessions/types.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import type { ContextEngine } from "../../context-engine/types.js"; import { createEmptyPluginRegistry } from "../../plugins/registry-empty.js"; @@ -139,6 +139,7 @@ async function prepareCompactionScenario(params: { contextTokens: 1_000, totalTokens: 950, totalTokensFresh: true, + totalTokensVersion: SESSION_TOTAL_TOKENS_VERSION, ...params.sessionEntry, }; const sessionStore: Record = { [sessionKey]: sessionEntry }; @@ -240,6 +241,19 @@ describe("runCliTurnCompactionLifecycle", () => { await fs.rm(tmpDir, { recursive: true, force: true }); }); + it("ignores an unversioned fresh total on the first upgraded turn", async () => { + const scenario = await prepareCompactionScenario({ + suffix: "legacy-unversioned-total", + tmpDir, + sessionEntry: { totalTokens: 950, totalTokensFresh: true, totalTokensVersion: undefined }, + }); + + const updatedEntry = await scenario.run(); + + expect(scenario.compactCalls).toEqual([]); + expect(updatedEntry).toBe(scenario.sessionEntry); + }); + it("accepts no compactable entries only from a successful compaction result", async () => { let result = { ok: true, compacted: false, reason: "no real conversation messages" }; const scenario = await prepareCompactionScenario({ diff --git a/src/agents/command/cli-compaction.ts b/src/agents/command/cli-compaction.ts index e6879d637d22..4751590bb17b 100644 --- a/src/agents/command/cli-compaction.ts +++ b/src/agents/command/cli-compaction.ts @@ -5,7 +5,7 @@ import type { SessionTranscriptRuntimeTarget } from "../../config/sessions/sessi * This module decides when CLI-backed sessions need context compaction, chooses * native harness or context-engine compaction, and records resulting session state. */ -import type { SessionEntry } from "../../config/sessions/types.js"; +import { resolveFreshSessionTotalTokens, type SessionEntry } from "../../config/sessions/types.js"; import type { AgentCompactionMode } from "../../config/types.agent-defaults.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { buildGenericCliContextEngineHostSupport } from "../../context-engine/host-compat.js"; @@ -188,9 +188,7 @@ function getSessionBranchMessages(sessionManager: SessionManagerLike): AgentMess } function resolveSessionTokenSnapshot(sessionEntry: SessionEntry | undefined): number | undefined { - return resolvePositiveInteger( - sessionEntry?.totalTokensFresh === false ? undefined : sessionEntry?.totalTokens, - ); + return resolvePositiveInteger(resolveFreshSessionTotalTokens(sessionEntry)); } function isNativeHarnessCompactionSession( diff --git a/src/agents/command/session-store.ts b/src/agents/command/session-store.ts index becbbe147dea..b0abbffbe5f4 100644 --- a/src/agents/command/session-store.ts +++ b/src/agents/command/session-store.ts @@ -2,7 +2,11 @@ * Updates persisted session metadata after agent command runs. */ import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; -import { setSessionRuntimeModel, type SessionEntry } from "../../config/sessions.js"; +import { + SESSION_TOTAL_TOKENS_VERSION, + setSessionRuntimeModel, + type SessionEntry, +} from "../../config/sessions.js"; import { patchSessionEntry } from "../../config/sessions/session-accessor.js"; import { projectSessionSnapshotChanges } from "../../config/sessions/session-snapshot-merge.js"; import { resolveMaintenanceConfigFromInput } from "../../config/sessions/store-maintenance.js"; @@ -226,6 +230,7 @@ export async function updateSessionStoreAfterAgentRun(params: { if (useCompactionSnapshot) { next.totalTokens = compactionTokensAfter; next.totalTokensFresh = true; + next.totalTokensVersion = SESSION_TOTAL_TOKENS_VERSION; next.inputTokens = undefined; next.outputTokens = undefined; next.cacheRead = undefined; @@ -234,9 +239,11 @@ export async function updateSessionStoreAfterAgentRun(params: { } else if (hasUsageTotalTokens) { next.totalTokens = totalTokens; next.totalTokensFresh = true; + next.totalTokensVersion = SESSION_TOTAL_TOKENS_VERSION; } else { next.totalTokens = undefined; next.totalTokensFresh = false; + next.totalTokensVersion = undefined; } if (!useCompactionSnapshot) { next.cacheRead = usage.cacheRead ?? 0; @@ -251,6 +258,7 @@ export async function updateSessionStoreAfterAgentRun(params: { } else if (compactionTokensAfter !== undefined && !preserveUserFacingRunState) { next.totalTokens = compactionTokensAfter; next.totalTokensFresh = true; + next.totalTokensVersion = SESSION_TOTAL_TOKENS_VERSION; next.inputTokens = undefined; next.outputTokens = undefined; next.cacheRead = undefined; @@ -264,6 +272,7 @@ export async function updateSessionStoreAfterAgentRun(params: { ) { next.totalTokens = entry.totalTokens; next.totalTokensFresh = false; + next.totalTokensVersion = undefined; } if (compactionsThisRun > 0 && !preserveUserFacingRunState) { next.compactionCount = (entry.compactionCount ?? 0) + compactionsThisRun; @@ -514,12 +523,14 @@ export async function recordCliCompactionInStore(params: { if (tokensAfterCompaction !== undefined) { next.totalTokens = Math.floor(tokensAfterCompaction); next.totalTokensFresh = true; + next.totalTokensVersion = SESSION_TOTAL_TOKENS_VERSION; next.inputTokens = undefined; next.outputTokens = undefined; next.cacheRead = undefined; next.cacheWrite = undefined; } else { next.totalTokensFresh = false; + next.totalTokensVersion = undefined; next.inputTokens = undefined; next.outputTokens = undefined; next.cacheRead = undefined; diff --git a/src/agents/embedded-agent-runner/run.attempt-normalization.direct.test.ts b/src/agents/embedded-agent-runner/run.attempt-normalization.direct.test.ts index ddf047cacdeb..42684f10b416 100644 --- a/src/agents/embedded-agent-runner/run.attempt-normalization.direct.test.ts +++ b/src/agents/embedded-agent-runner/run.attempt-normalization.direct.test.ts @@ -178,7 +178,9 @@ describe("normalizeEmbeddedRunAttempt", () => { }); attempt.toolMetas = [{ toolName: "read", isError: false }]; - const result = await normalizeEmbeddedRunAttempt(makeNormalizationInput(attempt, state)); + const input = makeNormalizationInput(attempt, state); + input.lastRunPromptUsage = { input: 42_000, output: 1_000, total: 43_000 }; + const result = await normalizeEmbeddedRunAttempt(input); expect(result.action).toBe("retry"); if (result.action !== "retry") { @@ -238,4 +240,60 @@ describe("normalizeEmbeddedRunAttempt", () => { } expect(clean.replayState).toEqual({ replayInvalid: true, hadPotentialSideEffects: true }); }); + + it("does not promote historical CLI usage without context provenance", async () => { + const state = makePromptState(); + const legacyAssistant = { + role: "assistant", + api: "cli", + provider: "openai", + model: "gpt-5.6-luna", + content: [{ type: "text", text: "legacy reply" }], + usage: { input: 128_814, output: 3_000, cacheRead: 992_953, totalTokens: 1_124_767 }, + stopReason: "error", + timestamp: 1, + }; + const attempt = makeAttempt(); + attempt.messagesSnapshot = [legacyAssistant] as never; + attempt.lastAssistant = legacyAssistant as never; + + const result = await normalizeEmbeddedRunAttempt(makeNormalizationInput(attempt, state)); + + expect(result.action).toBe("proceed"); + if (result.action !== "proceed") { + throw new Error(`expected proceed, got ${result.action}`); + } + expect(result.lastRunPromptUsage).toEqual({ contextUsage: { state: "unavailable" } }); + }); + + it("keeps the unavailable sentinel across a retry instead of reviving prior usage", async () => { + const state = makePromptState(); + const legacyAssistant = { + role: "assistant", + api: "cli", + provider: "openai", + model: "gpt-5.6-luna", + content: [{ type: "text", text: "legacy reply" }], + usage: { input: 128_814, output: 3_000, cacheRead: 992_953, totalTokens: 1_124_767 }, + stopReason: "stop", + timestamp: 1, + }; + const attempt = makeAttempt({ + route: "compact_only", + handled: true, + truncatedCount: 0, + }); + attempt.messagesSnapshot = [legacyAssistant] as never; + attempt.lastAssistant = legacyAssistant as never; + const input = makeNormalizationInput(attempt, state); + input.lastRunPromptUsage = { input: 42_000, output: 1_000, total: 43_000 }; + + const result = await normalizeEmbeddedRunAttempt(input); + + expect(result.action).toBe("retry"); + if (result.action !== "retry") { + throw new Error(`expected retry, got ${result.action}`); + } + expect(result.lastRunPromptUsage).toEqual({ contextUsage: { state: "unavailable" } }); + }); }); diff --git a/src/agents/embedded-agent-runner/run/attempt-normalization.ts b/src/agents/embedded-agent-runner/run/attempt-normalization.ts index c19ecda4ca1c..deee26a615fd 100644 --- a/src/agents/embedded-agent-runner/run/attempt-normalization.ts +++ b/src/agents/embedded-agent-runner/run/attempt-normalization.ts @@ -18,6 +18,7 @@ import { resolveRunFailoverDecision } from "./failover-policy.js"; import { buildErrorAgentMeta, isAssistantForModelRef, + normalizeAssistantUsageForContext, resolveActiveErrorContext, resolveLatestCallUsage, } from "./helpers.js"; @@ -158,8 +159,8 @@ export async function normalizeEmbeddedRunAttempt(input: { ]), ) : input.bootstrapPromptWarningSignaturesSeen); - const lastAssistantUsage = normalizeUsage(sessionLastAssistant?.usage as UsageLike); - const currentAttemptAssistantUsage = normalizeUsage(currentAttemptAssistant?.usage as UsageLike); + const lastAssistantUsage = normalizeAssistantUsageForContext(sessionLastAssistant); + const currentAttemptAssistantUsage = normalizeAssistantUsageForContext(currentAttemptAssistant); const promptCacheLastCallUsage = normalizeUsage(attempt.promptCache?.lastCallUsage as UsageLike); const callUsage = resolveLatestCallUsage({ currentAttemptCandidates: [currentAttemptAssistantUsage, promptCacheLastCallUsage], diff --git a/src/agents/embedded-agent-runner/run/helpers.test.ts b/src/agents/embedded-agent-runner/run/helpers.test.ts index 36c8f6fa6589..0bfabd453efa 100644 --- a/src/agents/embedded-agent-runner/run/helpers.test.ts +++ b/src/agents/embedded-agent-runner/run/helpers.test.ts @@ -217,6 +217,17 @@ describe("resolveLatestCallUsage", () => { }); describe("buildUsageAgentMetaFields", () => { + it("selects unavailable over older prompt usage", () => { + const fields = buildUsageAgentMetaFields({ + usageAccumulator: createUsageAccumulator(), + lastAssistantUsage: { contextUsage: { state: "unavailable" } }, + lastRunPromptUsage: { input: 42_000, output: 1_000, total: 43_000 }, + }); + + expect(fields.lastCallUsage).toEqual({ contextUsage: { state: "unavailable" } }); + expect(fields.promptTokens).toBeUndefined(); + }); + it("keeps cumulative usage separate from the latest context snapshot", () => { const usageAccumulator = createUsageAccumulator(); mergeUsageIntoAccumulator(usageAccumulator, { @@ -325,6 +336,23 @@ describe("buildUsageAgentMetaFields", () => { }); describe("buildErrorAgentMeta", () => { + it("does not promote historical CLI usage without context provenance", () => { + const fields = buildErrorAgentMeta({ + sessionId: "session-error", + provider: "openai", + model: "gpt-5.6-luna", + usageAccumulator: createUsageAccumulator(), + lastRunPromptUsage: { input: 42_000, output: 1_000, total: 43_000 }, + lastAssistant: { + api: "cli", + usage: { input: 128_814, output: 3_000, cacheRead: 992_953, totalTokens: 1_124_767 }, + }, + }); + + expect(fields.lastCallUsage).toEqual({ contextUsage: { state: "unavailable" } }); + expect(fields.promptTokens).toBeUndefined(); + }); + it("keeps cumulative usage separate from the latest call on error exits", () => { const usageAccumulator = createUsageAccumulator(); mergeUsageIntoAccumulator(usageAccumulator, { diff --git a/src/agents/embedded-agent-runner/run/helpers.ts b/src/agents/embedded-agent-runner/run/helpers.ts index 79a906fb2339..4bc526032b0c 100644 --- a/src/agents/embedded-agent-runner/run/helpers.ts +++ b/src/agents/embedded-agent-runner/run/helpers.ts @@ -201,6 +201,21 @@ export function resolveLatestCallUsage(params: { }; } +export function normalizeAssistantUsageForContext( + assistant: { api?: string; usage?: unknown } | null | undefined, +): NormalizedUsage | undefined { + if ( + assistant?.api === "cli" && + assistant.usage && + typeof assistant.usage === "object" && + !Array.isArray(assistant.usage) && + (assistant.usage as { contextUsage?: unknown }).contextUsage === undefined + ) { + return { contextUsage: { state: "unavailable" } }; + } + return normalizeUsage(assistant?.usage as UsageSnapshot | undefined); +} + export function buildUsageAgentMetaFields(params: { usageAccumulator: UsageAccumulator; lastAssistantUsage?: UsageSnapshot | null; @@ -214,7 +229,7 @@ export function buildUsageAgentMetaFields(params: { ? params.lastRunPromptUsage : undefined; const promptTokens = deriveContextPromptTokens({ - lastCallUsage: params.lastRunPromptUsage, + lastCallUsage, }); return { usage, @@ -237,11 +252,11 @@ export function buildErrorAgentMeta(params: { contextTokens?: number; usageAccumulator: UsageAccumulator; lastRunPromptUsage: UsageSnapshot | undefined; - lastAssistant?: { usage?: unknown } | null; + lastAssistant?: { api?: string; usage?: unknown } | null; }): EmbeddedAgentMeta { const usageMeta = buildUsageAgentMetaFields({ usageAccumulator: params.usageAccumulator, - lastAssistantUsage: params.lastAssistant?.usage as UsageSnapshot | undefined, + lastAssistantUsage: normalizeAssistantUsageForContext(params.lastAssistant), lastRunPromptUsage: params.lastRunPromptUsage, }); return { diff --git a/src/agents/embedded-agent-runner/run/session-bootstrap.ts b/src/agents/embedded-agent-runner/run/session-bootstrap.ts index 2a7404dda433..18bb39911312 100644 --- a/src/agents/embedded-agent-runner/run/session-bootstrap.ts +++ b/src/agents/embedded-agent-runner/run/session-bootstrap.ts @@ -1,7 +1,7 @@ import path from "node:path"; import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; import { sanitizeForLog } from "../../../../packages/terminal-core/src/ansi.js"; -import { resolveStorePath } from "../../../config/sessions.js"; +import { resolveStorePath, SESSION_TOTAL_TOKENS_VERSION } from "../../../config/sessions.js"; import { parseSqliteSessionFileMarker } from "../../../config/sessions/legacy-sqlite-marker.js"; import { listSessionEntries, @@ -141,6 +141,7 @@ export async function resetNoRealConversationTokenSnapshot(params: { async () => ({ totalTokens: 0, totalTokensFresh: true, + totalTokensVersion: SESSION_TOTAL_TOKENS_VERSION, inputTokens: undefined, outputTokens: undefined, cacheRead: undefined, diff --git a/src/agents/embedded-agent-runner/usage-reporting.test-support.ts b/src/agents/embedded-agent-runner/usage-reporting.test-support.ts index 631bfd8741e8..060eee9a5ff6 100644 --- a/src/agents/embedded-agent-runner/usage-reporting.test-support.ts +++ b/src/agents/embedded-agent-runner/usage-reporting.test-support.ts @@ -274,10 +274,10 @@ describe("runEmbeddedAgent usage reporting", () => { expect(attemptInput.memoryFlushWritePath).toBe("memory/2026-03-10.md"); }); - it("uses current-attempt usage when the persisted assistant snapshot is zeroed", async () => { + it("keeps Anthropic multi-call billing usage separate from the final context snapshot", async () => { mockedRunEmbeddedAttempt.mockResolvedValueOnce( makeAttemptResult({ - assistantTexts: ["Response 1", "Response 2"], + assistantTexts: ["Tool loop complete"], lastAssistant: makeAssistantMessage({ usage: { input: 0, @@ -288,9 +288,18 @@ describe("runEmbeddedAgent usage reporting", () => { } as unknown as AssistantMessage["usage"], }), currentAttemptAssistant: makeAssistantMessage({ - usage: { input: 150, output: 50, total: 200 } as unknown as AssistantMessage["usage"], + api: "anthropic-messages", + provider: "minimax", + model: "Minimax-M3", + usage: { + input: 67_932, + output: 2_000, + cacheRead: 18_944, + totalTokens: 88_876, + } as unknown as AssistantMessage["usage"], }), - attemptUsage: { input: 250, output: 100, total: 350 }, + // Three model calls in one tool loop; this remains cumulative billing data. + attemptUsage: { input: 110_337, output: 4_000, cacheRead: 40_000, total: 154_337 }, }), ); @@ -301,20 +310,21 @@ describe("runEmbeddedAgent usage reporting", () => { workspaceDir: "/tmp/workspace", prompt: "hello", timeoutMs: 30000, - runId: "run-zeroed-persisted-usage", + runId: "run-anthropic-multi-call-usage", }); expect(result.meta.agentMeta?.usage).toMatchObject({ - input: 250, - output: 100, - total: 350, + input: 110_337, + output: 4_000, + cacheRead: 40_000, + total: 154_337, }); expect(result.meta.agentMeta?.lastCallUsage).toMatchObject({ - input: 150, - output: 50, - total: 200, + input: 67_932, + output: 2_000, + cacheRead: 18_944, }); - expect(result.meta.agentMeta?.promptTokens).toBe(150); + expect(result.meta.agentMeta?.promptTokens).toBe(86_876); }); it("reports the resolved model provider when OpenClaw marks the assistant message as the native runtime", async () => { diff --git a/src/agents/sessions/agent-session-loop-correctness.test.ts b/src/agents/sessions/agent-session-loop-correctness.test.ts index 7c1044ad7318..b9df326b9bec 100644 --- a/src/agents/sessions/agent-session-loop-correctness.test.ts +++ b/src/agents/sessions/agent-session-loop-correctness.test.ts @@ -363,6 +363,60 @@ describe("AgentSession loop correctness", () => { ); }); + it("does not pre-prompt compact from usage before a zero unavailable marker", async () => { + const model = { ...testModel, contextWindow: 1_000 }; + const sessionManager = SessionManager.inMemory(); + appendHistory( + sessionManager, + createAssistant(model, [{ type: "text", text: "old cumulative turn" }], "stop", 950), + ); + sessionManager.appendMessage({ role: "user", content: "CLI prompt", timestamp: Date.now() }); + sessionManager.appendMessage({ + ...createAssistant(model, [{ type: "text", text: "usage unavailable" }]), + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + contextUsage: { state: "unavailable" }, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + }); + const settingsManager = SettingsManager.inMemory({ + compaction: { enabled: true, reserveTokens: 100, keepRecentTokens: 20 }, + retry: { enabled: false }, + }); + const compactionEvents: AgentSessionEvent[] = []; + streamMocks.streamSimple.mockImplementation((activeModel: Model) => + createAssistantResultStream( + createAssistant(activeModel, [{ type: "text", text: "complete answer" }], "stop", 20), + ), + ); + const { session } = await createTestSession({ + model, + sessionManager, + settingsManager, + resourceLoader: createResourceLoader(createCompactionHandlers()), + }); + session.subscribe((event) => { + if (event.type === "compaction_start" || event.type === "compaction_end") { + compactionEvents.push(event); + } + }); + + expect(session.messages.at(-1)).toMatchObject({ + role: "assistant", + usage: { contextUsage: { state: "unavailable" } }, + }); + expect(session.getContextUsage()?.tokens).toBeLessThan(900); + await session.prompt("continue after CLI turn"); + + expect(streamMocks.streamSimple).toHaveBeenCalledOnce(); + expect(compactionEvents).toEqual([]); + expect(session.getLastAssistantText()).toBe("complete answer"); + }); + it("skips threshold maintenance when embedded auto-compaction is disabled", async () => { const settingsManager = SettingsManager.inMemory({ compaction: { enabled: false, reserveTokens: 0, keepRecentTokens: 1 }, diff --git a/src/agents/subagent-announce-output.ts b/src/agents/subagent-announce-output.ts index 2341ff97c5ba..c35951270837 100644 --- a/src/agents/subagent-announce-output.ts +++ b/src/agents/subagent-announce-output.ts @@ -7,6 +7,7 @@ import { asFiniteNumber } from "@openclaw/normalization-core/number-coercion"; import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import { isSilentReplyText, SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js"; import type { SessionTranscriptRuntimeTarget } from "../config/sessions/session-accessor.js"; +import { resolveFreshSessionTotalTokens } from "../config/sessions/types.js"; import { isFastTestRuntimeEnv } from "../infra/env.js"; import { formatDurationCompact } from "../infra/format-time/format-duration.js"; import { buildAgentRunTerminalOutcomeFromWaitResult } from "./agent-run-terminal-outcome.js"; @@ -648,7 +649,7 @@ export async function buildCompactAnnounceStatsLine(params: { const hasTokenData = typeof entry?.inputTokens === "number" || typeof entry?.outputTokens === "number" || - typeof entry?.totalTokens === "number"; + resolveFreshSessionTotalTokens(entry) !== undefined; if (hasTokenData) { break; } @@ -663,7 +664,7 @@ export async function buildCompactAnnounceStatsLine(params: { const input = typeof entry?.inputTokens === "number" ? entry.inputTokens : 0; const output = typeof entry?.outputTokens === "number" ? entry.outputTokens : 0; const ioTotal = input + output; - const promptCache = typeof entry?.totalTokens === "number" ? entry.totalTokens : undefined; + const promptCache = resolveFreshSessionTotalTokens(entry); const runtimeMs = typeof params.startedAt === "number" && typeof params.endedAt === "number" ? Math.max(0, params.endedAt - params.startedAt) diff --git a/src/agents/subagent-list.test.ts b/src/agents/subagent-list.test.ts index e82c7fd8b87c..3fd26bfbdd98 100644 --- a/src/agents/subagent-list.test.ts +++ b/src/agents/subagent-list.test.ts @@ -376,6 +376,8 @@ describe("buildSubagentList", () => { inputTokens: 12, outputTokens: 1000, totalTokens: 197000, + totalTokensFresh: true, + totalTokensVersion: 1, model: "opencode/claude-opus-4-6", }, ); diff --git a/src/agents/tools/goal-tools.test.ts b/src/agents/tools/goal-tools.test.ts index c2f55134f7cb..40a156630de8 100644 --- a/src/agents/tools/goal-tools.test.ts +++ b/src/agents/tools/goal-tools.test.ts @@ -56,6 +56,7 @@ describe("goal tools", () => { updatedAt: 1, totalTokens: 125, totalTokensFresh: true, + totalTokensVersion: 1, goal: { schemaVersion: 1, id: "goal-1", diff --git a/src/auto-reply/reply/agent-runner-memory.preflight-stale-tokens.test.ts b/src/auto-reply/reply/agent-runner-memory.preflight-stale-tokens.test.ts index da7e31d5f2eb..8bb14a972e60 100644 --- a/src/auto-reply/reply/agent-runner-memory.preflight-stale-tokens.test.ts +++ b/src/auto-reply/reply/agent-runner-memory.preflight-stale-tokens.test.ts @@ -140,6 +140,7 @@ describe("runPreflightCompactionIfNeeded stale totalTokens gating", () => { updatedAt: Date.now(), totalTokens: 200_000, totalTokensFresh: true, + totalTokensVersion: 1, }; await writeTestSessionStore( path.join(rootDir, "sessions.json"), @@ -201,6 +202,7 @@ describe("runPreflightCompactionIfNeeded stale totalTokens gating", () => { updatedAt: Date.now(), totalTokens: 200_000, totalTokensFresh: true, + totalTokensVersion: 1, }; await writeTestSessionStore(storePath, "main", sessionEntry); diff --git a/src/auto-reply/reply/agent-runner-memory.test.ts b/src/auto-reply/reply/agent-runner-memory.test.ts index 9f6acbf86b2e..c5bcdc328dc4 100644 --- a/src/auto-reply/reply/agent-runner-memory.test.ts +++ b/src/auto-reply/reply/agent-runner-memory.test.ts @@ -385,6 +385,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, }; const sessionStore = { [sessionKey]: sessionEntry }; @@ -501,6 +503,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, }; @@ -585,6 +589,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, }; @@ -620,6 +626,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, thinkingLevel: "ultra", }; const sessionStore = { [sessionKey]: sessionEntry }; @@ -678,6 +686,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, thinkingLevel: "high", }; const sessionStore = { [sessionKey]: sessionEntry }; @@ -712,6 +722,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "catalog-adopted-session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, agentHarnessId: "codex", agentRuntimeOverride: "claude-cli", @@ -769,6 +781,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, }; const sessionStore = { main: sessionEntry }; @@ -832,6 +846,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, }; const visibleErrorPayloads: Array<{ text?: string; isError?: boolean }> = []; @@ -874,6 +890,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, }; const visibleErrorPayloads: Array<{ text?: string; isError?: boolean }> = []; @@ -911,6 +929,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, }; const visibleErrorPayloads: Array<{ text?: string; isError?: boolean }> = []; @@ -949,6 +969,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, }; const visibleErrorPayloads: Array<{ text?: string; isError?: boolean }> = []; @@ -982,6 +1004,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, }; await writeTestSessionStore(storePath, "main", sessionEntry); @@ -1024,6 +1048,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, }; await writeTestSessionStore(storePath, "main", sessionEntry); @@ -1057,6 +1083,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, memoryFlush: { kind: "failed", failureCount: 2 }, }; @@ -1088,6 +1116,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, memoryFlush: { kind: "failed", failureCount: TEST_MAX_FLUSH_FAILURES - 1 }, }; @@ -1140,6 +1170,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, }; await writeTestSessionStore(storePath, "main", sessionEntry); @@ -1175,6 +1207,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, }; await writeTestSessionStore(storePath, "main", sessionEntry); @@ -1215,6 +1249,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, }; @@ -1282,6 +1318,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, agentRuntimeOverride: "codex", }; @@ -1356,6 +1394,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, agentRuntimeOverride: "unsupported-runtime", }; @@ -1394,6 +1434,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, }; @@ -1421,6 +1463,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "incognito-session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, }; @@ -1449,6 +1493,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "rematerialized-session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, }; @@ -1476,6 +1522,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, agentRuntimeOverride: "claude-cli", }; @@ -1506,6 +1554,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, }; @@ -1569,6 +1619,7 @@ describe("runMemoryFlushIfNeeded", () => { updatedAt: Date.now(), totalTokens: 120, totalTokensFresh: true, + totalTokensVersion: 1, agentHarnessId: "openclaw", modelSelectionLocked: true, }; @@ -1663,6 +1714,7 @@ describe("runMemoryFlushIfNeeded", () => { updatedAt: Date.now(), totalTokens: 120, totalTokensFresh: true, + totalTokensVersion: 1, }; await expect( @@ -1710,6 +1762,7 @@ describe("runMemoryFlushIfNeeded", () => { updatedAt: Date.now(), totalTokens: 120, totalTokensFresh: true, + totalTokensVersion: 1, }; await runPreflightCompactionIfNeeded({ @@ -1770,6 +1823,7 @@ describe("runMemoryFlushIfNeeded", () => { updatedAt: Date.now(), totalTokens: 120, totalTokensFresh: true, + totalTokensVersion: 1, }; const sessionStore = { "agent:main:telegram:group:redacted": sessionEntry }; @@ -1822,6 +1876,7 @@ describe("runMemoryFlushIfNeeded", () => { updatedAt: Date.now(), totalTokens: 120, totalTokensFresh: true, + totalTokensVersion: 1, }; const sessionStore = { "agent:main:telegram:group:redacted": sessionEntry }; @@ -1876,6 +1931,7 @@ describe("runMemoryFlushIfNeeded", () => { updatedAt: Date.now(), totalTokens: 120, totalTokensFresh: true, + totalTokensVersion: 1, }; const sessionStore = { "agent:main:telegram:group:redacted": sessionEntry }; @@ -1910,6 +1966,7 @@ describe("runMemoryFlushIfNeeded", () => { updatedAt: Date.now(), totalTokens: 245_000, totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 0, }; @@ -1953,6 +2010,7 @@ describe("runMemoryFlushIfNeeded", () => { updatedAt: Date.now(), totalTokens: 985, totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 0, }; await upsertSessionEntry({ agentId: "main", sessionKey, storePath }, sessionEntry); @@ -1992,6 +2050,7 @@ describe("runMemoryFlushIfNeeded", () => { totalTokens: 985, outputTokens: 50_000, totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 0, }; @@ -2014,6 +2073,158 @@ describe("runMemoryFlushIfNeeded", () => { expect(compactEmbeddedAgentSessionMock).not.toHaveBeenCalled(); }); + it("stops at unavailable context and accepts only a later valid transcript snapshot", async () => { + const sessionKey = "agent:main:main"; + const storePath = path.join(rootDir, "sessions.json"); + const oldCumulative = { + type: "message", + message: { + role: "assistant", + content: "old cumulative turn", + usage: { input: 128_814, output: 3_000, cacheRead: 992_953, totalTokens: 1_124_767 }, + }, + }; + const unavailable = { + type: "message", + message: { + role: "assistant", + content: "usage unavailable", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + contextUsage: { state: "unavailable" }, + }, + }, + }; + await writeTestSessionTranscript({ rootDir, sessionKey, events: [oldCumulative, unavailable] }); + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: Date.now(), + totalTokensFresh: false, + compactionCount: 0, + }; + const sessionStore = { [sessionKey]: sessionEntry }; + const run = () => + runPreflightCompactionIfNeeded({ + cfg: { agents: { defaults: { compaction: { memoryFlush: {} } } } }, + followupRun: createTestFollowupRun({ + provider: "anthropic", + model: "claude", + sessionId: "session", + sessionKey, + }), + promptForEstimate: "", + defaultModel: "anthropic/claude", + agentCfgContextTokens: 100_000, + sessionEntry, + sessionStore, + sessionKey, + storePath, + isHeartbeat: false, + replyOperation: createReplyOperation(), + }); + + await run(); + expect(compactEmbeddedAgentSessionMock).not.toHaveBeenCalled(); + + await writeTestSessionTranscript({ + rootDir, + sessionKey, + events: [ + oldCumulative, + unavailable, + { + type: "message", + message: { + role: "assistant", + content: "valid later turn", + usage: { input: 67_932, output: 2_000, cacheRead: 18_944, totalTokens: 88_876 }, + }, + }, + ], + }); + await run(); + + expect(compactEmbeddedAgentSessionMock).toHaveBeenCalledTimes(1); + expect(requireCompactEmbeddedAgentSessionCall().currentTokenCount).toBe(88_876); + }); + it("ignores unversioned fresh state and legacy CLI usage on the first upgraded turn", async () => { + const sessionKey = "agent:main:main"; + const storePath = path.join(rootDir, "sessions.json"); + const legacyCli = { + type: "message", + message: { + role: "assistant", + api: "cli", + content: "legacy cumulative turn", + usage: { input: 128_814, output: 3_000, cacheRead: 992_953, totalTokens: 1_124_767 }, + }, + }; + await writeTestSessionTranscript({ rootDir, sessionKey, events: [legacyCli] }); + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: Date.now(), + totalTokens: 1_124_767, + totalTokensFresh: true, + compactionCount: 0, + }; + const sessionStore = { [sessionKey]: sessionEntry }; + const run = () => + runPreflightCompactionIfNeeded({ + cfg: { agents: { defaults: { compaction: { memoryFlush: {} } } } }, + followupRun: createTestFollowupRun({ + provider: "anthropic", + model: "claude", + sessionId: "session", + sessionKey, + }), + promptForEstimate: "", + defaultModel: "anthropic/claude", + agentCfgContextTokens: 100_000, + sessionEntry, + sessionStore, + sessionKey, + storePath, + isHeartbeat: false, + replyOperation: createReplyOperation(), + }); + + await run(); + expect(compactEmbeddedAgentSessionMock).not.toHaveBeenCalled(); + + await writeTestSessionTranscript({ + rootDir, + sessionKey, + events: [ + legacyCli, + { + type: "message", + message: { + role: "assistant", + api: "cli", + content: "repaired exact turn", + usage: { + input: 67_932, + output: 2_000, + cacheRead: 18_944, + totalTokens: 88_876, + contextUsage: { + state: "available", + promptTokens: 86_876, + totalTokens: 88_876, + }, + }, + }, + }, + ], + }); + await run(); + + expect(requireCompactEmbeddedAgentSessionCall().currentTokenCount).toBe(88_876); + }); it("updates the active preflight run after transcript rotation", async () => { const sessionFile = path.join(rootDir, "session.jsonl"); const successorFile = path.join(rootDir, "session-rotated.jsonl"); @@ -2128,6 +2339,103 @@ describe("runMemoryFlushIfNeeded", () => { expect(compactCall.currentTokenCount).toBeGreaterThanOrEqual(100_000); }); + it("keeps nonzero unavailable output as growth after the previous exact snapshot", async () => { + await writeTestSessionTranscript({ + rootDir, + events: [ + { + type: "message", + message: { + role: "assistant", + content: "large answer", + usage: { + input: 128_814, + output: 10_000, + cacheRead: 992_953, + totalTokens: 1_131_767, + contextUsage: { state: "unavailable" }, + }, + }, + }, + ], + }); + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: Date.now(), + totalTokens: 70_000, + totalTokensFresh: true, + totalTokensVersion: 1, + }; + + await runPreflightCompactionIfNeeded({ + cfg: { agents: { defaults: { compaction: { memoryFlush: {} } } } }, + followupRun: createTestFollowupRun({ sessionId: "session", sessionKey: "main" }), + promptForEstimate: "continue", + defaultModel: "anthropic/claude-opus-4-6", + agentCfgContextTokens: 100_000, + sessionEntry, + sessionStore: { main: sessionEntry }, + sessionKey: "main", + storePath: path.join(rootDir, "sessions.json"), + isHeartbeat: false, + replyOperation: createReplyOperation(), + }); + + expect(requireCompactEmbeddedAgentSessionCall().currentTokenCount).toBeGreaterThanOrEqual( + 80_000, + ); + }); + + it("does not add unavailable output twice when full-message estimation already includes it", async () => { + await writeTestSessionTranscript({ + rootDir, + events: [ + { + type: "message", + message: { + role: "assistant", + content: "x".repeat(3_600), + usage: { + input: 1, + output: 200, + totalTokens: 201, + contextUsage: { state: "unavailable" }, + }, + }, + }, + ], + }); + registerMemoryFlushPlanResolverForTest(() => ({ + softThresholdTokens: 0, + forceFlushTranscriptBytes: 1_000_000_000, + reserveTokensFloor: 0, + prompt: "Pre-compaction memory flush.\nNO_REPLY", + systemPrompt: "Write memory to memory/YYYY-MM-DD.md.", + relativePath: "memory/2023-11-14.md", + })); + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: Date.now(), + totalTokensFresh: false, + }; + + await runPreflightCompactionIfNeeded({ + cfg: { agents: { defaults: { compaction: { memoryFlush: {} } } } }, + followupRun: createTestFollowupRun({ sessionId: "session", sessionKey: "main" }), + promptForEstimate: "", + defaultModel: "anthropic/claude-opus-4-6", + agentCfgContextTokens: 1_000, + sessionEntry, + sessionStore: { main: sessionEntry }, + sessionKey: "main", + storePath: path.join(rootDir, "sessions.json"), + isHeartbeat: false, + replyOperation: createReplyOperation(), + }); + + expect(compactEmbeddedAgentSessionMock).not.toHaveBeenCalled(); + }); + it("reads flush usage and byte size from SQLite without statting a retired transcript path", async () => { const sessionFile = path.join(rootDir, "memory-flush-usage-and-size.jsonl"); await writeTestSessionTranscript({ @@ -2195,6 +2503,7 @@ describe("runMemoryFlushIfNeeded", () => { updatedAt: Date.now(), totalTokens: 180_499, totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 0, }; const sessionStore = { main: sessionEntry }; @@ -2287,6 +2596,7 @@ describe("runMemoryFlushIfNeeded", () => { updatedAt: Date.now(), totalTokens: 347_000, totalTokensFresh: true, + totalTokensVersion: 1, agentRuntimeOverride: "codex", agentHarnessId: "openclaw", }; @@ -2334,6 +2644,7 @@ describe("runMemoryFlushIfNeeded", () => { updatedAt: Date.now(), totalTokens: 347_000, totalTokensFresh: true, + totalTokensVersion: 1, agentRuntimeOverride: "claude-cli", }; @@ -2764,6 +3075,7 @@ describe("runMemoryFlushIfNeeded", () => { updatedAt: Date.now(), totalTokens: 10, totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 0, }; const sessionStore = { main: sessionEntry }; @@ -2818,6 +3130,7 @@ describe("runMemoryFlushIfNeeded", () => { updatedAt: Date.now(), totalTokens: 10, totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 0, agentRuntimeOverride: "codex", agentHarnessId: "openclaw", @@ -2877,6 +3190,7 @@ describe("runMemoryFlushIfNeeded", () => { updatedAt: Date.now(), totalTokens: 347_000, totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 0, agentRuntimeOverride: "codex", agentHarnessId: "openclaw", @@ -2937,6 +3251,7 @@ describe("runMemoryFlushIfNeeded", () => { updatedAt: Date.now(), totalTokens: 10, totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 0, }; const cfg = { @@ -3008,6 +3323,7 @@ describe("runMemoryFlushIfNeeded", () => { updatedAt: Date.now(), totalTokens: 10, totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 0, }; const replyOperation = createReplyOperation(); @@ -3050,6 +3366,7 @@ describe("runMemoryFlushIfNeeded", () => { updatedAt: Date.now(), totalTokens: 90_000, totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 0, }; @@ -3184,6 +3501,7 @@ describe("runMemoryFlushIfNeeded", () => { updatedAt: Date.now(), totalTokens: 10, totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 0, }; const replyOperation = createReplyOperation(); @@ -3219,6 +3537,7 @@ describe("runMemoryFlushIfNeeded", () => { updatedAt: Date.now(), totalTokens: 120, totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 0, }; const onCompactionNotice = vi.fn(); @@ -3266,6 +3585,7 @@ describe("runMemoryFlushIfNeeded", () => { updatedAt: Date.now(), totalTokens: 120, totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 0, }; const onCompactionNotice = vi.fn(); @@ -3308,6 +3628,8 @@ describe("runMemoryFlushIfNeeded", () => { sessionId: "session", updatedAt: Date.now(), totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 1, systemPromptReport: { source: "run", diff --git a/src/auto-reply/reply/agent-runner-memory.ts b/src/auto-reply/reply/agent-runner-memory.ts index cef29f8e5bb8..958f1f2d9cf1 100644 --- a/src/auto-reply/reply/agent-runner-memory.ts +++ b/src/auto-reply/reply/agent-runner-memory.ts @@ -36,6 +36,7 @@ import { resolveAgentIdFromSessionKey, resolveFreshSessionTotalTokens, resolveStorePath, + SESSION_TOTAL_TOKENS_VERSION, type SessionEntry, } from "../../config/sessions.js"; import { @@ -360,6 +361,17 @@ type SessionTranscriptUsageSnapshot = { trailingBytesTokens?: number; }; +function isUnavailableContextBarrier( + usage: NonNullable>, +): boolean { + if (usage.contextUsage?.state !== "unavailable") { + return false; + } + return [usage.input, usage.output, usage.cacheRead, usage.cacheWrite, usage.total].every( + (value) => !(typeof value === "number" && value > 0), + ); +} + // Keep a generous near-threshold window so large assistant outputs still trigger // transcript reads in time to flip memory-flush gating when needed. const TRANSCRIPT_OUTPUT_READ_BUFFER_TOKENS = 8192; @@ -414,9 +426,18 @@ function readLatestNonzeroUsageSnapshotFromTranscriptEvents( } const message = record.message && typeof record.message === "object" && !Array.isArray(record.message) - ? (record.message as { usage?: UsageLike }) + ? (record.message as { api?: unknown; usage?: UsageLike }) : undefined; - const usage = normalizeUsage(message?.usage ?? record.usage); + const rawUsage = message?.usage ?? record.usage; + if (message?.api === "cli" && rawUsage && rawUsage.contextUsage === undefined) { + return undefined; + } + const usage = normalizeUsage(rawUsage); + if (usage && isUnavailableContextBarrier(usage)) { + // This turn supersedes older context facts without supplying a replacement. + // Stop the reverse scan so pre-fix cumulative records cannot become fresh again. + return undefined; + } if (usage && hasNonzeroUsage(usage)) { return { usage, trailingBytes }; } @@ -544,6 +565,7 @@ function readSessionLogSnapshot(params: { type TranscriptTokenEstimate = { promptTokens: number; outputTokens?: number; + promptIncludesOutput?: boolean; transcriptByteSize?: number; transcriptBytesTokens?: number; }; @@ -630,6 +652,13 @@ async function estimatePromptTokensFromSessionTranscript(params: { } return { promptTokens: Math.ceil(estimatedTokens), + // Full-message estimation already includes assistant content. Preserve + // output only for projection against a separate persisted prompt fact. + promptIncludesOutput: true, + outputTokens: + typeof outputTokens === "number" && Number.isFinite(outputTokens) && outputTokens > 0 + ? Math.ceil(outputTokens) + : undefined, transcriptByteSize: snapshot.byteSize, transcriptBytesTokens, }; @@ -717,11 +746,6 @@ export async function runPreflightCompactionIfNeeded(params: { const reserveTokensFloor = memoryFlushPlan?.reserveTokensFloor ?? 20_000; const softThresholdTokens = memoryFlushPlan?.softThresholdTokens ?? 4_000; const freshPersistedTokens = resolveFreshSessionTotalTokens(entry); - const persistedTotalTokens = entry.totalTokens; - const hasPersistedTotalTokens = - typeof persistedTotalTokens === "number" && - Number.isFinite(persistedTotalTokens) && - persistedTotalTokens > 0; const promptTokenEstimate = estimatePromptTokensForMemoryFlush( params.promptForEstimate ?? params.followupRun.prompt, ); @@ -778,17 +802,16 @@ export async function runPreflightCompactionIfNeeded(params: { ); return entry ?? params.sessionEntry; } - const stalePersistedPromptTokens = - hasPersistedTotalTokens && entry.totalTokensFresh !== false - ? Math.floor(persistedTotalTokens) - : undefined; const transcriptPromptTokens = transcriptUsageTokens?.promptTokens; const transcriptOutputTokens = transcriptUsageTokens?.outputTokens; + const transcriptEstimateOutputTokens = transcriptUsageTokens?.promptIncludesOutput + ? undefined + : transcriptOutputTokens; const usageProjectedTokenCount = typeof transcriptPromptTokens === "number" ? resolveEffectivePromptTokens( transcriptPromptTokens, - transcriptOutputTokens, + transcriptEstimateOutputTokens, promptTokenEstimate, ) : undefined; @@ -803,7 +826,6 @@ export async function runPreflightCompactionIfNeeded(params: { const projectedTokenCount = Math.max( usageProjectedTokenCount ?? 0, freshProjectedTokenCount ?? 0, - stalePersistedPromptTokens ?? 0, ); const tokenCountForCompaction = Number.isFinite(projectedTokenCount) && projectedTokenCount > 0 @@ -1069,8 +1091,7 @@ export async function runMemoryFlushIfNeeded(params: { persistedPromptTokensRaw > 0 ? persistedPromptTokensRaw : undefined; - const hasFreshPersistedPromptTokens = - typeof persistedPromptTokens === "number" && entry?.totalTokensFresh === true; + const hasFreshPersistedPromptTokens = resolveFreshSessionTotalTokens(entry) !== undefined; const flushThreshold = contextWindowTokens - memoryFlushPlan.reserveTokensFloor - memoryFlushPlan.softThresholdTokens; @@ -1138,6 +1159,7 @@ export async function runMemoryFlushIfNeeded(params: { ...entry, totalTokens: transcriptPromptTokens, totalTokensFresh: true, + totalTokensVersion: SESSION_TOTAL_TOKENS_VERSION, }; entry = nextEntry; if (params.sessionKey && params.sessionStore) { @@ -1150,7 +1172,11 @@ export async function runMemoryFlushIfNeeded(params: { storePath: params.storePath, sessionKey: params.sessionKey, }, - () => ({ totalTokens: transcriptPromptTokens, totalTokensFresh: true }), + () => ({ + totalTokens: transcriptPromptTokens, + totalTokensFresh: true, + totalTokensVersion: SESSION_TOTAL_TOKENS_VERSION, + }), { skipMaintenance: true, takeCacheOwnership: true, diff --git a/src/auto-reply/reply/agent-runner-result-accounting.ts b/src/auto-reply/reply/agent-runner-result-accounting.ts index dd0a23a2aa34..79d6877d5a92 100644 --- a/src/auto-reply/reply/agent-runner-result-accounting.ts +++ b/src/auto-reply/reply/agent-runner-result-accounting.ts @@ -252,7 +252,6 @@ export async function accountAgentTurn(context: AgentTurnAccountingContext) { lastCallUsage: runResult.meta?.agentMeta?.lastCallUsage, compactionTokensAfter: runResult.meta?.agentMeta?.compactionTokensAfter, promptTokens, - usageIsContextSnapshot: usedCliProvider ? true : undefined, isHeartbeat, preserveRuntimeModel: fallbackExhausted, preserveUserFacingSessionModelState: preserveUserFacingSessionState, diff --git a/src/auto-reply/reply/agent-runner-session-reset.ts b/src/auto-reply/reply/agent-runner-session-reset.ts index 4c439e9cbafc..285114ef347b 100644 --- a/src/auto-reply/reply/agent-runner-session-reset.ts +++ b/src/auto-reply/reply/agent-runner-session-reset.ts @@ -86,6 +86,7 @@ export async function resetReplyRunSession(params: { outputTokens: undefined, totalTokens: undefined, totalTokensFresh: false, + totalTokensVersion: undefined, estimatedCostUsd: undefined, cacheRead: undefined, cacheWrite: undefined, diff --git a/src/auto-reply/reply/agent-runner.runreplyagent.e2e.test.ts b/src/auto-reply/reply/agent-runner.runreplyagent.e2e.test.ts index e91bc678cfbd..6f50394395b5 100644 --- a/src/auto-reply/reply/agent-runner.runreplyagent.e2e.test.ts +++ b/src/auto-reply/reply/agent-runner.runreplyagent.e2e.test.ts @@ -4796,8 +4796,10 @@ describe("runReplyAgent typing (heartbeat)", () => { } }); - it("does not persist fallback state for an equivalent CLI runtime alias", async () => { + it("does not persist cumulative CLI usage as a fresh context snapshot", async () => { const sessionEntry = makeSessionEntry({ + totalTokens: 42_000, + totalTokensFresh: true, fallbackNotice: { kind: "active", selectedModel: "anthropic/claude-opus-4-7", @@ -4816,7 +4818,7 @@ describe("runReplyAgent typing (heartbeat)", () => { agentMeta: { provider: "claude-cli", model: "claude-opus-4-7", - usage: { input: 36_000, output: 19_000 }, + usage: { input: 205_000, output: 19_000 }, }, }, }); @@ -4839,8 +4841,8 @@ describe("runReplyAgent typing (heartbeat)", () => { expect(stored.fallbackNotice).toBeUndefined(); expect(stored.modelProvider).toBe("claude-cli"); expect(stored.model).toBe("claude-opus-4-7"); - expect(stored.totalTokens).toBe(36_000); - expect(stored.totalTokensFresh).toBe(true); + expect(stored.totalTokens).toBeUndefined(); + expect(stored.totalTokensFresh).toBe(false); }); it("surfaces overflow fallback when embedded run returns empty payloads", async () => { diff --git a/src/auto-reply/reply/commands-context-report.test.ts b/src/auto-reply/reply/commands-context-report.test.ts index 31e76293acb0..5d6447312271 100644 --- a/src/auto-reply/reply/commands-context-report.test.ts +++ b/src/auto-reply/reply/commands-context-report.test.ts @@ -306,6 +306,7 @@ describe("buildContextReply", () => { updatedAt: params.sessionEntry?.updatedAt ?? 1, totalTokens: 111, totalTokensFresh: true, + totalTokensVersion: 1, inputTokens: 100, outputTokens: 11, } satisfies SessionEntry; @@ -315,6 +316,7 @@ describe("buildContextReply", () => { ...sessionEntry, totalTokens: 900, totalTokensFresh: true, + totalTokensVersion: 1, inputTokens: 700, outputTokens: 200, }, diff --git a/src/auto-reply/reply/commands-context-report.ts b/src/auto-reply/reply/commands-context-report.ts index 267143d93417..80f1625e43dc 100644 --- a/src/auto-reply/reply/commands-context-report.ts +++ b/src/auto-reply/reply/commands-context-report.ts @@ -188,8 +188,8 @@ export async function buildContextReply(params: HandleCommandsParams): Promise { await upsertSessionEntry({ storePath, sessionKey, - entry: { sessionId: "sess-main", updatedAt: 1, totalTokens: 0, totalTokensFresh: true }, + entry: { + sessionId: "sess-main", + updatedAt: 1, + totalTokens: 0, + totalTokensFresh: true, + totalTokensVersion: 1, + }, }); const params = buildGoalParams("/goal build a 3d game", storePath); @@ -130,7 +136,13 @@ describe("goal commands", () => { await upsertSessionEntry({ storePath, sessionKey, - entry: { sessionId: "sess-main", updatedAt: 1, totalTokens: 0, totalTokensFresh: true }, + entry: { + sessionId: "sess-main", + updatedAt: 1, + totalTokens: 0, + totalTokensFresh: true, + totalTokensVersion: 1, + }, }); const slashParams = buildGoalParams("/goal start /status", storePath); @@ -146,7 +158,13 @@ describe("goal commands", () => { await upsertSessionEntry({ storePath: bangStorePath, sessionKey, - entry: { sessionId: "sess-main", updatedAt: 1, totalTokens: 0, totalTokensFresh: true }, + entry: { + sessionId: "sess-main", + updatedAt: 1, + totalTokens: 0, + totalTokensFresh: true, + totalTokensVersion: 1, + }, }); const bangParams = buildGoalParams("/goal start !npm test", bangStorePath); @@ -294,6 +312,7 @@ describe("goal commands", () => { updatedAt: 1, totalTokens: 25, totalTokensFresh: true, + totalTokensVersion: 1, goal: { schemaVersion: 1, id: "goal-1", diff --git a/src/auto-reply/reply/commands-status.test.ts b/src/auto-reply/reply/commands-status.test.ts index 9255a3afcd9c..1893fd5573f3 100644 --- a/src/auto-reply/reply/commands-status.test.ts +++ b/src/auto-reply/reply/commands-status.test.ts @@ -715,6 +715,7 @@ describe("buildStatusReply subagent summary", () => { model: "kimi-k2.7-code", totalTokens: 0, totalTokensFresh: true, + totalTokensVersion: 1, }, sessionKey: "agent:main:main", parentSessionKey: "agent:main:main", @@ -1404,6 +1405,7 @@ describe("buildStatusReply subagent summary", () => { }, totalTokens: 49_000, totalTokensFresh: true, + totalTokensVersion: 1, contextTokens: 1_048_576, }, sessionKey: "agent:main:main", @@ -1470,6 +1472,7 @@ describe("buildStatusReply subagent summary", () => { }, totalTokens: 49_000, totalTokensFresh: true, + totalTokensVersion: 1, contextTokens: 1_048_576, }, sessionKey: "agent:main:main", diff --git a/src/auto-reply/reply/session-parent-fork-prepare.ts b/src/auto-reply/reply/session-parent-fork-prepare.ts index 3357e23a792e..730231614951 100644 --- a/src/auto-reply/reply/session-parent-fork-prepare.ts +++ b/src/auto-reply/reply/session-parent-fork-prepare.ts @@ -66,6 +66,7 @@ export async function prepareReplySessionParentFork(params: { forkedFromParent: true, totalTokens: undefined, totalTokensFresh: false, + totalTokensVersion: undefined, }; return forkedEntry; } diff --git a/src/auto-reply/reply/session-updates.ts b/src/auto-reply/reply/session-updates.ts index 07992e956a1c..7ee671886912 100644 --- a/src/auto-reply/reply/session-updates.ts +++ b/src/auto-reply/reply/session-updates.ts @@ -5,7 +5,7 @@ import { type ExecPolicyOverrides, resolveNodeExecEligibility, } from "../../agents/exec-defaults.js"; -import type { SessionEntry } from "../../config/sessions.js"; +import { SESSION_TOTAL_TOKENS_VERSION, type SessionEntry } from "../../config/sessions.js"; import { formatSqliteSessionFileMarker } from "../../config/sessions/legacy-sqlite-marker.js"; import { patchSessionEntry, updateSessionEntry } from "../../config/sessions/session-accessor.js"; import { resolveSessionStorePathForScope } from "../../config/sessions/session-store-path.js"; @@ -362,6 +362,7 @@ export async function incrementCompactionCount(params: { if (tokensAfterCompaction !== undefined) { updates.totalTokens = tokensAfterCompaction; updates.totalTokensFresh = true; + updates.totalTokensVersion = SESSION_TOTAL_TOKENS_VERSION; // Clear input/output breakdown since we only have the total estimate after compaction updates.inputTokens = undefined; updates.outputTokens = undefined; @@ -369,6 +370,7 @@ export async function incrementCompactionCount(params: { updates.cacheWrite = undefined; } else if (incrementBy > 0) { updates.totalTokensFresh = false; + updates.totalTokensVersion = undefined; } const nextEntry = projectCanonicalSessionEntryShape({ ...entry, ...updates }); sessionStore[sessionKey] = nextEntry; diff --git a/src/auto-reply/reply/session-usage.ts b/src/auto-reply/reply/session-usage.ts index c3b0b3fc7555..cb8e558cae73 100644 --- a/src/auto-reply/reply/session-usage.ts +++ b/src/auto-reply/reply/session-usage.ts @@ -12,6 +12,7 @@ import { import { getRuntimeConfig } from "../../config/config.js"; import { resolveSessionGoalDisplayState, + SESSION_TOTAL_TOKENS_VERSION, type SessionSystemPromptReport, type SessionEntry, } from "../../config/sessions.js"; @@ -107,7 +108,6 @@ export async function persistSessionUsageUpdate(params: { providerUsed?: string; contextTokensUsed?: number; promptTokens?: number; - usageIsContextSnapshot?: boolean; isHeartbeat?: boolean; systemPromptReport?: SessionSystemPromptReport; cliSessionId?: string; @@ -133,10 +133,7 @@ export async function persistSessionUsageUpdate(params: { params.promptTokens > 0; const hasUsableLastCallUsage = Boolean(params.lastCallUsage) && params.lastCallUsage?.contextUsage?.state !== "unavailable"; - const hasUsableUsageContextSnapshot = - params.usageIsContextSnapshot === true && params.usage?.contextUsage?.state !== "unavailable"; - const hasFreshContextSnapshot = - hasUsableLastCallUsage || hasPromptTokens || hasUsableUsageContextSnapshot; + const hasFreshContextSnapshot = hasUsableLastCallUsage || hasPromptTokens; const compactionTokensAfter = resolveNonNegativeTokenCount(params.compactionTokensAfter); const hasCompactionSnapshot = compactionTokensAfter !== undefined; @@ -161,13 +158,10 @@ export async function persistSessionUsageUpdate(params: { // `usage.input` sums input tokens from every API call in the run // (tool-use loops, compaction retries), overstating actual context. // `lastCallUsage` reflects only the final API call — the true context. - const usageForContext = - params.lastCallUsage ?? - (params.usageIsContextSnapshot === true ? params.usage : undefined); const usageTotalTokens = hasFreshContextSnapshot && !preserveUserFacingRunState ? deriveSessionTotalTokens({ - usage: usageForContext, + lastCallUsage: params.lastCallUsage, contextTokens: resolvedContextTokens, promptTokens: params.promptTokens, }) @@ -224,9 +218,10 @@ export async function persistSessionUsageUpdate(params: { if (runEstimatedCostUsd !== undefined) { patch.estimatedCostUsd = runEstimatedCostUsd; } - if ((hasFreshContextSnapshot || hasCompactionSnapshot) && !preserveUserFacingRunState) { + if ((hasPositiveUsageTotal || hasCompactionSnapshot) && !preserveUserFacingRunState) { patch.totalTokens = totalTokens; patch.totalTokensFresh = true; + patch.totalTokensVersion = SESSION_TOTAL_TOKENS_VERSION; const accountedGoal = resolveSessionGoalDisplayState({ ...entry, ...patch }, updatedAt); if (accountedGoal) { patch.goal = accountedGoal; @@ -236,7 +231,9 @@ export async function persistSessionUsageUpdate(params: { (params.preserveFreshTotalTokensOnStaleUsage !== true || entry.totalTokensFresh !== true) ) { + patch.totalTokens = undefined; patch.totalTokensFresh = false; + patch.totalTokensVersion = undefined; } return preserveUserFacingRunState ? patch @@ -288,6 +285,7 @@ export async function persistSessionUsageUpdate(params: { // A completed run without a context snapshot invalidates any fresh // zero persisted for the previously empty session. patch.totalTokensFresh = false; + patch.totalTokensVersion = undefined; } return preserveUserFacingRunState ? patch diff --git a/src/auto-reply/reply/session.test.ts b/src/auto-reply/reply/session.test.ts index 94b0a12d3c1e..1564e3a8d7af 100644 --- a/src/auto-reply/reply/session.test.ts +++ b/src/auto-reply/reply/session.test.ts @@ -4549,7 +4549,7 @@ describe("persistSessionUsageUpdate", () => { update: { isHeartbeat: true, usage: { input: 1_200, output: 100 }, - usageIsContextSnapshot: true, + lastCallUsage: { input: 1_200, output: 100 }, providerUsed: "claude-cli", modelUsed: "claude-sonnet-4-6", cliSessionBinding: { @@ -4589,7 +4589,7 @@ describe("persistSessionUsageUpdate", () => { update: { isHeartbeat: true, usage: { input: 1_200, output: 100 }, - usageIsContextSnapshot: true, + lastCallUsage: { input: 1_200, output: 100 }, providerUsed: "claude-cli", modelUsed: "claude-sonnet-4-6", clearCliSessionBinding: true, @@ -4604,11 +4604,11 @@ describe("persistSessionUsageUpdate", () => { }, }, { - name: "treats CLI usage as a fresh context snapshot when requested", + name: "treats CLI last-call usage as a fresh context snapshot", seed: {}, update: { usage: { input: 24_000, output: 2_000, cacheRead: 8_000 }, - usageIsContextSnapshot: true, + lastCallUsage: { input: 24_000, output: 2_000, cacheRead: 8_000 }, providerUsed: "claude-cli", cliSessionBinding: { sessionId: "cli-session-1", @@ -4643,7 +4643,7 @@ describe("persistSessionUsageUpdate", () => { }, update: { usage: { input: 24_000, output: 2_000, cacheRead: 8_000 }, - usageIsContextSnapshot: true, + lastCallUsage: { input: 24_000, output: 2_000, cacheRead: 8_000 }, providerUsed: "claude-cli", clearCliSessionBinding: true, }, @@ -4666,7 +4666,6 @@ describe("persistSessionUsageUpdate", () => { update: { usage: { input: 20, output: 10_855, cacheRead: 1_761_324, cacheWrite: 33_047 }, lastCallUsage: { input: 20, output: 10_855, cacheRead: 1_761_324, cacheWrite: 33_047 }, - usageIsContextSnapshot: true, providerUsed: "claude-cli", contextTokensUsed: 1_048_576, compactionTokensAfter: 0, @@ -4777,7 +4776,7 @@ describe("persistSessionUsageUpdate", () => { }, }, { - name: "keeps the prior total stale when last-call context is unavailable", + name: "clears the prior total when last-call context is unavailable", seed: { totalTokens: 148_874, totalTokensFresh: true }, update: { usage: { input: 12, output: 15_104, cacheRead: 819_661, cacheWrite: 93_130 }, @@ -4791,7 +4790,7 @@ describe("persistSessionUsageUpdate", () => { }, }, expected: { - totalTokens: 148_874, + totalTokens: undefined, totalTokensFresh: false, inputTokens: 12, cacheRead: 819_661, @@ -4840,10 +4839,10 @@ describe("persistSessionUsageUpdate", () => { expected: { totalTokens: 42_000, totalTokensFresh: true }, }, { - name: "marks older fresh totalTokens stale when no compaction preservation is requested", + name: "clears older totalTokens when no compaction preservation is requested", seed: { totalTokens: 42_000, totalTokensFresh: true }, update: { usage: { input: 50_000, output: 5_000, total: 55_000 } }, - expected: { totalTokens: 42_000, totalTokensFresh: false }, + expected: { totalTokens: undefined, totalTokensFresh: false }, }, { name: "uses promptTokens when available without lastCallUsage", diff --git a/src/auto-reply/reply/session.ts b/src/auto-reply/reply/session.ts index d46910417c52..5c75fa7f3ab2 100644 --- a/src/auto-reply/reply/session.ts +++ b/src/auto-reply/reply/session.ts @@ -45,6 +45,7 @@ import { } from "../../config/sessions/terminal-status.js"; import { DEFAULT_RESET_TRIGGERS, + SESSION_TOTAL_TOKENS_VERSION, type GroupKeyResolution, type SessionEntry, type SessionScope, @@ -952,6 +953,7 @@ async function initSessionStateAttemptLocked( // inherit history without a fresh count, so keep those explicitly unknown. sessionEntry.totalTokens = 0; sessionEntry.totalTokensFresh = true; + sessionEntry.totalTokensVersion = SESSION_TOTAL_TOKENS_VERSION; sessionEntry.inputTokens = undefined; sessionEntry.outputTokens = undefined; sessionEntry.estimatedCostUsd = undefined; diff --git a/src/auto-reply/status.test.ts b/src/auto-reply/status.test.ts index 4c4143d8b9e1..fbc5786e8e0f 100644 --- a/src/auto-reply/status.test.ts +++ b/src/auto-reply/status.test.ts @@ -164,6 +164,7 @@ function makeFallbackContextStatusArgs({ }, totalTokens: 49_000, totalTokensFresh: true, + totalTokensVersion: 1 as const, ...(sessionContextTokens === undefined ? {} : { contextTokens: sessionContextTokens }), }, sessionKey: "agent:main:main", @@ -209,6 +210,7 @@ describe("buildStatusMessage", () => { outputTokens: 800, totalTokens: 16_000, totalTokensFresh: true, + totalTokensVersion: 1 as const, contextTokens: 32_000, thinkingLevel: "low", verboseLevel: "on", @@ -315,15 +317,15 @@ describe("buildStatusMessage", () => { unexpectedContext: "Context: 3.8m/1.0m", }, { - name: "preserves legacy unknown-freshness totalTokens as context usage", + name: "treats legacy unknown-freshness totalTokens as unknown context", sessionEntry: { sessionId: "abc", updatedAt: 0, totalTokens: 25_000, contextTokens: 1_000_000, }, - expectedContext: "Context: 25k/1.0m", - unexpectedContext: "Context: ?/1.0m", + expectedContext: "Context: ?/1.0m", + unexpectedContext: "Context: 25k/1.0m", }, ])("$name", ({ sessionEntry, expectedContext, unexpectedContext }) => { const text = buildStatusMessage({ @@ -381,6 +383,7 @@ describe("buildStatusMessage", () => { updatedAt: 0, totalTokens: 36_000, totalTokensFresh: true, + totalTokensVersion: 1 as const, contextTokens: 1_000_000, contextBudgetStatus: makeContextBudgetStatus(), }, @@ -893,6 +896,7 @@ describe("buildStatusMessage", () => { groupId: "123", totalTokens: 49_000, totalTokensFresh: true, + totalTokensVersion: 1, contextTokens: 1_048_576, }, sessionKey: "agent:main:main", @@ -929,6 +933,7 @@ describe("buildStatusMessage", () => { updatedAt: 0, totalTokens: 200_000, totalTokensFresh: true, + totalTokensVersion: 1, }, sessionKey: "agent:main:main", sessionScope: "per-sender", @@ -948,6 +953,7 @@ describe("buildStatusMessage", () => { updatedAt: 0, totalTokens: 200_000, totalTokensFresh: true, + totalTokensVersion: 1, }, sessionKey: "agent:main:main", sessionScope: "per-sender", @@ -968,6 +974,7 @@ describe("buildStatusMessage", () => { contextTokens: 4_096, totalTokens: 1_024, totalTokensFresh: true, + totalTokensVersion: 1 as const, }; applyModelOverrideToSessionEntry({ @@ -1007,6 +1014,7 @@ describe("buildStatusMessage", () => { model: "deepseek-v4-pro", totalTokens: 501, totalTokensFresh: true, + totalTokensVersion: 1, contextTokens: 1_000_000, }, sessionKey: "agent:main:main", @@ -1047,6 +1055,7 @@ describe("buildStatusMessage", () => { model: "kimi-k2.7-code", totalTokens: 0, totalTokensFresh: true, + totalTokensVersion: 1, contextTokens: 262_144, }, sessionKey: "agent:main:main", @@ -1159,6 +1168,7 @@ describe("buildStatusMessage", () => { cacheRead: 3_000_000, totalTokens: 36_000, totalTokensFresh: true, + totalTokensVersion: 1, contextTokens: 1_000_000, }, sessionKey: "agent:main:main", @@ -2187,6 +2197,7 @@ describe("buildStatusMessage", () => { updatedAt: 0, totalTokens: 1205, totalTokensFresh: true, + totalTokensVersion: 1, model: "google/gemini-2.5-pro", }, sessionKey: "agent:main:main", @@ -2233,6 +2244,7 @@ describe("buildStatusMessage", () => { }, totalTokens: 49_000, totalTokensFresh: true, + totalTokensVersion: 1, }, sessionKey: "agent:main:main", sessionScope: "per-sender", @@ -2269,6 +2281,7 @@ describe("buildStatusMessage", () => { model: "openai/gpt-4o", totalTokens: 49_000, totalTokensFresh: true, + totalTokensVersion: 1, }, sessionKey: "agent:main:main", sessionScope: "per-sender", @@ -2346,6 +2359,7 @@ describe("buildStatusMessage", () => { updatedAt: 0, totalTokens: 25_000, totalTokensFresh: true, + totalTokensVersion: 1, }, sessionKey: "agent:main:main", sessionScope: "per-sender", @@ -2374,6 +2388,7 @@ describe("buildStatusMessage", () => { updatedAt: 0, totalTokens: 25_000, totalTokensFresh: true, + totalTokensVersion: 1, }, sessionKey: "agent:main:main", sessionScope: "per-sender", @@ -2399,6 +2414,7 @@ describe("buildStatusMessage", () => { updatedAt: 0, totalTokens: 25_000, totalTokensFresh: true, + totalTokensVersion: 1, }, sessionKey: "agent:main:main", sessionScope: "per-sender", diff --git a/src/commands/sessions-table.ts b/src/commands/sessions-table.ts index 82d4b204f3c5..a72e5af03e7d 100644 --- a/src/commands/sessions-table.ts +++ b/src/commands/sessions-table.ts @@ -42,6 +42,7 @@ export type SessionDisplayRow = { outputTokens?: number; totalTokens?: number; totalTokensFresh?: boolean; + totalTokensVersion?: 1; model?: string; modelProvider?: string; providerOverride?: string; @@ -87,6 +88,7 @@ export function toSessionDisplayRow(key: string, entry: SessionEntry): SessionDi outputTokens: entry?.outputTokens, totalTokens: entry?.totalTokens, totalTokensFresh: entry?.totalTokensFresh, + totalTokensVersion: entry?.totalTokensVersion, model: entry?.model, modelProvider: entry?.modelProvider, providerOverride: entry?.providerOverride, diff --git a/src/commands/sessions.test.ts b/src/commands/sessions.test.ts index 6a4cbd9796e2..eb31374291f9 100644 --- a/src/commands/sessions.test.ts +++ b/src/commands/sessions.test.ts @@ -39,6 +39,7 @@ describe("sessionsCommand", () => { outputTokens: 800, totalTokens: 2000, totalTokensFresh: true, + totalTokensVersion: 1, model: "test:opus", }, }); @@ -157,6 +158,7 @@ describe("sessionsCommand", () => { outputTokens: 800, totalTokens: 2000, totalTokensFresh: true, + totalTokensVersion: 1, model: "test:opus", }, "agent:main:quietchat:group:demo": { diff --git a/src/commands/sessions.ts b/src/commands/sessions.ts index 5ed298f7bad1..bf2fa5af560a 100644 --- a/src/commands/sessions.ts +++ b/src/commands/sessions.ts @@ -19,7 +19,7 @@ import { import { resolveRuntimePolicySessionKey } from "../auto-reply/reply/runtime-policy-session-key.js"; import { normalizeChatType } from "../channels/chat-type.js"; import { getRuntimeConfig } from "../config/config.js"; -import { resolveSessionTotalTokens } from "../config/sessions.js"; +import { resolveFreshSessionTotalTokens } from "../config/sessions.js"; import { listSessionEntriesReadOnly } from "../config/sessions/session-accessor.js"; import { resolveSqliteTargetFromSessionStorePath } from "../config/sessions/session-sqlite-target.js"; import type { SessionEntry } from "../config/sessions/types.js"; @@ -460,9 +460,8 @@ export async function sessionsCommand( const modelRef = row.displayModelRef; return { ...r, - totalTokens: resolveSessionTotalTokens(r) ?? null, - totalTokensFresh: - typeof r.totalTokens === "number" ? r.totalTokensFresh !== false : false, + totalTokens: resolveFreshSessionTotalTokens(r) ?? null, + totalTokensFresh: resolveFreshSessionTotalTokens(r) !== undefined, // Prefer row-level context tokens, then config/model lookup, so JSON // mirrors the terminal percentage calculation. contextTokens: @@ -525,7 +524,7 @@ export async function sessionsCommand( configuredContextTokens ?? (await lookupContextTokensForDisplay(model)) ?? configContextTokens; - const total = resolveSessionTotalTokens(row); + const total = resolveFreshSessionTotalTokens(row); const line = [ ...(showAgentColumn diff --git a/src/commands/status.test.ts b/src/commands/status.test.ts index 3eec60f21ce6..a494944f0489 100644 --- a/src/commands/status.test.ts +++ b/src/commands/status.test.ts @@ -27,6 +27,7 @@ function createDefaultSessionStoreEntry() { cacheWrite: 1_000, totalTokens: 5_000, totalTokensFresh: true as boolean, + totalTokensVersion: 1 as const, contextTokens: 10_000, model: "test:opus", sessionId: "abc123", @@ -569,12 +570,11 @@ vi.mock("../config/sessions/session-accessor.js", () => ({ })), })); vi.mock("../config/sessions/types.js", () => ({ - resolveSessionTotalTokens: vi.fn((entry?: { totalTokens?: number }) => - typeof entry?.totalTokens === "number" ? entry.totalTokens : undefined, - ), resolveFreshSessionTotalTokens: vi.fn( - (entry?: { totalTokens?: number; totalTokensFresh?: boolean }) => - typeof entry?.totalTokens === "number" && entry?.totalTokensFresh !== false + (entry?: { totalTokens?: number; totalTokensFresh?: boolean; totalTokensVersion?: number }) => + typeof entry?.totalTokens === "number" && + entry?.totalTokensFresh === true && + entry.totalTokensVersion === 1 ? entry.totalTokens : undefined, ), diff --git a/src/config/sessions/goals.test.ts b/src/config/sessions/goals.test.ts index 9a9dd9ebcc31..aad6343a4614 100644 --- a/src/config/sessions/goals.test.ts +++ b/src/config/sessions/goals.test.ts @@ -50,6 +50,7 @@ describe("session goals", () => { updatedAt: 1, totalTokens, totalTokensFresh: true, + totalTokensVersion: 1, }, }); } @@ -61,6 +62,8 @@ describe("session goals", () => { entry: { ...getSessionEntry({ storePath: fixture.storePath(), sessionKey })!, totalTokens: 100, + totalTokensFresh: true, + totalTokensVersion: 1, }, }); @@ -90,6 +93,7 @@ describe("session goals", () => { updatedAt: 1, totalTokens: 10, totalTokensFresh: true, + totalTokensVersion: 1, }, now: 10, }); @@ -219,6 +223,7 @@ describe("session goals", () => { ...getSessionEntry({ storePath: fixture.storePath(), sessionKey })!, totalTokens: 125, totalTokensFresh: true, + totalTokensVersion: 1, }, }); @@ -230,7 +235,7 @@ describe("session goals", () => { expect(snapshot.goal?.status).toBe("active"); }); - it("treats token snapshots as fresh unless explicitly stale", async () => { + it("accounts token snapshots with current context provenance", async () => { await upsertSessionEntry({ storePath: fixture.storePath(), sessionKey, @@ -238,6 +243,8 @@ describe("session goals", () => { sessionId: "sess-1", updatedAt: 1, totalTokens: 100, + totalTokensFresh: true, + totalTokensVersion: 1, }, }); await createSessionGoal({ @@ -252,6 +259,8 @@ describe("session goals", () => { entry: { ...getSessionEntry({ storePath: fixture.storePath(), sessionKey })!, totalTokens: 125, + totalTokensFresh: true, + totalTokensVersion: 1, }, }); @@ -440,6 +449,7 @@ describe("session goals", () => { { totalTokens: 140, totalTokensFresh: true, + totalTokensVersion: 1, goal: { schemaVersion: 1, id: "goal-1", @@ -465,6 +475,7 @@ describe("session goals", () => { { totalTokens: 140, totalTokensFresh: true, + totalTokensVersion: 1, goal: { schemaVersion: 1, id: "goal-1", diff --git a/src/config/sessions/goals.ts b/src/config/sessions/goals.ts index ca0061fb31f9..d86b9c4bad96 100644 --- a/src/config/sessions/goals.ts +++ b/src/config/sessions/goals.ts @@ -49,13 +49,13 @@ function normalizeTokenCount(value: number | undefined): number | undefined { } function resolveEntryFreshTotalTokens( - entry: Pick, + entry: Pick, ): number | undefined { return normalizeTokenCount(resolveFreshSessionTotalTokens(entry)); } function resolveEntryGoalStartTokens( - entry: Pick, + entry: Pick, ): number { return resolveEntryFreshTotalTokens(entry) ?? 0; } @@ -84,7 +84,7 @@ function recordGoalChange( } export function resolveSessionGoalDisplayState( - entry: Pick, + entry: Pick, now?: number, options?: { adoptFreshBaseline?: boolean }, ): SessionGoal | undefined { @@ -92,7 +92,7 @@ export function resolveSessionGoalDisplayState( } function accountGoalUsage( - entry: Pick, + entry: Pick, now: number, options?: { adoptFreshBaseline?: boolean }, ): SessionGoal | undefined { diff --git a/src/config/sessions/session-accessor.conformance.test.ts b/src/config/sessions/session-accessor.conformance.test.ts index 4ead2a6d9bc3..82fa664b86b8 100644 --- a/src/config/sessions/session-accessor.conformance.test.ts +++ b/src/config/sessions/session-accessor.conformance.test.ts @@ -2105,6 +2105,7 @@ describe("sqlite session normalization", () => { reason: "manual", tokensBefore: 42, tokensAfter: 84, + tokensVersion: 1, preCompaction: { sessionId: "pre-compaction-session", leafId: "pre-msg", @@ -2172,6 +2173,7 @@ describe("sqlite session normalization", () => { parentSessionKey: sourceEntryScope.sessionKey, totalTokens: 42, totalTokensFresh: true, + totalTokensVersion: 1, }), ); expect((result.entry as InternalSessionEntry).lifecycleRunId).toBeUndefined(); @@ -2205,6 +2207,7 @@ describe("sqlite session normalization", () => { reason: "manual", tokensBefore: 100, tokensAfter: 25, + tokensVersion: 1, preCompaction: { sessionId: "missing-pre-session", leafId: "missing-pre-msg", @@ -2249,6 +2252,7 @@ describe("sqlite session normalization", () => { expect.objectContaining({ id: "post-msg", type: "message" }), ]); expect(result.entry.totalTokens).toBe(25); + expect(result.entry.totalTokensVersion).toBe(1); }); it("restores a checkpoint by copying SQLite rows and replacing the entry transactionally", async () => { @@ -2278,6 +2282,7 @@ describe("sqlite session normalization", () => { reason: "manual", tokensBefore: 12, tokensAfter: 24, + tokensVersion: 1, preCompaction: { sessionId: "pre-compaction-session", leafId: "pre-msg", @@ -2326,6 +2331,7 @@ describe("sqlite session normalization", () => { compactionCheckpoints: [checkpoint], totalTokens: 12, totalTokensFresh: true, + totalTokensVersion: 1, }), ); await expect(loadSqliteTranscriptEvents(restoredScope)).resolves.toEqual([ diff --git a/src/config/sessions/session-accessor.parent-fork.test.ts b/src/config/sessions/session-accessor.parent-fork.test.ts index ae97f3cf931c..f73ce2bef83e 100644 --- a/src/config/sessions/session-accessor.parent-fork.test.ts +++ b/src/config/sessions/session-accessor.parent-fork.test.ts @@ -7,7 +7,9 @@ import { afterEach, describe, expect, it } from "vitest"; import { SessionManager } from "../../agents/sessions/session-manager.js"; import { parseSqliteSessionFileMarker } from "./legacy-sqlite-marker.js"; import { + forkSessionEntryFromParentTarget, forkSessionFromParentTranscript, + loadSessionEntry, loadTranscriptEvents, replaceSessionEntry, replaceTranscriptEvents, @@ -727,4 +729,51 @@ describe("forkSessionFromParentTranscript", () => { sessionId: parentSessionId, }); }); + + it("clears a reused child token snapshot after parent identity spread", async () => { + const root = await makeRoot("openclaw-parent-fork-reused-child-"); + const storePath = path.join(root, "sessions.json"); + const parentKey = "agent:main:main"; + const childKey = "agent:main:child"; + const parentSessionId = "parent-reused-child"; + await replaceSessionEntry( + { sessionKey: parentKey, storePath }, + { sessionId: parentSessionId, updatedAt: 1 }, + ); + await replaceSessionEntry( + { sessionKey: childKey, storePath }, + { + sessionId: "old-child", + updatedAt: 1, + totalTokens: 88_876, + totalTokensFresh: true, + totalTokensVersion: 1, + }, + ); + await seedParentTranscript({ + storePath, + parentSessionId, + events: [ + { type: "session", version: 3, id: parentSessionId, timestamp: "2026-05-01T00:00:00Z" }, + { + type: "message", + id: "parent-user", + parentId: null, + message: { role: "user", content: "fork me" }, + }, + ], + }); + + const result = await forkSessionEntryFromParentTarget({ + storePath, + parentTarget: { canonicalKey: parentKey, storeKeys: [parentKey] }, + sessionTarget: { canonicalKey: childKey, storeKeys: [childKey] }, + }); + + expect(result.status).toBe("forked"); + const childEntry = loadSessionEntry({ sessionKey: childKey, storePath }); + expect(childEntry?.totalTokens).toBeUndefined(); + expect(childEntry?.totalTokensFresh).toBe(false); + expect(childEntry?.totalTokensVersion).toBeUndefined(); + }); }); diff --git a/src/config/sessions/session-accessor.sqlite-checkpoint.ts b/src/config/sessions/session-accessor.sqlite-checkpoint.ts index 885853b07dc2..239424c6fbdf 100644 --- a/src/config/sessions/session-accessor.sqlite-checkpoint.ts +++ b/src/config/sessions/session-accessor.sqlite-checkpoint.ts @@ -27,7 +27,11 @@ import { readTranscriptIdentityByEventId, } from "./session-accessor.sqlite-transcript-store.js"; import { createSessionTranscriptHeader } from "./transcript-header.js"; -import type { InternalSessionEntry as SessionEntry, SessionCompactionCheckpoint } from "./types.js"; +import { + SESSION_TOTAL_TOKENS_VERSION, + type InternalSessionEntry as SessionEntry, + type SessionCompactionCheckpoint, +} from "./types.js"; // Compaction checkpoint branch/restore owner. @@ -340,12 +344,13 @@ function resolveSqliteCheckpointTranscriptForkSources( checkpoint: SessionCompactionCheckpoint, ): SqliteCheckpointTranscriptForkSource[] { const sources: SqliteCheckpointTranscriptForkSource[] = []; + const checkpointTokensTrusted = checkpoint.tokensVersion === SESSION_TOTAL_TOKENS_VERSION; if (checkpoint.preCompaction.sessionId) { const preLeafId = checkpoint.preCompaction.entryId ?? checkpoint.preCompaction.leafId; sources.push({ sessionId: checkpoint.preCompaction.sessionId, ...(preLeafId ? { leafId: preLeafId } : {}), - ...(typeof checkpoint.tokensBefore === "number" + ...(checkpointTokensTrusted && typeof checkpoint.tokensBefore === "number" ? { totalTokens: checkpoint.tokensBefore } : {}), }); @@ -356,7 +361,7 @@ function resolveSqliteCheckpointTranscriptForkSources( sources.push({ sessionId: checkpoint.postCompaction.sessionId, leafId: postLeafId, - ...(typeof checkpoint.tokensAfter === "number" + ...(checkpointTokensTrusted && typeof checkpoint.tokensAfter === "number" ? { totalTokens: checkpoint.tokensAfter } : {}), }); @@ -440,6 +445,7 @@ function cloneSqliteCheckpointSessionEntry(params: { estimatedCostUsd: undefined, totalTokens: hasTotalTokens ? params.totalTokens : undefined, totalTokensFresh: hasTotalTokens ? true : undefined, + totalTokensVersion: hasTotalTokens ? SESSION_TOTAL_TOKENS_VERSION : undefined, label: params.label ?? params.currentEntry.label, parentSessionKey: params.parentSessionKey ?? params.currentEntry.parentSessionKey, compactionCheckpoints: params.preserveCompactionCheckpoints diff --git a/src/config/sessions/session-accessor.sqlite-message-cut.ts b/src/config/sessions/session-accessor.sqlite-message-cut.ts index 9a429f299299..ef81cef32bd7 100644 --- a/src/config/sessions/session-accessor.sqlite-message-cut.ts +++ b/src/config/sessions/session-accessor.sqlite-message-cut.ts @@ -487,6 +487,7 @@ function cloneMessageCutSessionEntry(params: { estimatedCostUsd: undefined, totalTokens: undefined, totalTokensFresh: undefined, + totalTokensVersion: undefined, // A rotated transcript cannot resume provider/runtime identity from the old tail. // Clear transcript-derived accounting too so the next turn rebuilds canonical state. contextTokens: undefined, diff --git a/src/config/sessions/session-accessor.sqlite-parent-fork.ts b/src/config/sessions/session-accessor.sqlite-parent-fork.ts index 0cddcc4c4be3..221cf1e5e9a7 100644 --- a/src/config/sessions/session-accessor.sqlite-parent-fork.ts +++ b/src/config/sessions/session-accessor.sqlite-parent-fork.ts @@ -13,7 +13,7 @@ import { selectSessionTranscriptTreePathNodes, } from "./transcript-tree.js"; import type { SessionEntry } from "./types.js"; -import { resolveFreshSessionTotalTokens, resolveSessionTotalTokens } from "./types.js"; +import { resolveFreshSessionTotalTokens } from "./types.js"; export type SqliteParentForkSourceTranscript = { appendMode?: "side"; @@ -47,11 +47,7 @@ export function resolveSqliteParentForkDecision( transcriptEstimate?: SqliteTranscriptParentTokenEstimate, ): SessionParentForkDecision { const maxTokens = DEFAULT_PARENT_FORK_MAX_TOKENS; - const parentTokens = - resolveFreshSessionTotalTokens(parentEntry) ?? - (transcriptEstimate?.kind === "exact-context" - ? transcriptEstimate.tokens - : maxPositiveTokenCount(transcriptEstimate?.tokens, resolveSessionTotalTokens(parentEntry))); + const parentTokens = resolveFreshSessionTotalTokens(parentEntry) ?? transcriptEstimate?.tokens; if (typeof parentTokens === "number" && parentTokens > maxTokens) { return { status: "skip", @@ -97,6 +93,12 @@ export function estimateSqliteTranscriptPromptTokens( continue; } const contextUsage = readTranscriptContextUsage(usageRaw); + if (message?.api === "cli" && contextUsage === undefined) { + latestUsageEstimate = undefined; + latestUsageEstimateIsExactContext = false; + trailingBytes = 0; + continue; + } if (contextUsage?.state === "unavailable") { latestUsageEstimate = undefined; latestUsageEstimateIsExactContext = false; @@ -161,17 +163,6 @@ function normalizePositiveTokenCount(value: unknown): number | undefined { : undefined; } -function maxPositiveTokenCount(...values: Array): number | undefined { - let max: number | undefined; - for (const value of values) { - const normalized = normalizePositiveTokenCount(value); - if (normalized !== undefined && (max === undefined || normalized > max)) { - max = normalized; - } - } - return max; -} - function readTranscriptContextUsage( usageRaw: Record, ): { state: "available"; totalTokens: number } | { state: "unavailable" } | undefined { diff --git a/src/config/sessions/session-accessor.sqlite-parent-session.ts b/src/config/sessions/session-accessor.sqlite-parent-session.ts index 0fa6cd8a4d49..cae103501eb3 100644 --- a/src/config/sessions/session-accessor.sqlite-parent-session.ts +++ b/src/config/sessions/session-accessor.sqlite-parent-session.ts @@ -227,6 +227,9 @@ export async function forkSqliteSessionEntryFromParentTarget( forkedFromParent: true, lifecycleRunId: undefined, sessionId: fork.transcript.sessionId, + totalTokens: undefined, + totalTokensFresh: false, + totalTokensVersion: undefined, }; const next = mergeSessionEntry(freshBase, forkIdentityPatch); previousIdentity = readSqliteSessionIdentitySnapshot(writeDatabase, sessionTarget.storeKeys); diff --git a/src/config/sessions/session-accessor.sqlite-transcript-write.ts b/src/config/sessions/session-accessor.sqlite-transcript-write.ts index 316b78413827..dd1636bdfd00 100644 --- a/src/config/sessions/session-accessor.sqlite-transcript-write.ts +++ b/src/config/sessions/session-accessor.sqlite-transcript-write.ts @@ -229,6 +229,7 @@ export async function trimSqliteTranscriptForManualCompact( delete nextEntry.outputTokens; delete nextEntry.totalTokens; delete nextEntry.totalTokensFresh; + delete nextEntry.totalTokensVersion; nextEntry.updatedAt = options.nowMs ?? Date.now(); // The transcript rewrite and token invalidation describe one generation. // Keep them in this transaction so either both become visible or neither does. diff --git a/src/config/sessions/types.ts b/src/config/sessions/types.ts index 1c22ece67eef..10915637ef0f 100644 --- a/src/config/sessions/types.ts +++ b/src/config/sessions/types.ts @@ -28,6 +28,7 @@ import type { AgentPatchedSessionModelFallback } from "./session-model-fallback. export type SessionScope = "per-sender" | "global"; export type SessionChatType = ChatType; +export const SESSION_TOTAL_TOKENS_VERSION = 1 as const; type SessionVisibility = "shared" | "read-only" | "suggest" | "draft"; export type SessionToolOverrides = { @@ -163,6 +164,7 @@ export type SessionCompactionCheckpoint = { reason: SessionCompactionCheckpointReason; tokensBefore?: number; tokensAfter?: number; + tokensVersion?: typeof SESSION_TOTAL_TOKENS_VERSION; summary?: string; firstKeptEntryId?: string; preCompaction: SessionCompactionTranscriptReference; @@ -535,6 +537,8 @@ type SessionEntryCore = SessionRestartRecoveryState & * totalTokens as stale/unknown for context-utilization displays. */ totalTokensFresh?: boolean; + /** Version 1 records totalTokens as the current prompt/context snapshot only. */ + totalTokensVersion?: typeof SESSION_TOTAL_TOKENS_VERSION; estimatedCostUsd?: number; cacheRead?: number; cacheWrite?: number; @@ -796,9 +800,7 @@ export function mergeSessionEntryPreserveActivity( }); } -export function resolveSessionTotalTokens( - entry?: Pick | null, -): number | undefined { +function resolveSessionTotalTokensValue(entry?: Pick | null) { const total = entry?.totalTokens; if (typeof total !== "number" || !Number.isFinite(total) || total < 0) { return undefined; @@ -807,13 +809,16 @@ export function resolveSessionTotalTokens( } export function resolveFreshSessionTotalTokens( - entry?: Pick | null, + entry?: Pick | null, ): number | undefined { - const total = resolveSessionTotalTokens(entry); + const total = resolveSessionTotalTokensValue(entry); if (total === undefined) { return undefined; } - if (entry?.totalTokensFresh === false) { + if ( + entry?.totalTokensFresh !== true || + entry.totalTokensVersion !== SESSION_TOTAL_TOKENS_VERSION + ) { return undefined; } return total; diff --git a/src/cron/isolated-agent/run-finalize.ts b/src/cron/isolated-agent/run-finalize.ts index c3a2aa44a457..9e33854acae7 100644 --- a/src/cron/isolated-agent/run-finalize.ts +++ b/src/cron/isolated-agent/run-finalize.ts @@ -5,6 +5,7 @@ import { hasCommittedMessagingToolDeliveryEvidence } from "../../agents/embedded import { deriveContextPromptTokens } from "../../agents/usage.js"; import { stripHeartbeatToken } from "../../auto-reply/heartbeat.js"; import { HEARTBEAT_TOKEN, isSilentReplyPayloadText } from "../../auto-reply/tokens.js"; +import { SESSION_TOTAL_TOKENS_VERSION } from "../../config/sessions.js"; import { emitTrustedDiagnosticEvent, isDiagnosticsEnabled } from "../../infra/diagnostic-events.js"; import { createChildDiagnosticTraceContext, @@ -166,9 +167,11 @@ export async function finalizeCronRun(params: { if (typeof totalTokens === "number" && Number.isFinite(totalTokens) && totalTokens > 0) { prepared.cronSession.sessionEntry.totalTokens = totalTokens; prepared.cronSession.sessionEntry.totalTokensFresh = true; + prepared.cronSession.sessionEntry.totalTokensVersion = SESSION_TOTAL_TOKENS_VERSION; } else { prepared.cronSession.sessionEntry.totalTokens = undefined; prepared.cronSession.sessionEntry.totalTokensFresh = false; + prepared.cronSession.sessionEntry.totalTokensVersion = undefined; } prepared.cronSession.sessionEntry.cacheRead = cacheRead; prepared.cronSession.sessionEntry.cacheWrite = cacheWrite; diff --git a/src/gateway/server-methods/sessions-compact.ts b/src/gateway/server-methods/sessions-compact.ts index d79301cafa37..4846f58b6359 100644 --- a/src/gateway/server-methods/sessions-compact.ts +++ b/src/gateway/server-methods/sessions-compact.ts @@ -10,6 +10,7 @@ import { resolveEmbeddedSessionLane } from "../../agents/embedded-agent-runner/l import { hasPendingFollowupQueueWork } from "../../auto-reply/reply/queue/state.js"; import { resolveSessionWorkStartError, + SESSION_TOTAL_TOKENS_VERSION, SESSION_LIFECYCLE_CHANGED_ERROR_REASON, } from "../../config/sessions.js"; import { @@ -434,9 +435,11 @@ export const sessionCompactHandlers: GatewayRequestHandlers = { ) { entryToUpdate.totalTokens = result.result.tokensAfter; entryToUpdate.totalTokensFresh = true; + entryToUpdate.totalTokensVersion = SESSION_TOTAL_TOKENS_VERSION; } else { delete entryToUpdate.totalTokens; delete entryToUpdate.totalTokensFresh; + delete entryToUpdate.totalTokensVersion; } return { ok: true, entry: entryToUpdate }; }, diff --git a/src/gateway/server.sessions.compaction.test.ts b/src/gateway/server.sessions.compaction.test.ts index a4b4eeb47d8a..d72495de519e 100644 --- a/src/gateway/server.sessions.compaction.test.ts +++ b/src/gateway/server.sessions.compaction.test.ts @@ -87,13 +87,14 @@ function compactionCheckpointEntry( tokensBefore?: number; tokensAfter?: number; }, -) { +): SessionCompactionCheckpoint { return { checkpointId: options.checkpointId, sessionKey: options.sessionKey, sessionId: fixture.sessionId, createdAt: options.createdAt, reason: options.reason, + tokensVersion: 1, summary: options.summary, ...(options.tokensBefore === undefined ? {} : { tokensBefore: options.tokensBefore }), ...(options.tokensAfter === undefined ? {} : { tokensAfter: options.tokensAfter }), diff --git a/src/gateway/server.sessions.create.test.ts b/src/gateway/server.sessions.create.test.ts index 3c769184f60b..d6267621a298 100644 --- a/src/gateway/server.sessions.create.test.ts +++ b/src/gateway/server.sessions.create.test.ts @@ -3091,6 +3091,7 @@ test("sessions.create forks the parent transcript into the new session", async ( sessionFile: parent.sessionFile, totalTokens: 123, totalTokensFresh: true, + totalTokensVersion: 1, }), }, }); @@ -3328,6 +3329,7 @@ test("sessions.create rejects fork when the parent exceeds the fork size cap", a // Fresh persisted usage above DEFAULT_PARENT_FORK_MAX_TOKENS (100K). totalTokens: 200_000, totalTokensFresh: true, + totalTokensVersion: 1, }), }, }); diff --git a/src/gateway/session-compaction-checkpoints.ts b/src/gateway/session-compaction-checkpoints.ts index 56f6b54e41c3..b7c761ca6e31 100644 --- a/src/gateway/session-compaction-checkpoints.ts +++ b/src/gateway/session-compaction-checkpoints.ts @@ -8,10 +8,11 @@ import { SessionManager, type FileEntry as SessionFileEntry, } from "../agents/sessions/session-manager.js"; -import type { - SessionCompactionCheckpoint, - SessionCompactionCheckpointReason, - SessionEntry, +import { + SESSION_TOTAL_TOKENS_VERSION, + type SessionCompactionCheckpoint, + type SessionCompactionCheckpointReason, + type SessionEntry, } from "../config/sessions.js"; import { isCompactionCheckpointTranscriptFileName } from "../config/sessions/artifacts.js"; import { readFileRangeAsync } from "../config/sessions/file-range.js"; @@ -450,12 +451,13 @@ function readSessionLeafStateFromRecords( function resolveCheckpointTranscriptForkSource( checkpoint: SessionCompactionCheckpoint, ): { sourceFile: string; sourceLeafId?: string; totalTokens?: number } | null { + const checkpointTokensTrusted = checkpoint.tokensVersion === SESSION_TOTAL_TOKENS_VERSION; const preCompactionFile = checkpoint.preCompaction.sessionFile?.trim(); if (preCompactionFile) { return { sourceFile: preCompactionFile, sourceLeafId: checkpoint.preCompaction.entryId ?? checkpoint.preCompaction.leafId, - totalTokens: checkpoint.tokensBefore, + totalTokens: checkpointTokensTrusted ? checkpoint.tokensBefore : undefined, }; } @@ -471,7 +473,7 @@ function resolveCheckpointTranscriptForkSource( return { sourceFile: postCompactionFile, sourceLeafId: postCompactionLeafId, - totalTokens: checkpoint.tokensAfter, + totalTokens: checkpointTokensTrusted ? checkpoint.tokensAfter : undefined, }; } @@ -721,6 +723,7 @@ async function persistSessionCompactionCheckpoint( sessionId: params.sessionId, createdAt, reason: params.reason, + tokensVersion: SESSION_TOTAL_TOKENS_VERSION, ...(typeof params.tokensBefore === "number" ? { tokensBefore: params.tokensBefore } : {}), ...(typeof params.tokensAfter === "number" ? { tokensAfter: params.tokensAfter } : {}), ...(params.summary?.trim() ? { summary: params.summary.trim() } : {}), diff --git a/src/gateway/session-create-fork-entry.ts b/src/gateway/session-create-fork-entry.ts index b2a2e402ff69..d29facd91278 100644 --- a/src/gateway/session-create-fork-entry.ts +++ b/src/gateway/session-create-fork-entry.ts @@ -19,5 +19,6 @@ export function buildForkedGatewaySessionEntry( : {}), totalTokens: undefined, totalTokensFresh: false, + totalTokensVersion: undefined, }; } diff --git a/src/gateway/session-reset-service.ts b/src/gateway/session-reset-service.ts index 2d5301e0797a..6e6fab82a92f 100644 --- a/src/gateway/session-reset-service.ts +++ b/src/gateway/session-reset-service.ts @@ -32,6 +32,7 @@ import { cleanupBrowserSessionsForLifecycleEnd } from "../browser-lifecycle-clea import { getRuntimeConfig } from "../config/io.js"; import { resolveSessionWorkStartError, + SESSION_TOTAL_TOKENS_VERSION, type SessionEntry, deleteSessionEntryLifecycle, resetSessionEntryLifecycle, @@ -1527,6 +1528,7 @@ export async function performGatewaySessionReset(params: { outputTokens: 0, totalTokens: 0, totalTokensFresh: true, + totalTokensVersion: SESSION_TOTAL_TOKENS_VERSION, }; // Drop CLI provider bindings so the next turn after reset starts a fresh // CLI conversation on the provider side. Preserved only for spawned diff --git a/src/gateway/session-transcript-derived-readers.ts b/src/gateway/session-transcript-derived-readers.ts index ad72509ed23f..0dc66d9b6ff5 100644 --- a/src/gateway/session-transcript-derived-readers.ts +++ b/src/gateway/session-transcript-derived-readers.ts @@ -11,6 +11,7 @@ function extractSqliteUsageSnapshot(message: unknown): SessionTranscriptUsageSna return null; } const record = message as { + api?: unknown; model?: unknown; provider?: unknown; usage?: unknown; @@ -21,7 +22,8 @@ function extractSqliteUsageSnapshot(message: unknown): SessionTranscriptUsageSna : undefined; const usage = normalizeUsage(usageRaw); const normalizedUsage = usage ?? {}; - const totalTokens = deriveSessionTotalTokens({ usage }); + const legacyCliUsage = record.api === "cli" && usageRaw && usageRaw.contextUsage === undefined; + const totalTokens = legacyCliUsage ? undefined : deriveSessionTotalTokens({ usage }); const modelProvider = typeof record.provider === "string" ? record.provider.trim() : undefined; const model = typeof record.model === "string" ? record.model.trim() : undefined; const costUsd = @@ -50,6 +52,11 @@ function extractSqliteUsageSnapshot(message: unknown): SessionTranscriptUsageSna ...(typeof normalizedUsage.cacheWrite === "number" ? { cacheWrite: normalizedUsage.cacheWrite } : {}), + ...(legacyCliUsage + ? { contextUsage: { state: "unavailable" } as const } + : normalizedUsage.contextUsage + ? { contextUsage: normalizedUsage.contextUsage } + : {}), ...(typeof totalTokens === "number" ? { totalTokens, totalTokensFresh: true } : {}), ...(typeof costUsd === "number" && Number.isFinite(costUsd) ? { costUsd } : {}), }; @@ -98,7 +105,17 @@ export function aggregateSqliteUsageSnapshots( cacheWrite += snapshot.cacheWrite; sawCacheWrite = true; } - if (typeof snapshot.totalTokens === "number") { + if (snapshot.contextUsage) { + aggregate.contextUsage = snapshot.contextUsage; + } else if (typeof snapshot.totalTokens === "number") { + delete aggregate.contextUsage; + } + if (snapshot.contextUsage?.state === "unavailable") { + // Match JSONL aggregation: the marker clears older context until a later + // per-call snapshot replaces it during this forward scan. + delete aggregate.totalTokens; + delete aggregate.totalTokensFresh; + } else if (typeof snapshot.totalTokens === "number") { aggregate.totalTokens = snapshot.totalTokens; aggregate.totalTokensFresh = true; } diff --git a/src/gateway/session-utils-creators.test.ts b/src/gateway/session-utils-creators.test.ts index 84b787df491d..dfd3a6549970 100644 --- a/src/gateway/session-utils-creators.test.ts +++ b/src/gateway/session-utils-creators.test.ts @@ -338,6 +338,7 @@ it("keeps the serialized list response deterministic for the current filter path subject: "needle global", totalTokens: 1, totalTokensFresh: true, + totalTokensVersion: 1, updatedAt: 999_999, }, }, diff --git a/src/gateway/session-utils-row.ts b/src/gateway/session-utils-row.ts index 6cfd6d7a66ce..c2ff9c6b57c3 100644 --- a/src/gateway/session-utils-row.ts +++ b/src/gateway/session-utils-row.ts @@ -20,6 +20,7 @@ import { buildGroupDisplayTitle, resolveFreshSessionTotalTokens, resolveSessionGoalDisplayState, + SESSION_TOTAL_TOKENS_VERSION, type SessionEntry, } from "../config/sessions.js"; import { sessionEntryForkedFromParent } from "../config/sessions/session-entry-lineage.js"; @@ -285,6 +286,7 @@ export function buildGatewaySessionRow(params: { goal: entry.goal, totalTokens, totalTokensFresh, + totalTokensVersion: totalTokensFresh ? SESSION_TOTAL_TOKENS_VERSION : undefined, }, now, // Session listing is read-only; stale goal baselines are adopted only diff --git a/src/gateway/session-utils.fs.test.ts b/src/gateway/session-utils.fs.test.ts index cda0acbeab4d..70140919368d 100644 --- a/src/gateway/session-utils.fs.test.ts +++ b/src/gateway/session-utils.fs.test.ts @@ -1437,6 +1437,64 @@ describe("readLatestSessionUsageFromTranscript", () => { } }); + test("treats unavailable JSONL context as terminal until a later valid snapshot", async () => { + const sessionId = "usage-unavailable-upgrade-sequence"; + const oldCumulative = { + message: { + role: "assistant", + api: "cli", + provider: "claude-cli", + model: "claude-opus-4-7", + usage: { input: 128_814, output: 3_000, cacheRead: 992_953, totalTokens: 1_124_767 }, + }, + }; + const unavailable = { + message: { + role: "assistant", + provider: "claude-cli", + model: "claude-opus-4-7", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + contextUsage: { state: "unavailable" }, + }, + }, + }; + writeTranscript(tmpDir, sessionId, [oldCumulative]); + const legacySnapshot = await readLatestSessionUsageFromTranscriptAsync(sessionId, storePath); + expect(legacySnapshot?.contextUsage).toEqual({ state: "unavailable" }); + expect(legacySnapshot?.totalTokens).toBeUndefined(); + writeTranscript(tmpDir, sessionId, [oldCumulative, unavailable]); + + const unavailableSnapshot = await readLatestSessionUsageFromTranscriptAsync( + sessionId, + storePath, + ); + expect(unavailableSnapshot?.contextUsage).toEqual({ state: "unavailable" }); + expect(unavailableSnapshot?.totalTokens).toBeUndefined(); + expect(unavailableSnapshot?.totalTokensFresh).toBeUndefined(); + + writeTranscript(tmpDir, sessionId, [ + oldCumulative, + unavailable, + { + message: { + role: "assistant", + provider: "claude-cli", + model: "claude-opus-4-7", + usage: { input: 67_932, output: 2_000, cacheRead: 18_944, totalTokens: 88_876 }, + }, + }, + ]); + expectUsageFields(await readLatestSessionUsageFromTranscriptAsync(sessionId, storePath), { + totalTokens: 86_876, + totalTokensFresh: true, + }); + }); + test("estimates transcript context when local model telemetry is missing", async () => { const sessionId = "usage-local-missing-telemetry"; const userText = "local prompt ".repeat(200); diff --git a/src/gateway/session-utils.fs.ts b/src/gateway/session-utils.fs.ts index ab2fa48266f6..4d3fccc937f4 100644 --- a/src/gateway/session-utils.fs.ts +++ b/src/gateway/session-utils.fs.ts @@ -13,6 +13,7 @@ import { hasNonzeroUsage, normalizeUsage, type ContextUsage, + type UsageLike, } from "../agents/usage.js"; import { materializeSessionArchiveForRead } from "../config/sessions/archive-compression.js"; import type { TranscriptEvent } from "../config/sessions/session-accessor.js"; @@ -1078,8 +1079,14 @@ function extractUsageSnapshotFromTranscriptLine( : parsed.usage && typeof parsed.usage === "object" && !Array.isArray(parsed.usage) ? parsed.usage : undefined; - const usage = normalizeUsage(usageRaw); - const totalTokens = resolvePositiveUsageNumber(deriveSessionTotalTokens({ usage })); + const usageRecord = usageRaw as UsageLike | undefined; + const usage = normalizeUsage(usageRecord); + const api = typeof message.api === "string" ? message.api.trim() : undefined; + const legacyCliUsage = + api === "cli" && usageRecord !== undefined && usageRecord.contextUsage === undefined; + const totalTokens = legacyCliUsage + ? undefined + : resolvePositiveUsageNumber(deriveSessionTotalTokens({ usage })); const costUsd = extractTranscriptUsageCost(usageRaw); const modelProvider = typeof message.provider === "string" @@ -1127,7 +1134,9 @@ function extractUsageSnapshotFromTranscriptLine( if (typeof usage?.cacheWrite === "number" && Number.isFinite(usage.cacheWrite)) { snapshot.cacheWrite = usage.cacheWrite; } - if (usage?.contextUsage) { + if (legacyCliUsage) { + snapshot.contextUsage = { state: "unavailable" }; + } else if (usage?.contextUsage) { snapshot.contextUsage = usage.contextUsage; } if (typeof totalTokens === "number") { @@ -1198,10 +1207,12 @@ function extractAggregateUsageFromTranscriptLines( } if (current.contextUsage) { snapshot.contextUsage = current.contextUsage; - } else { + } else if (typeof current.totalTokens === "number") { delete snapshot.contextUsage; } if (current.contextUsage?.state === "unavailable") { + // Unavailable invalidates every older total; only a later numeric snapshot + // may restore freshness as the forward scan continues. delete snapshot.totalTokens; delete snapshot.totalTokensFresh; } else if (typeof current.totalTokens === "number") { @@ -1234,6 +1245,7 @@ function extractAggregateUsageFromTranscriptLines( } if ( typeof snapshot.totalTokens !== "number" && + snapshot.contextUsage?.state !== "unavailable" && sawEstimatedTranscriptContent && sawEstimateModelIdentity ) { diff --git a/src/gateway/session-utils.search.test.ts b/src/gateway/session-utils.search.test.ts index 28cc8a7a7971..cce35ad2324a 100644 --- a/src/gateway/session-utils.search.test.ts +++ b/src/gateway/session-utils.search.test.ts @@ -668,6 +668,7 @@ describe("listSessionsFromStore search", () => { updatedAt: now, totalTokens: 1200, totalTokensFresh: true, + totalTokensVersion: 1, } as SessionEntry, "agent:main:stale": { sessionId: "sess-stale", @@ -935,6 +936,7 @@ describe("listSessionsFromStore search", () => { contextTokens: 200_000, totalTokens: TRANSCRIPT_TOTAL_TOKENS, totalTokensFresh: true, + totalTokensVersion: 1, }), }); diff --git a/src/gateway/session-utils.test.ts b/src/gateway/session-utils.test.ts index 187f7ff30d3d..b9246be8f9a7 100644 --- a/src/gateway/session-utils.test.ts +++ b/src/gateway/session-utils.test.ts @@ -1104,6 +1104,7 @@ describe("gateway session utils", () => { updatedAt: 1, totalTokens: 0, totalTokensFresh: true, + totalTokensVersion: 1, }, }); @@ -1152,6 +1153,113 @@ describe("gateway session utils", () => { }); }); + test("SQLite unavailable context blocks old totals until a later valid snapshot", async () => { + await withStateDirEnv("session-utils-unavailable-usage-", async ({ stateDir }) => { + const sessionId = "unavailable-usage"; + const sessionKey = "agent:main:main"; + const storePath = path.join(stateDir, "agents", "main", "sessions", "sessions.json"); + const entry: SessionEntry = { + sessionId, + updatedAt: 1, + totalTokens: 1_124_767, + totalTokensFresh: false, + }; + await seedSessionEntries(storePath, { [sessionKey]: entry }); + appendTranscriptMessages({ + sessionId, + sessionKey, + storePath, + messages: [ + { + role: "assistant", + api: "cli", + content: "old cumulative turn", + usage: { + input: 128_814, + output: 3_000, + cacheRead: 992_953, + totalTokens: 1_124_767, + }, + }, + ], + }); + + const legacyRow = buildGatewaySessionRow({ + cfg: createModelDefaultsConfig({ primary: "anthropic/claude-opus-4-7" }), + storePath, + store: { [sessionKey]: entry }, + key: sessionKey, + entry, + }); + expect(legacyRow.totalTokens).toBeUndefined(); + expect(legacyRow.totalTokensFresh).toBe(false); + + appendTranscriptMessages({ + sessionId, + sessionKey, + storePath, + messages: [ + { + role: "assistant", + api: "cli", + content: "usage unavailable", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + contextUsage: { state: "unavailable" }, + }, + }, + ], + }); + + const unavailableRow = buildGatewaySessionRow({ + cfg: createModelDefaultsConfig({ primary: "anthropic/claude-opus-4-7" }), + storePath, + store: { [sessionKey]: entry }, + key: sessionKey, + entry, + }); + expect(unavailableRow.totalTokens).toBeUndefined(); + expect(unavailableRow.totalTokensFresh).toBe(false); + + appendTranscriptMessages({ + sessionId, + sessionKey, + storePath, + messages: [ + { + role: "assistant", + api: "cli", + content: "valid later turn", + usage: { + input: 67_932, + output: 2_000, + cacheRead: 18_944, + totalTokens: 88_876, + contextUsage: { + state: "available", + promptTokens: 86_876, + totalTokens: 88_876, + }, + }, + }, + ], + }); + const validRow = buildGatewaySessionRow({ + cfg: createModelDefaultsConfig({ primary: "anthropic/claude-opus-4-7" }), + storePath, + store: { [sessionKey]: entry }, + key: sessionKey, + entry, + }); + expect(validRow.totalTokens).toBe(86_876); + expect(validRow.totalTokensFresh).toBe(true); + }); + }); + test("session rows use per-agent thinking default from config", () => { const cfg = { agents: { @@ -2963,6 +3071,7 @@ describe("listSessionsFromStore selected model display", () => { model: "gpt-5.4", totalTokens: 1, totalTokensFresh: true, + totalTokensVersion: 1, contextTokens: 1, estimatedCostUsd: 0, } as SessionEntry; diff --git a/src/plugins/session-entry-slot-keys.ts b/src/plugins/session-entry-slot-keys.ts index cc99b76ed9f8..215ccd33e14d 100644 --- a/src/plugins/session-entry-slot-keys.ts +++ b/src/plugins/session-entry-slot-keys.ts @@ -130,6 +130,7 @@ const SESSION_ENTRY_RESERVED_SLOT_KEY_LIST = [ "restartRecoveryTerminalDeliveryEvidence", "restartRecoveryTerminalRunIds", "totalTokensFresh", + "totalTokensVersion", "estimatedCostUsd", "cacheRead", "cacheWrite", diff --git a/src/shared/subagents-format.test.ts b/src/shared/subagents-format.test.ts index 5228eab54450..e02f0d7b07e0 100644 --- a/src/shared/subagents-format.test.ts +++ b/src/shared/subagents-format.test.ts @@ -7,6 +7,12 @@ import { truncateLine, } from "./subagents-format.js"; +const freshUsage = (totalTokens: number) => ({ + totalTokens, + totalTokensFresh: true, + totalTokensVersion: 1 as const, +}); + describe("shared/subagents-format", () => { it("re-exports the canonical formatter with second-level precision", () => { expect(formatDurationCompact()).toBeUndefined(); @@ -20,17 +26,17 @@ describe("shared/subagents-format", () => { it("formats token counts with integer, kilo, and million branches", () => { expect(formatTokenUsageDisplay()).toBe(""); - expect(formatTokenUsageDisplay({ totalTokens: 999.9 })).toBe("tokens 999 prompt/cache"); - expect(formatTokenUsageDisplay({ totalTokens: 1_500 })).toBe("tokens 1.5k prompt/cache"); - expect(formatTokenUsageDisplay({ totalTokens: 10_000 })).toBe("tokens 10k prompt/cache"); - expect(formatTokenUsageDisplay({ totalTokens: 15_400 })).toBe("tokens 15k prompt/cache"); + expect(formatTokenUsageDisplay(freshUsage(999.9))).toBe("tokens 999 prompt/cache"); + expect(formatTokenUsageDisplay(freshUsage(1_500))).toBe("tokens 1.5k prompt/cache"); + expect(formatTokenUsageDisplay(freshUsage(10_000))).toBe("tokens 10k prompt/cache"); + expect(formatTokenUsageDisplay(freshUsage(15_400))).toBe("tokens 15k prompt/cache"); // Rollover boundary: rounding to thousands must not emit an out-of-scheme // "1000k" — it has to advance to the million branch. - expect(formatTokenUsageDisplay({ totalTokens: 999_499 })).toBe("tokens 999k prompt/cache"); - expect(formatTokenUsageDisplay({ totalTokens: 999_500 })).toBe("tokens 1m prompt/cache"); - expect(formatTokenUsageDisplay({ totalTokens: 999_999 })).toBe("tokens 1m prompt/cache"); - expect(formatTokenUsageDisplay({ totalTokens: 1_000_000 })).toBe("tokens 1m prompt/cache"); - expect(formatTokenUsageDisplay({ totalTokens: 1_250_000 })).toBe("tokens 1.3m prompt/cache"); + expect(formatTokenUsageDisplay(freshUsage(999_499))).toBe("tokens 999k prompt/cache"); + expect(formatTokenUsageDisplay(freshUsage(999_500))).toBe("tokens 1m prompt/cache"); + expect(formatTokenUsageDisplay(freshUsage(999_999))).toBe("tokens 1m prompt/cache"); + expect(formatTokenUsageDisplay(freshUsage(1_000_000))).toBe("tokens 1m prompt/cache"); + expect(formatTokenUsageDisplay(freshUsage(1_250_000))).toBe("tokens 1.3m prompt/cache"); }); it("truncates lines only when needed", () => { @@ -72,7 +78,7 @@ describe("shared/subagents-format", () => { it("resolves token totals and io breakdowns from valid numeric fields only", () => { expect(resolveTotalTokens()).toBeUndefined(); - expect(resolveTotalTokens({ totalTokens: 42 })).toBe(42); + expect(resolveTotalTokens(freshUsage(42))).toBe(42); expect(resolveTotalTokens({ inputTokens: 10, outputTokens: 5 })).toBe(15); expect(resolveTotalTokens({ inputTokens: Number.NaN, outputTokens: 5 })).toBeUndefined(); @@ -88,16 +94,16 @@ describe("shared/subagents-format", () => { formatTokenUsageDisplay({ inputTokens: 1_200, outputTokens: 300, - totalTokens: 2_100, + ...freshUsage(2_100), }), ).toBe("tokens 1.5k (in 1.2k / out 300), prompt/cache 2.1k"); - expect(formatTokenUsageDisplay({ totalTokens: 500 })).toBe("tokens 500 prompt/cache"); + expect(formatTokenUsageDisplay(freshUsage(500))).toBe("tokens 500 prompt/cache"); expect( formatTokenUsageDisplay({ inputTokens: 1_200, outputTokens: 300, - totalTokens: 1_500, + ...freshUsage(1_500), }), ).toBe("tokens 1.5k (in 1.2k / out 300)"); expect(formatTokenUsageDisplay({ inputTokens: 0, outputTokens: 0, totalTokens: 0 })).toBe(""); diff --git a/src/shared/subagents-format.ts b/src/shared/subagents-format.ts index caeb2c5bb0ce..904dcec0b37c 100644 --- a/src/shared/subagents-format.ts +++ b/src/shared/subagents-format.ts @@ -41,6 +41,8 @@ export function truncateLine(value: string, maxLength: number) { type TokenUsageLike = { totalTokens?: unknown; + totalTokensFresh?: unknown; + totalTokensVersion?: unknown; inputTokens?: unknown; outputTokens?: unknown; }; @@ -50,7 +52,12 @@ export function resolveTotalTokens(entry?: TokenUsageLike) { if (!entry || typeof entry !== "object") { return undefined; } - if (typeof entry.totalTokens === "number" && Number.isFinite(entry.totalTokens)) { + if ( + typeof entry.totalTokens === "number" && + Number.isFinite(entry.totalTokens) && + entry.totalTokensFresh === true && + entry.totalTokensVersion === 1 + ) { return entry.totalTokens; } const input = typeof entry.inputTokens === "number" ? entry.inputTokens : 0; diff --git a/src/status/status-message.test.ts b/src/status/status-message.test.ts index ef09a7b35e0c..5122c0cccbea 100644 --- a/src/status/status-message.test.ts +++ b/src/status/status-message.test.ts @@ -112,6 +112,7 @@ describe("buildStatusMessageParts presentation", () => { sessionId: "status-meter-session", totalTokens: 87_000, totalTokensFresh: true, + totalTokensVersion: 1, compactionCount: 2, updatedAt: 1_751_529_500_000, }, @@ -169,6 +170,7 @@ describe("buildStatusMessage context window", () => { model: "deepseek-v4-pro", totalTokens: 128_393, totalTokensFresh: true, + totalTokensVersion: 1, }, sessionKey: "agent:main:telegram:direct:584667058", sessionScope: "per-sender", @@ -253,6 +255,7 @@ describe("buildStatusMessage context window", () => { model: "claude-haiku-4-5", totalTokens: 36_000, totalTokensFresh: true, + totalTokensVersion: 1, }, sessionKey: "agent:main:main", sessionScope: "per-sender", @@ -300,6 +303,7 @@ describe("buildStatusMessage context window", () => { model: "deepseek-v4-pro", totalTokens: 50_000, totalTokensFresh: true, + totalTokensVersion: 1, }, sessionKey: "agent:main:telegram:direct:auto-fallback", sessionScope: "per-sender", diff --git a/src/status/status-message.ts b/src/status/status-message.ts index a171f574520e..3befbc85b14b 100644 --- a/src/status/status-message.ts +++ b/src/status/status-message.ts @@ -670,9 +670,8 @@ export function buildStatusMessageParts(args: StatusArgs): StatusMessageParts { let cacheRead = entry?.cacheRead; let cacheWrite = entry?.cacheWrite; const freshTotalTokens = resolveFreshSessionTotalTokens(entry); - // Undefined freshness is legacy, not stale: keep persisted totals for /status, - // but let a fresh transcript prompt snapshot replace them when available. - const allowTranscriptContextUsage = entry?.totalTokensFresh !== false; + const allowTranscriptContextUsage = + entry?.totalTokensFresh !== false && freshTotalTokens === undefined; let totalTokens = freshTotalTokens; // Explicitly stale session/cache usage can still hydrate Tokens/Cache lines @@ -693,10 +692,7 @@ export function buildStatusMessageParts(args: StatusArgs): StatusMessageParts { allowTranscriptContextUsage && candidate !== undefined && candidate > 0 && - (entry?.totalTokensFresh !== true || - !totalTokens || - totalTokens === 0 || - candidate > totalTokens) + (!totalTokens || totalTokens === 0 || candidate > totalTokens) ) { totalTokens = candidate; } diff --git a/src/status/summary.ts b/src/status/summary.ts index 29d88f870387..621c3d77e423 100644 --- a/src/status/summary.ts +++ b/src/status/summary.ts @@ -11,7 +11,7 @@ import { } from "../config/sessions/model-override-provenance.js"; import { resolveStorePath } from "../config/sessions/paths.js"; import { listSessionEntriesReadOnly } from "../config/sessions/session-accessor.js"; -import { resolveSessionTotalTokens, type SessionEntry } from "../config/sessions/types.js"; +import { resolveFreshSessionTotalTokens, type SessionEntry } from "../config/sessions/types.js"; import type { OpenClawConfig } from "../config/types.js"; import { listGatewayAgentsBasic } from "../gateway/agent-list.js"; import { resolveHeartbeatSummaryForAgent } from "../infra/heartbeat-summary.js"; @@ -457,9 +457,8 @@ export async function getStatusSummary( fallbackContextTokens: configContextTokens ?? undefined, allowAsyncLoad: false, }) ?? null; - const total = resolveSessionTotalTokens(entry); - const totalTokensFresh = - typeof entry?.totalTokens === "number" ? entry?.totalTokensFresh !== false : false; + const total = resolveFreshSessionTotalTokens(entry); + const totalTokensFresh = total !== undefined; const remaining = contextTokens != null && total !== undefined ? Math.max(0, contextTokens - total) : null; const pct =