diff --git a/docs/api-reference.md b/docs/api-reference.md index 984e206a..59c63ce9 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -402,18 +402,22 @@ Each item in `items` (shared by `tool_info` and `approve_request`): "total_tokens": 1280, "context_window": 131072, "pct": 1.0, - "effort": "medium" + "effort": "medium", + "cache_creation_tokens": 800, + "cache_read_tokens": 200 } ``` -| Field | Type | Description | -|---------------------|--------|----------------------------------------------| -| `prompt_tokens` | int | Tokens in the prompt | -| `completion_tokens` | int | Tokens generated by the model | -| `total_tokens` | int | `prompt_tokens + completion_tokens` | -| `context_window` | int | Total context window size in tokens | -| `pct` | float | Percentage of context window used | -| `effort` | string | Reasoning effort level (`low`/`medium`/`high`) | +| Field | Type | Description | +|--------------------------|--------|------------------------------------------------------| +| `prompt_tokens` | int | Tokens in the prompt | +| `completion_tokens` | int | Tokens generated by the model | +| `total_tokens` | int | `prompt_tokens + completion_tokens` | +| `context_window` | int | Total context window size in tokens | +| `pct` | float | Percentage of context window used | +| `effort` | string | Reasoning effort level (`low`/`medium`/`high`) | +| `cache_creation_tokens` | int | Tokens written to prompt cache (Anthropic) | +| `cache_read_tokens` | int | Tokens served from prompt cache (Anthropic + OpenAI) | **`plan_review`** -- the model is proposing a plan and wants feedback. The client must respond via `POST /v1/api/plan`. diff --git a/docs/architecture.md b/docs/architecture.md index 496bc694..23fc6c2b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -596,7 +596,7 @@ LLMProvider (protocol) | `StreamChunk` | `content_delta`, `reasoning_delta`, `tool_call_deltas`, `info_delta`, `usage`, `finish_reason` | | `CompletionResult` | `content`, `tool_calls`, `finish_reason`, `usage` | | `ModelCapabilities` | `context_window`, `max_output_tokens`, `supports_temperature`, `token_param`, `thinking_mode`, `supports_effort`, `supports_web_search`, `supports_tool_search`, `supports_vision` | -| `UsageInfo` | `prompt_tokens`, `completion_tokens`, `total_tokens` | +| `UsageInfo` | `prompt_tokens`, `completion_tokens`, `total_tokens`, `cache_creation_tokens`, `cache_read_tokens` | **OpenAIProvider** (`_openai.py`): passes messages through unchanged (they are already in OpenAI format), including multi-part content blocks (text + images) @@ -604,7 +604,10 @@ in tool results. Model capability lookup table covers GPT-5/5.1/5.2/5.3/5.4, O-series, and search models (`gpt-5-search-api`) — all with `supports_vision`. For search models, injects `web_search_options` and removes the `web_search` function tool (the model always searches). Citations from `url_citation` -annotations are formatted as footnotes. Unknown models (local servers) get +annotations are formatted as footnotes. Extended prompt cache retention +(`prompt_cache_retention: "24h"`) is enabled for GPT-5.x models at no +additional cost. Cached token counts are extracted from +`usage.prompt_tokens_details.cached_tokens`. Unknown models (local servers) get permissive defaults with `supports_vision=False` and use Tavily for web search. **AnthropicProvider** (`_anthropic.py`): converts OpenAI-format messages to @@ -618,8 +621,14 @@ Sonnet 4.6. Replaces the `web_search` function tool with Anthropic's native `web_search_20250305` server-side tool — Claude decides when to search, the API executes it, and results stream back as `server_tool_use` / `web_search_tool_result` content blocks (emitted as `info_delta` for UI -display). The `anthropic` SDK is imported lazily so it remains an optional -dependency (`pip install turnstone[anthropic]`). +display). Automatic prompt caching is enabled via top-level `cache_control: +{"type": "ephemeral"}` — the API places the cache breakpoint on the last +cacheable block and advances it as conversations grow (90% input cost +reduction on cache hits, 1.25x write on first turn). Cache metrics +(`cache_creation_input_tokens`, `cache_read_input_tokens`) are extracted from +both streaming and non-streaming responses. The `anthropic` SDK is imported +lazily so it remains an optional dependency (`pip install +turnstone[anthropic]`). **Factory functions** (`__init__.py`): `create_provider(name)` returns a singleton provider instance (thread-safe). `create_client(name, base_url, diff --git a/docs/diagrams/03-core-engine-classes.puml b/docs/diagrams/03-core-engine-classes.puml index a336d502..d2b2afa1 100644 --- a/docs/diagrams/03-core-engine-classes.puml +++ b/docs/diagrams/03-core-engine-classes.puml @@ -84,6 +84,8 @@ class "OpenAIProvider" as OpenAIProv { in OpenAI format. Search models: web_search_options + url_citation annotations. + Extended cache: 24h retention + for GPT-5.x (free). -- core/providers/_openai.py } @@ -94,6 +96,8 @@ class "AnthropicProvider" as AnthropicProv { Adaptive + manual thinking. Native web search via web_search_20250305 server tool. + Auto prompt caching via + cache_control: ephemeral. Lazy anthropic SDK import. -- core/providers/_anthropic.py diff --git a/docs/diagrams/06-mq-protocol.puml b/docs/diagrams/06-mq-protocol.puml index 854e172f..cb0eddc3 100644 --- a/docs/diagrams/06-mq-protocol.puml +++ b/docs/diagrams/06-mq-protocol.puml @@ -3,8 +3,9 @@ title Turnstone — Message Queue Protocol Types skinparam classAttributeIconSize 0 skinparam packageStyle rectangle +skinparam packageBorderThickness 2 -package "Inbound Messages (Client → Bridge)" #FFF3E0 { +package "Inbound Messages (Client → Bridge)" as InPkg #FFF3E0 { abstract class "InboundMessage" as IM { + type: str @@ -99,7 +100,7 @@ package "Inbound Messages (Client → Bridge)" #FFF3E0 { IM <|-- CancelMessage } -package "Outbound Events (Bridge → Client)" #E3F2FD { +package "Outbound Events (Bridge → Client)" as OutPkg #E3F2FD { abstract class "OutboundEvent" as OE { + type: str @@ -167,6 +168,8 @@ package "Outbound Events (Bridge → Client)" #E3F2FD { + context_window: int + pct: float + effort: str + + cache_creation_tokens: int + + cache_read_tokens: int } class StateChangeEvent { type = "state_change" @@ -246,6 +249,8 @@ package "Outbound Events (Bridge → Client)" #E3F2FD { OE <|-- ClusterStateEvent } +SendMessage -[hidden]down- OE + note bottom of IM **Deserialization**: Strict type-dispatch via _INBOUND_REGISTRY. Unknown type raises ValueError. diff --git a/docs/diagrams/19-governance-architecture.puml b/docs/diagrams/19-governance-architecture.puml index afa242f2..ea280075 100644 --- a/docs/diagrams/19-governance-architecture.puml +++ b/docs/diagrams/19-governance-architecture.puml @@ -33,7 +33,7 @@ package "Governance Storage" { package "Runtime Enforcement" { [evaluate_tool_policies_batch()] as eval [WebUI.approve_tools()] as approve - [record_usage_event()] as usage + [record_usage_event()\n+cache_creation/read_tokens] as usage [record_audit()] as audit } diff --git a/docs/diagrams/png/03-core-engine-classes.png b/docs/diagrams/png/03-core-engine-classes.png index 33647da3..7c661874 100644 --- a/docs/diagrams/png/03-core-engine-classes.png +++ b/docs/diagrams/png/03-core-engine-classes.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:01fbb3338df6426cefc2811541a865f268673b4febf32f524c264d120bc068fa -size 589546 +oid sha256:0e605963c649574c7bf987b2d338257c78bc1fc68b524ac8276bb25365035e06 +size 594096 diff --git a/docs/diagrams/png/06-mq-protocol.png b/docs/diagrams/png/06-mq-protocol.png index e810b5f4..37430ca1 100644 --- a/docs/diagrams/png/06-mq-protocol.png +++ b/docs/diagrams/png/06-mq-protocol.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6e94a10f039a7f69517e84d0946e0c649035c15b38ebc2314e7b9cd501eb244d -size 192559 +oid sha256:06df9a7fa962755dd270c8f5a8b784041094a3ce44a473b653bb83112324ae07 +size 319204 diff --git a/docs/diagrams/png/19-governance-architecture.png b/docs/diagrams/png/19-governance-architecture.png index 671fbf91..d806e114 100644 --- a/docs/diagrams/png/19-governance-architecture.png +++ b/docs/diagrams/png/19-governance-architecture.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3aaca1ae4c6c255dc9569f59e3ccc24f8b3bab0ac2a9b08c85e2af72d6a400c7 -size 218575 +oid sha256:a2ff35e2b8a42e82ebae35273587268ac09a0d993cd3d0e8efd79845b955a1cd +size 221837 diff --git a/docs/governance.md b/docs/governance.md index 8e0a9ff3..57079bbd 100644 --- a/docs/governance.md +++ b/docs/governance.md @@ -110,9 +110,16 @@ Workstream templates are behavioral profiles applied at workstream creation — Per-LLM-request token and tool call metrics: - **Recording**: `on_status()` in `WebUI` records a `usage_event` after each - LLM response with prompt/completion tokens, tool call count, model, ws_id + LLM response with prompt/completion tokens, cache tokens, tool call count, + model, ws_id +- **Prompt caching**: Anthropic automatic caching (`cache_control: ephemeral`) + and OpenAI extended retention (`prompt_cache_retention: 24h` for GPT-5.x) + are enabled by default. `cache_creation_tokens` and `cache_read_tokens` are + tracked per request in `usage_events` and surfaced in the Usage admin tab - **Querying**: `GET /v1/api/admin/usage` with `group_by` (day/hour/model/user) - and time range filtering + and time range filtering — includes cache token aggregates +- **Prometheus**: `turnstone_tokens_total{type="cache_creation|cache_read"}` + counters on `/metrics` - **Pruning**: `prune_usage_events(retention_days=90)` and `prune_audit_events(retention_days=365)` run automatically via the console scheduler's periodic cleanup cycle @@ -140,7 +147,7 @@ Migration 008 adds 7 tables: | `user_roles` | User-to-role assignments (composite PK) | | `tool_policies` | Per-tool approve/deny/ask rules | | `prompt_templates` | Reusable system message templates | -| `usage_events` | Per-request token/tool metrics | +| `usage_events` | Per-request token/tool/cache metrics | | `audit_events` | Admin action log | Also adds `org_id` column to `users` table. diff --git a/docs/sdk.md b/docs/sdk.md index 2c23b196..ec2214f8 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -133,7 +133,7 @@ SSE events are deserialized into typed dataclasses. Use `event.type` to discrimi | `approve_request` | `ApproveRequestEvent` | `items` | | `tool_result` | `ToolResultEvent` | `call_id`, `name`, `output` | | `tool_output_chunk` | `ToolOutputChunkEvent` | `call_id`, `chunk` | -| `status` | `StatusEvent` | `prompt_tokens`, `total_tokens`, `pct`, `effort` | +| `status` | `StatusEvent` | `prompt_tokens`, `total_tokens`, `pct`, `effort`, `cache_creation_tokens`, `cache_read_tokens` | | `plan_review` | `PlanReviewEvent` | `content` | | `error` | `ErrorEvent` | `message` | | `info` | `InfoEvent` | `message` | diff --git a/sdk/typescript/openapi-console.json b/sdk/typescript/openapi-console.json index c8dd448f..6320a7dd 100644 --- a/sdk/typescript/openapi-console.json +++ b/sdk/typescript/openapi-console.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "turnstone Console API", - "version": "0.6.2", + "version": "0.7.0", "description": "Cluster-wide visibility and control across all turnstone nodes." }, "paths": { @@ -890,6 +890,64 @@ } } }, + "/v1/api/admin/users/{user_id}/oidc-identities": { + "get": { + "summary": "List OIDC identities linked to a user", + "operationId": "v1_api_admin_users_{user_id}_oidc-identities_get", + "tags": [ + "Admin" + ], + "parameters": [ + { + "name": "user_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + } + } + }, + "/v1/api/admin/oidc-identities": { + "delete": { + "summary": "Unlink an OIDC identity (issuer + subject as query params)", + "operationId": "v1_api_admin_oidc-identities_delete", + "tags": [ + "Admin" + ], + "responses": { + "200": { + "description": "Success" + }, + "400": { + "description": "Error 400", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "404": { + "description": "Error 404", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, "/v1/api/admin/schedules": { "get": { "summary": "List all scheduled tasks", @@ -2761,6 +2819,137 @@ } } }, + "/v1/api/admin/mcp-registry/search": { + "get": { + "summary": "Search the MCP Registry for available servers", + "operationId": "v1_api_admin_mcp-registry_search_get", + "tags": [ + "Admin" + ], + "parameters": [ + { + "name": "search", + "in": "query", + "required": false, + "schema": { + "type": "string" + }, + "description": "Search query" + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer" + }, + "description": "Max results (default 20, max 100)" + }, + { + "name": "cursor", + "in": "query", + "required": false, + "schema": { + "type": "string" + }, + "description": "Pagination cursor for next page" + } + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegistrySearchResponse" + } + } + } + }, + "502": { + "description": "Error 502", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, + "/v1/api/admin/mcp-registry/install": { + "post": { + "summary": "Install an MCP server from the registry", + "operationId": "v1_api_admin_mcp-registry_install_post", + "tags": [ + "Admin" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegistryInstallRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/McpServerDetail" + } + } + } + }, + "400": { + "description": "Error 400", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "404": { + "description": "Error 404", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "409": { + "description": "Error 409", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "502": { + "description": "Error 502", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, "/v1/api/admin/mcp-servers": { "get": { "summary": "List MCP server definitions with live status", @@ -5187,6 +5376,16 @@ "default": 0, "title": "Tool Calls Count", "type": "integer" + }, + "cache_creation_tokens": { + "default": 0, + "title": "Cache Creation Tokens", + "type": "integer" + }, + "cache_read_tokens": { + "default": 0, + "title": "Cache Read Tokens", + "type": "integer" } }, "title": "UsageBreakdownItem", @@ -5726,6 +5925,28 @@ "title": "Created By", "type": "string" }, + "registry_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Registry Name" + }, + "registry_version": { + "default": "", + "title": "Registry Version", + "type": "string" + }, + "registry_meta": { + "default": "{}", + "title": "Registry Meta", + "type": "string" + }, "created": { "title": "Created", "type": "string" @@ -6063,6 +6284,243 @@ "title": "McpReloadResponse", "type": "object" }, + "RegistrySearchResponse": { + "properties": { + "servers": { + "items": { + "$ref": "#/components/schemas/RegistryServerInfo" + }, + "title": "Servers", + "type": "array" + }, + "total": { + "default": 0, + "title": "Total", + "type": "integer" + }, + "next_cursor": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Next Cursor" + } + }, + "required": [ + "servers" + ], + "title": "RegistrySearchResponse", + "type": "object" + }, + "RegistryPackageInfo": { + "properties": { + "registry_type": { + "default": "", + "title": "Registry Type", + "type": "string" + }, + "identifier": { + "default": "", + "title": "Identifier", + "type": "string" + }, + "version": { + "default": "", + "title": "Version", + "type": "string" + }, + "transport_type": { + "default": "stdio", + "title": "Transport Type", + "type": "string" + }, + "environment_variables": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Environment Variables", + "type": "array" + } + }, + "title": "RegistryPackageInfo", + "type": "object" + }, + "RegistryRemoteInfo": { + "properties": { + "type": { + "default": "streamable-http", + "title": "Type", + "type": "string" + }, + "url": { + "default": "", + "title": "Url", + "type": "string" + }, + "headers": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Headers", + "type": "array" + }, + "variables": { + "additionalProperties": { + "additionalProperties": true, + "type": "object" + }, + "title": "Variables", + "type": "object" + } + }, + "title": "RegistryRemoteInfo", + "type": "object" + }, + "RegistryServerInfo": { + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "description": { + "default": "", + "title": "Description", + "type": "string" + }, + "title": { + "default": "", + "title": "Title", + "type": "string" + }, + "version": { + "default": "", + "title": "Version", + "type": "string" + }, + "website_url": { + "default": "", + "title": "Website Url", + "type": "string" + }, + "repository": { + "additionalProperties": { + "type": "string" + }, + "title": "Repository", + "type": "object" + }, + "icons": { + "items": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "title": "Icons", + "type": "array" + }, + "remotes": { + "items": { + "$ref": "#/components/schemas/RegistryRemoteInfo" + }, + "title": "Remotes", + "type": "array" + }, + "packages": { + "items": { + "$ref": "#/components/schemas/RegistryPackageInfo" + }, + "title": "Packages", + "type": "array" + }, + "meta": { + "additionalProperties": true, + "title": "Meta", + "type": "object" + }, + "installed": { + "default": false, + "title": "Installed", + "type": "boolean" + }, + "installed_server_id": { + "default": "", + "title": "Installed Server Id", + "type": "string" + }, + "installed_version": { + "default": "", + "title": "Installed Version", + "type": "string" + }, + "update_available": { + "default": false, + "title": "Update Available", + "type": "boolean" + } + }, + "required": [ + "name" + ], + "title": "RegistryServerInfo", + "type": "object" + }, + "RegistryInstallRequest": { + "properties": { + "registry_name": { + "title": "Registry Name", + "type": "string" + }, + "source": { + "title": "Source", + "type": "string" + }, + "index": { + "default": 0, + "title": "Index", + "type": "integer" + }, + "name": { + "default": "", + "title": "Name", + "type": "string" + }, + "variables": { + "additionalProperties": { + "type": "string" + }, + "title": "Variables", + "type": "object" + }, + "env": { + "additionalProperties": { + "type": "string" + }, + "title": "Env", + "type": "object" + }, + "headers": { + "additionalProperties": { + "type": "string" + }, + "title": "Headers", + "type": "object" + } + }, + "required": [ + "registry_name", + "source" + ], + "title": "RegistryInstallRequest", + "type": "object" + }, "PromptTemplateSummary": { "properties": { "name": { diff --git a/sdk/typescript/openapi-server.json b/sdk/typescript/openapi-server.json index 2f9d3b31..a6ea96da 100644 --- a/sdk/typescript/openapi-server.json +++ b/sdk/typescript/openapi-server.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "turnstone Server API", - "version": "0.6.2", + "version": "0.7.0", "description": "Single-node workstream management, chat interaction, and real-time streaming." }, "paths": { @@ -1181,7 +1181,7 @@ }, "always": { "default": false, - "description": "Enable auto-approve for this tool", + "description": "Auto-approve the tools in this batch going forward", "title": "Always", "type": "boolean" }, diff --git a/sdk/typescript/src/events.ts b/sdk/typescript/src/events.ts index f500ac20..68af7404 100644 --- a/sdk/typescript/src/events.ts +++ b/sdk/typescript/src/events.ts @@ -75,6 +75,8 @@ export interface StatusEvent { context_window: number; pct: number; effort: string; + cache_creation_tokens?: number; + cache_read_tokens?: number; } export interface PlanReviewEvent { diff --git a/tests/test_governance_storage.py b/tests/test_governance_storage.py index e84de9af..0c3aaa90 100644 --- a/tests/test_governance_storage.py +++ b/tests/test_governance_storage.py @@ -689,6 +689,78 @@ class TestUsageEvents: result = db.query_usage(since="2000-01-01T00:00:00") assert result[0]["prompt_tokens"] == 20 + def test_record_and_query_cache_tokens(self, db): + """Cache token columns are recorded and aggregated in query_usage.""" + db.record_usage_event( + "ev1", + model="claude-sonnet-4-6", + prompt_tokens=100, + completion_tokens=50, + cache_creation_tokens=80, + cache_read_tokens=0, + ) + db.record_usage_event( + "ev2", + model="claude-sonnet-4-6", + prompt_tokens=100, + completion_tokens=50, + cache_creation_tokens=0, + cache_read_tokens=80, + ) + result = db.query_usage(since="2000-01-01T00:00:00") + assert len(result) == 1 + assert result[0]["cache_creation_tokens"] == 80 + assert result[0]["cache_read_tokens"] == 80 + + def test_query_cache_tokens_grouped_by_model(self, db): + """Cache tokens are included in grouped query results.""" + from turnstone.core.storage._schema import usage_events + + with db._engine.connect() as conn: + conn.execute( + sa.insert(usage_events), + [ + { + "event_id": "e1", + "timestamp": "2026-03-01T10:00:00", + "user_id": "", + "ws_id": "", + "node_id": "", + "model": "claude-sonnet-4-6", + "prompt_tokens": 100, + "completion_tokens": 50, + "tool_calls_count": 0, + "cache_creation_tokens": 90, + "cache_read_tokens": 0, + "created": "2026-03-01T10:00:00", + }, + { + "event_id": "e2", + "timestamp": "2026-03-01T14:00:00", + "user_id": "", + "ws_id": "", + "node_id": "", + "model": "gpt-5.1", + "prompt_tokens": 200, + "completion_tokens": 100, + "tool_calls_count": 0, + "cache_creation_tokens": 0, + "cache_read_tokens": 150, + "created": "2026-03-01T14:00:00", + }, + ], + ) + conn.commit() + + result = db.query_usage(since="2026-03-01T00:00:00", group_by="model") + assert len(result) == 2 + claude = next(r for r in result if r["key"] == "claude-sonnet-4-6") + gpt = next(r for r in result if r["key"] == "gpt-5.1") + assert claude["cache_creation_tokens"] == 90 + assert claude["cache_read_tokens"] == 0 + assert gpt["cache_creation_tokens"] == 0 + assert gpt["cache_read_tokens"] == 150 + # --------------------------------------------------------------------------- # Audit Events diff --git a/tests/test_providers.py b/tests/test_providers.py index 184fb96a..96abbe85 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -2184,3 +2184,331 @@ class TestAnthropicVisionConversion: assert result[1]["type"] == "image" assert result[1]["source"]["media_type"] == "image/jpeg" assert result[1]["source"]["data"] == "/9j/4AAQ" + + +# =========================================================================== +# TestPromptCaching +# =========================================================================== + + +class TestAnthropicPromptCaching: + """Tests for Anthropic prompt caching (cache_control).""" + + def setup_method(self) -> None: + from turnstone.core.providers._anthropic import AnthropicProvider + + self.provider = AnthropicProvider() + + def test_cache_control_set_in_kwargs(self) -> None: + """_build_thinking_and_kwargs includes cache_control: ephemeral.""" + caps = self.provider.get_capabilities("claude-sonnet-4-6") + kwargs = self.provider._build_thinking_and_kwargs( + caps=caps, + reasoning_effort="medium", + extra_params=None, + max_tokens=4096, + temperature=0.5, + converted_msgs=[{"role": "user", "content": "hi"}], + system_prompt="You are helpful.", + model="claude-sonnet-4-6", + tools=None, + ) + assert "cache_control" in kwargs + assert kwargs["cache_control"] == {"type": "ephemeral"} + + @patch("turnstone.core.providers._anthropic._ensure_anthropic") + def test_streaming_message_start_cache_metrics(self, mock_ensure: MagicMock) -> None: + """Cache metrics from message_start flow into UsageInfo.""" + msg_start = MagicMock() + msg_start.type = "message_start" + msg_usage = MagicMock() + msg_usage.input_tokens = 100 + msg_usage.cache_creation_input_tokens = 80 + msg_usage.cache_read_input_tokens = 0 + msg_start.message = MagicMock() + msg_start.message.usage = msg_usage + + text_event = _anthropic_event("content_block_delta", delta_type="text_delta", text="Hi") + + events = [msg_start, text_event] + stream_ctx = MagicMock() + stream_ctx.__enter__ = MagicMock(return_value=iter(events)) + stream_ctx.__exit__ = MagicMock(return_value=False) + + client = MagicMock() + client.messages.stream.return_value = stream_ctx + + results = list( + self.provider.create_streaming( + client=client, + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "hi"}], + ) + ) + start_chunks = [r for r in results if r.usage is not None and r.usage.prompt_tokens == 100] + assert len(start_chunks) == 1 + assert start_chunks[0].usage is not None + assert start_chunks[0].usage.cache_creation_tokens == 80 + assert start_chunks[0].usage.cache_read_tokens == 0 + + @patch("turnstone.core.providers._anthropic._ensure_anthropic") + def test_streaming_message_delta_cache_metrics(self, mock_ensure: MagicMock) -> None: + """Cache metrics from message_delta flow into UsageInfo.""" + text_event = _anthropic_event("content_block_delta", delta_type="text_delta", text="Hi") + + delta_event = MagicMock() + delta_event.type = "message_delta" + delta_usage = MagicMock() + delta_usage.input_tokens = 0 + delta_usage.output_tokens = 50 + delta_usage.cache_creation_input_tokens = 0 + delta_usage.cache_read_input_tokens = 120 + delta_event.usage = delta_usage + delta_event.delta = MagicMock() + delta_event.delta.stop_reason = "end_turn" + + events = [text_event, delta_event] + stream_ctx = MagicMock() + stream_ctx.__enter__ = MagicMock(return_value=iter(events)) + stream_ctx.__exit__ = MagicMock(return_value=False) + + client = MagicMock() + client.messages.stream.return_value = stream_ctx + + results = list( + self.provider.create_streaming( + client=client, + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "hi"}], + ) + ) + delta_chunks = [r for r in results if r.finish_reason is not None] + assert len(delta_chunks) == 1 + u = delta_chunks[0].usage + assert u is not None + assert u.cache_read_tokens == 120 + assert u.cache_creation_tokens == 0 + + @patch("turnstone.core.providers._anthropic._ensure_anthropic") + def test_completion_cache_metrics(self, mock_ensure: MagicMock) -> None: + """Non-streaming completion extracts cache metrics.""" + response = MagicMock() + text_block = MagicMock() + text_block.type = "text" + text_block.text = "Hello" + response.content = [text_block] + response.stop_reason = "end_turn" + + usage = MagicMock() + usage.input_tokens = 200 + usage.output_tokens = 30 + usage.cache_creation_input_tokens = 150 + usage.cache_read_input_tokens = 50 + response.usage = usage + + client = MagicMock() + client.messages.create.return_value = response + + result = self.provider.create_completion( + client=client, + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "hi"}], + ) + u = result.usage + assert u is not None + assert u.cache_creation_tokens == 150 + assert u.cache_read_tokens == 50 + + @patch("turnstone.core.providers._anthropic._ensure_anthropic") + def test_streaming_cache_metrics_missing_gracefully(self, mock_ensure: MagicMock) -> None: + """When cache attributes are absent, tokens default to 0.""" + import types + + msg_start = MagicMock() + msg_start.type = "message_start" + # SimpleNamespace with only input_tokens — no cache attributes at all + msg_usage = types.SimpleNamespace(input_tokens=50) + msg_start.message = MagicMock() + msg_start.message.usage = msg_usage + + text_event = _anthropic_event("content_block_delta", delta_type="text_delta", text="Hi") + + events = [msg_start, text_event] + stream_ctx = MagicMock() + stream_ctx.__enter__ = MagicMock(return_value=iter(events)) + stream_ctx.__exit__ = MagicMock(return_value=False) + + client = MagicMock() + client.messages.stream.return_value = stream_ctx + + results = list( + self.provider.create_streaming( + client=client, + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "hi"}], + ) + ) + start_chunks = [r for r in results if r.usage is not None] + assert len(start_chunks) >= 1 + u = start_chunks[0].usage + assert u is not None + assert u.cache_creation_tokens == 0 + assert u.cache_read_tokens == 0 + + +class TestOpenAIPromptCaching: + """Tests for OpenAI prompt caching (automatic + extended retention).""" + + def setup_method(self) -> None: + self.provider = OpenAIProvider() + + def test_cache_retention_set_for_gpt5(self) -> None: + """GPT-5.x models get prompt_cache_retention=24h.""" + for model in ("gpt-5", "gpt-5.1", "gpt-5.2", "gpt-5.4", "gpt-5-mini", "gpt-5-pro"): + kwargs: dict[str, Any] = {} + self.provider._apply_cache_retention(kwargs, model) + assert kwargs.get("prompt_cache_retention") == "24h", f"Failed for {model}" + + def test_cache_retention_not_set_for_non_gpt5(self) -> None: + """Non-GPT-5 models do not get cache retention.""" + for model in ("o3", "o4-mini", "local-model", "gpt-4o"): + kwargs: dict[str, Any] = {} + self.provider._apply_cache_retention(kwargs, model) + assert "prompt_cache_retention" not in kwargs, f"Unexpected retention for {model}" + + def test_streaming_cached_tokens_from_usage(self) -> None: + """Streaming usage extracts cached_tokens from prompt_tokens_details.""" + usage = MagicMock() + usage.prompt_tokens = 100 + usage.completion_tokens = 20 + usage.total_tokens = 120 + ptd = MagicMock() + ptd.cached_tokens = 80 + usage.prompt_tokens_details = ptd + + chunks = [ + _openai_stream_chunk(content="Hi"), + _openai_stream_chunk(empty_choices=True, usage=usage), + ] + client = MagicMock() + client.chat.completions.create.return_value = iter(chunks) + + results = list( + self.provider.create_streaming( + client=client, + model="gpt-5.1", + messages=[{"role": "user", "content": "hi"}], + ) + ) + usage_chunks = [r for r in results if r.usage is not None] + assert len(usage_chunks) == 1 + u = usage_chunks[0].usage + assert u is not None + assert u.cache_read_tokens == 80 + assert u.cache_creation_tokens == 0 + + def test_completion_cached_tokens(self) -> None: + """Non-streaming completion extracts cached_tokens.""" + response = MagicMock() + msg = MagicMock() + msg.content = "Hello" + msg.tool_calls = None + msg.annotations = None + choice = MagicMock() + choice.message = msg + choice.finish_reason = "stop" + response.choices = [choice] + + usage = MagicMock() + usage.prompt_tokens = 200 + usage.completion_tokens = 30 + usage.total_tokens = 230 + ptd = MagicMock() + ptd.cached_tokens = 150 + usage.prompt_tokens_details = ptd + response.usage = usage + + client = MagicMock() + client.chat.completions.create.return_value = response + + result = self.provider.create_completion( + client=client, + model="gpt-5.1", + messages=[{"role": "user", "content": "hi"}], + ) + u = result.usage + assert u is not None + assert u.cache_read_tokens == 150 + assert u.cache_creation_tokens == 0 + + def test_streaming_no_prompt_tokens_details(self) -> None: + """When prompt_tokens_details is absent, cache_read_tokens defaults to 0.""" + usage = MagicMock() + usage.prompt_tokens = 100 + usage.completion_tokens = 20 + usage.total_tokens = 120 + usage.prompt_tokens_details = None + + chunks = [ + _openai_stream_chunk(content="Hi"), + _openai_stream_chunk(empty_choices=True, usage=usage), + ] + client = MagicMock() + client.chat.completions.create.return_value = iter(chunks) + + results = list( + self.provider.create_streaming( + client=client, + model="gpt-5.1", + messages=[{"role": "user", "content": "hi"}], + ) + ) + usage_chunks = [r for r in results if r.usage is not None] + assert len(usage_chunks) == 1 + u = usage_chunks[0].usage + assert u is not None + assert u.cache_read_tokens == 0 + + +class TestUsageInfoCacheFields: + """Tests for cache fields on UsageInfo dataclass.""" + + def test_default_cache_fields(self) -> None: + u = UsageInfo(prompt_tokens=10, completion_tokens=5, total_tokens=15) + assert u.cache_creation_tokens == 0 + assert u.cache_read_tokens == 0 + + def test_explicit_cache_fields(self) -> None: + u = UsageInfo( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150, + cache_creation_tokens=80, + cache_read_tokens=20, + ) + assert u.cache_creation_tokens == 80 + assert u.cache_read_tokens == 20 + + +class TestMetricsCacheTokens: + """Tests for cache token recording in MetricsCollector.""" + + def test_record_cache_tokens(self) -> None: + from turnstone.core.metrics import MetricsCollector + + m = MetricsCollector() + m.record_cache_tokens(100, 200) + m.record_cache_tokens(50, 300) + assert m._tokens["cache_creation"] == 150 + assert m._tokens["cache_read"] == 500 + + def test_prometheus_output_includes_cache_tokens(self) -> None: + from turnstone.core.metrics import MetricsCollector + + m = MetricsCollector() + m.record_tokens(1000, 500) + m.record_cache_tokens(800, 200) + text = m.generate_text(workstream_states={}, total_workstreams=0) + assert 'turnstone_tokens_total{type="cache_creation"} 800' in text + assert 'turnstone_tokens_total{type="cache_read"} 200' in text + assert 'turnstone_tokens_total{type="prompt"} 1000' in text diff --git a/turnstone/api/console_schemas.py b/turnstone/api/console_schemas.py index 567764a9..df6d9c08 100644 --- a/turnstone/api/console_schemas.py +++ b/turnstone/api/console_schemas.py @@ -421,6 +421,8 @@ class UsageBreakdownItem(BaseModel): prompt_tokens: int = 0 completion_tokens: int = 0 tool_calls_count: int = 0 + cache_creation_tokens: int = 0 + cache_read_tokens: int = 0 class UsageResponse(BaseModel): diff --git a/turnstone/console/static/governance.js b/turnstone/console/static/governance.js index 913dc6e7..7fb65168 100644 --- a/turnstone/console/static/governance.js +++ b/turnstone/console/static/governance.js @@ -1418,6 +1418,13 @@ function _renderGovUsage(summary, breakdown) { var completion = s.completion_tokens || 0; var total = prompt + completion; var tools = s.tool_calls_count || 0; + var cacheWrite = s.cache_creation_tokens || 0; + var cacheRead = s.cache_read_tokens || 0; + + var cacheZero = cacheWrite === 0 && cacheRead === 0; + var cacheCls = + "usage-readout usage-readout-secondary" + + (cacheZero ? " usage-readout-zero" : ""); var html = '
' + @@ -1433,6 +1440,17 @@ function _renderGovUsage(summary, breakdown) { '
' + formatCount(tools) + 'tool calls
' + + '
' + + '
' + + formatTokens(cacheWrite) + + 'cache write
' + + '
' + + formatTokens(cacheRead) + + 'cache read
' + "
"; // Bar chart breakdown diff --git a/turnstone/console/static/style.css b/turnstone/console/static/style.css index cfcbb9da..c5c5a1e7 100644 --- a/turnstone/console/static/style.css +++ b/turnstone/console/static/style.css @@ -1449,6 +1449,18 @@ letter-spacing: 0.08em; color: var(--fg-dim); } +/* Secondary readouts (cache stats) — smaller to denote supplementary metrics */ +.usage-readout-secondary .usage-readout-value { font-size: 16px; font-weight: 500; color: var(--fg-dim); } +.usage-readout-secondary .usage-readout-label { font-size: 9px; } +/* Dim zero-value secondary readouts to reduce noise */ +.usage-readout-zero { opacity: 0.35; } +/* Vertical divider between primary and secondary readout groups */ +.usage-summary-divider { + width: 1px; + align-self: stretch; + background: var(--border); + margin: 2px 0; +} /* Usage bar chart */ .usage-chart { padding-top: 4px; } @@ -1595,7 +1607,9 @@ .admin-toolbar-filters { flex-wrap: wrap; } .admin-toolbar-filters input[type="search"] { width: 120px; } .mem-detail-grid { grid-template-columns: 1fr 1fr; } + .usage-summary { gap: 14px; } .usage-readout-value { font-size: 18px; } + .usage-readout-secondary .usage-readout-value { font-size: 14px; } .usage-bar-row { grid-template-columns: 70px 1fr 50px; } .perm-grid { grid-template-columns: 1fr; } } diff --git a/turnstone/core/metrics.py b/turnstone/core/metrics.py index 6175e313..a38c1d76 100644 --- a/turnstone/core/metrics.py +++ b/turnstone/core/metrics.py @@ -64,6 +64,11 @@ class MetricsCollector: self._tokens["prompt"] += prompt self._tokens["completion"] += completion + def record_cache_tokens(self, cache_creation: int, cache_read: int) -> None: + with self._lock: + self._tokens["cache_creation"] += cache_creation + self._tokens["cache_read"] += cache_read + def record_tool_call(self, tool_name: str) -> None: with self._lock: self._tool_calls[tool_name] += 1 @@ -232,7 +237,7 @@ class MetricsCollector: # turnstone_tokens_total lines.append("# HELP turnstone_tokens_total Total tokens consumed") lines.append("# TYPE turnstone_tokens_total counter") - for tok_type in ("prompt", "completion"): + for tok_type in ("prompt", "completion", "cache_creation", "cache_read"): lines.append(f'turnstone_tokens_total{{type="{tok_type}"}} {tokens.get(tok_type, 0)}') # turnstone_tool_calls_total diff --git a/turnstone/core/providers/_anthropic.py b/turnstone/core/providers/_anthropic.py index 94df12ad..fe8d2983 100644 --- a/turnstone/core/providers/_anthropic.py +++ b/turnstone/core/providers/_anthropic.py @@ -252,6 +252,10 @@ class AnthropicProvider: "messages": converted_msgs, caps.token_param: max_tokens, "temperature": temperature, + # Automatic prompt caching — the API places the cache breakpoint + # on the last cacheable block and advances it as conversation grows. + # 90% input cost reduction on cache hits; 1.25x write on first turn. + "cache_control": {"type": "ephemeral"}, } if system_prompt: kwargs["system"] = system_prompt @@ -584,6 +588,8 @@ class AnthropicProvider: total_tokens=( getattr(u, "input_tokens", 0) + getattr(u, "output_tokens", 0) ), + cache_creation_tokens=getattr(u, "cache_creation_input_tokens", 0) or 0, + cache_read_tokens=getattr(u, "cache_read_input_tokens", 0) or 0, ) if hasattr(event.delta, "stop_reason") and event.delta.stop_reason: sc.finish_reason = _normalize_finish_reason(event.delta.stop_reason) @@ -598,6 +604,8 @@ class AnthropicProvider: prompt_tokens=getattr(u, "input_tokens", 0), completion_tokens=0, total_tokens=getattr(u, "input_tokens", 0), + cache_creation_tokens=getattr(u, "cache_creation_input_tokens", 0) or 0, + cache_read_tokens=getattr(u, "cache_read_input_tokens", 0) or 0, ) has_content = sc.content_delta or sc.reasoning_delta or sc.tool_call_deltas @@ -673,6 +681,8 @@ class AnthropicProvider: prompt_tokens=u.input_tokens, completion_tokens=u.output_tokens, total_tokens=u.input_tokens + u.output_tokens, + cache_creation_tokens=getattr(u, "cache_creation_input_tokens", 0) or 0, + cache_read_tokens=getattr(u, "cache_read_input_tokens", 0) or 0, ) return CompletionResult( diff --git a/turnstone/core/providers/_openai.py b/turnstone/core/providers/_openai.py index 608f839c..6a8b8c7d 100644 --- a/turnstone/core/providers/_openai.py +++ b/turnstone/core/providers/_openai.py @@ -234,6 +234,21 @@ class OpenAIProvider: kwargs["web_search_options"] = {} return tools + # -- prompt cache retention ----------------------------------------------- + + @staticmethod + def _apply_cache_retention(kwargs: dict[str, Any], model: str) -> None: + """Enable 24-hour extended prompt cache retention for GPT-5.x models. + + OpenAI caching is automatic (no code changes for basic caching), but + the default TTL is only 5-10 minutes. Extended retention keeps cached + KV tensors for up to 24 hours at no additional cost, which is valuable + for workstreams with bursty activity patterns. + """ + # GPT-5, GPT-5.1, GPT-5.2, GPT-5.3, GPT-5.4 and variants + if model.startswith("gpt-5"): + kwargs["prompt_cache_retention"] = "24h" + # -- tool search --------------------------------------------------------- def _apply_tool_search( @@ -282,6 +297,7 @@ class OpenAIProvider: "stream_options": {"include_usage": True}, } self._apply_model_params(kwargs, caps, temperature, reasoning_effort) + self._apply_cache_retention(kwargs, model) tools = self._apply_web_search(kwargs, caps, tools) tools = self._apply_tool_search(caps, tools, deferred_names) if tools: @@ -310,10 +326,16 @@ class OpenAIProvider: ct = getattr(u, "completion_tokens", None) tt = getattr(u, "total_tokens", None) if pt is not None and ct is not None: + # Extract cached_tokens from prompt_tokens_details. + # OpenAI caching is automatic with no write premium, so + # cache_creation_tokens is always 0 (only Anthropic reports it). + ptd = getattr(u, "prompt_tokens_details", None) + cached = getattr(ptd, "cached_tokens", 0) if ptd else 0 sc.usage = UsageInfo( prompt_tokens=pt, completion_tokens=ct, total_tokens=tt or (pt + ct), + cache_read_tokens=cached or 0, ) if not chunk.choices: @@ -387,6 +409,7 @@ class OpenAIProvider: "stream": False, } self._apply_model_params(kwargs, caps, temperature, reasoning_effort) + self._apply_cache_retention(kwargs, model) tools = self._apply_web_search(kwargs, caps, tools) tools = self._apply_tool_search(caps, tools, deferred_names) if tools: @@ -421,11 +444,14 @@ class OpenAIProvider: usage = None if hasattr(response, "usage") and response.usage: u = response.usage + ptd = getattr(u, "prompt_tokens_details", None) + cached = getattr(ptd, "cached_tokens", 0) if ptd else 0 usage = UsageInfo( prompt_tokens=u.prompt_tokens, completion_tokens=u.completion_tokens, total_tokens=getattr(u, "total_tokens", None) or (u.prompt_tokens + u.completion_tokens), + cache_read_tokens=cached or 0, ) return CompletionResult( diff --git a/turnstone/core/providers/_protocol.py b/turnstone/core/providers/_protocol.py index 537e8451..c01ab4c5 100644 --- a/turnstone/core/providers/_protocol.py +++ b/turnstone/core/providers/_protocol.py @@ -31,6 +31,9 @@ class UsageInfo: prompt_tokens: int completion_tokens: int total_tokens: int + # Prompt caching metrics (provider-specific; 0 when not available) + cache_creation_tokens: int = 0 + cache_read_tokens: int = 0 @dataclass diff --git a/turnstone/core/session.py b/turnstone/core/session.py index 6e0d452a..2bf9e532 100644 --- a/turnstone/core/session.py +++ b/turnstone/core/session.py @@ -1308,6 +1308,8 @@ class ChatSession: "prompt_tokens": chunk.usage.prompt_tokens, "completion_tokens": chunk.usage.completion_tokens, "total_tokens": chunk.usage.total_tokens, + "cache_creation_tokens": chunk.usage.cache_creation_tokens, + "cache_read_tokens": chunk.usage.cache_read_tokens, } else: self._last_usage["prompt_tokens"] = max( @@ -1320,6 +1322,14 @@ class ChatSession: self._last_usage["prompt_tokens"] + self._last_usage["completion_tokens"] ) + self._last_usage["cache_creation_tokens"] = max( + self._last_usage.get("cache_creation_tokens", 0), + chunk.usage.cache_creation_tokens, + ) + self._last_usage["cache_read_tokens"] = max( + self._last_usage.get("cache_read_tokens", 0), + chunk.usage.cache_read_tokens, + ) if self.debug: parts = [] diff --git a/turnstone/core/storage/_postgresql.py b/turnstone/core/storage/_postgresql.py index cfc40766..2a4cb2c6 100644 --- a/turnstone/core/storage/_postgresql.py +++ b/turnstone/core/storage/_postgresql.py @@ -1796,6 +1796,8 @@ class PostgreSQLBackend: prompt_tokens: int = 0, completion_tokens: int = 0, tool_calls_count: int = 0, + cache_creation_tokens: int = 0, + cache_read_tokens: int = 0, ) -> None: now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") with self._engine.connect() as conn: @@ -1811,6 +1813,8 @@ class PostgreSQLBackend: "prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "tool_calls_count": tool_calls_count, + "cache_creation_tokens": cache_creation_tokens, + "cache_read_tokens": cache_read_tokens, "created": now, }, ) @@ -1849,7 +1853,8 @@ class PostgreSQLBackend: # No grouping — single summary row sql = ( f"SELECT SUM(prompt_tokens), SUM(completion_tokens), " - f"SUM(tool_calls_count) FROM usage_events WHERE {where}" + f"SUM(tool_calls_count), SUM(cache_creation_tokens), " + f"SUM(cache_read_tokens) FROM usage_events WHERE {where}" ) with self._engine.connect() as conn: row = conn.execute(sa.text(sql), params).fetchone() @@ -1859,13 +1864,24 @@ class PostgreSQLBackend: "prompt_tokens": row[0] or 0, "completion_tokens": row[1] or 0, "tool_calls_count": row[2] or 0, + "cache_creation_tokens": row[3] or 0, + "cache_read_tokens": row[4] or 0, } ] - return [{"prompt_tokens": 0, "completion_tokens": 0, "tool_calls_count": 0}] + return [ + { + "prompt_tokens": 0, + "completion_tokens": 0, + "tool_calls_count": 0, + "cache_creation_tokens": 0, + "cache_read_tokens": 0, + } + ] sql = ( f"SELECT {key_expr} AS key, SUM(prompt_tokens), SUM(completion_tokens), " - f"SUM(tool_calls_count) FROM usage_events WHERE {where} " + f"SUM(tool_calls_count), SUM(cache_creation_tokens), " + f"SUM(cache_read_tokens) FROM usage_events WHERE {where} " f"GROUP BY {key_expr} ORDER BY key ASC" ) with self._engine.connect() as conn: @@ -1876,6 +1892,8 @@ class PostgreSQLBackend: "prompt_tokens": r[1] or 0, "completion_tokens": r[2] or 0, "tool_calls_count": r[3] or 0, + "cache_creation_tokens": r[4] or 0, + "cache_read_tokens": r[5] or 0, } for r in rows ] diff --git a/turnstone/core/storage/_protocol.py b/turnstone/core/storage/_protocol.py index 697f0952..767e5089 100644 --- a/turnstone/core/storage/_protocol.py +++ b/turnstone/core/storage/_protocol.py @@ -675,6 +675,8 @@ class StorageBackend(Protocol): prompt_tokens: int, completion_tokens: int, tool_calls_count: int, + cache_creation_tokens: int = 0, + cache_read_tokens: int = 0, ) -> None: """Record a usage event (token counts, tool calls for one LLM request).""" ... diff --git a/turnstone/core/storage/_schema.py b/turnstone/core/storage/_schema.py index 908977e7..96cda5ad 100644 --- a/turnstone/core/storage/_schema.py +++ b/turnstone/core/storage/_schema.py @@ -372,6 +372,8 @@ usage_events = sa.Table( sa.Column("prompt_tokens", sa.Integer, nullable=False, server_default="0"), sa.Column("completion_tokens", sa.Integer, nullable=False, server_default="0"), sa.Column("tool_calls_count", sa.Integer, nullable=False, server_default="0"), + sa.Column("cache_creation_tokens", sa.Integer, nullable=False, server_default="0"), + sa.Column("cache_read_tokens", sa.Integer, nullable=False, server_default="0"), sa.Column("created", sa.Text, nullable=False), ) diff --git a/turnstone/core/storage/_sqlite.py b/turnstone/core/storage/_sqlite.py index 6c72d5ca..428e0dca 100644 --- a/turnstone/core/storage/_sqlite.py +++ b/turnstone/core/storage/_sqlite.py @@ -1820,6 +1820,8 @@ class SQLiteBackend: prompt_tokens: int = 0, completion_tokens: int = 0, tool_calls_count: int = 0, + cache_creation_tokens: int = 0, + cache_read_tokens: int = 0, ) -> None: now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") with self._engine.connect() as conn: @@ -1835,6 +1837,8 @@ class SQLiteBackend: "prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "tool_calls_count": tool_calls_count, + "cache_creation_tokens": cache_creation_tokens, + "cache_read_tokens": cache_read_tokens, "created": now, }, ) @@ -1873,7 +1877,8 @@ class SQLiteBackend: # No grouping — single summary row sql = ( f"SELECT SUM(prompt_tokens), SUM(completion_tokens), " - f"SUM(tool_calls_count) FROM usage_events WHERE {where}" + f"SUM(tool_calls_count), SUM(cache_creation_tokens), " + f"SUM(cache_read_tokens) FROM usage_events WHERE {where}" ) with self._engine.connect() as conn: row = conn.execute(sa.text(sql), params).fetchone() @@ -1883,13 +1888,24 @@ class SQLiteBackend: "prompt_tokens": row[0] or 0, "completion_tokens": row[1] or 0, "tool_calls_count": row[2] or 0, + "cache_creation_tokens": row[3] or 0, + "cache_read_tokens": row[4] or 0, } ] - return [{"prompt_tokens": 0, "completion_tokens": 0, "tool_calls_count": 0}] + return [ + { + "prompt_tokens": 0, + "completion_tokens": 0, + "tool_calls_count": 0, + "cache_creation_tokens": 0, + "cache_read_tokens": 0, + } + ] sql = ( f"SELECT {key_expr} AS key, SUM(prompt_tokens), SUM(completion_tokens), " - f"SUM(tool_calls_count) FROM usage_events WHERE {where} " + f"SUM(tool_calls_count), SUM(cache_creation_tokens), " + f"SUM(cache_read_tokens) FROM usage_events WHERE {where} " f"GROUP BY {key_expr} ORDER BY key ASC" ) with self._engine.connect() as conn: @@ -1900,6 +1916,8 @@ class SQLiteBackend: "prompt_tokens": r[1] or 0, "completion_tokens": r[2] or 0, "tool_calls_count": r[3] or 0, + "cache_creation_tokens": r[4] or 0, + "cache_read_tokens": r[5] or 0, } for r in rows ] diff --git a/turnstone/core/storage/migrations/versions/020_usage_cache_tokens.py b/turnstone/core/storage/migrations/versions/020_usage_cache_tokens.py new file mode 100644 index 00000000..2daf6370 --- /dev/null +++ b/turnstone/core/storage/migrations/versions/020_usage_cache_tokens.py @@ -0,0 +1,33 @@ +"""Add prompt cache token columns to usage_events. + +Tracks cache_creation_tokens and cache_read_tokens per LLM call so +operators can monitor prompt caching effectiveness across providers. + +Revision ID: 020 +Revises: 019 +Create Date: 2026-03-16 +""" + +import sqlalchemy as sa +from alembic import op + +revision = "020" +down_revision = "019" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "usage_events", + sa.Column("cache_creation_tokens", sa.Integer, nullable=False, server_default="0"), + ) + op.add_column( + "usage_events", + sa.Column("cache_read_tokens", sa.Integer, nullable=False, server_default="0"), + ) + + +def downgrade() -> None: + op.drop_column("usage_events", "cache_read_tokens") + op.drop_column("usage_events", "cache_creation_tokens") diff --git a/turnstone/mq/bridge.py b/turnstone/mq/bridge.py index 9db73257..de2f2d59 100644 --- a/turnstone/mq/bridge.py +++ b/turnstone/mq/bridge.py @@ -626,6 +626,8 @@ class Bridge: context_window=data.get("context_window", 0), pct=data.get("pct", 0), effort=data.get("effort", ""), + cache_creation_tokens=data.get("cache_creation_tokens", 0), + cache_read_tokens=data.get("cache_read_tokens", 0), ), ) elif etype == "error": diff --git a/turnstone/mq/protocol.py b/turnstone/mq/protocol.py index 62163628..8d071978 100644 --- a/turnstone/mq/protocol.py +++ b/turnstone/mq/protocol.py @@ -251,6 +251,8 @@ class StatusEvent(OutboundEvent): context_window: int = 0 pct: float = 0.0 effort: str = "" + cache_creation_tokens: int = 0 + cache_read_tokens: int = 0 @dataclass diff --git a/turnstone/sdk/events.py b/turnstone/sdk/events.py index 525466be..d86b3896 100644 --- a/turnstone/sdk/events.py +++ b/turnstone/sdk/events.py @@ -125,6 +125,8 @@ class StatusEvent(ServerEvent): context_window: int = 0 pct: float = 0.0 effort: str = "" + cache_creation_tokens: int = 0 + cache_read_tokens: int = 0 @dataclass diff --git a/turnstone/server.py b/turnstone/server.py index c200ff31..ff174c46 100644 --- a/turnstone/server.py +++ b/turnstone/server.py @@ -384,7 +384,10 @@ class WebUI: def on_status(self, usage: dict[str, Any], context_window: int, effort: str) -> None: total_tok = usage["prompt_tokens"] + usage["completion_tokens"] pct = total_tok / context_window * 100 if context_window > 0 else 0 + cache_creation = usage.get("cache_creation_tokens", 0) + cache_read = usage.get("cache_read_tokens", 0) _metrics.record_tokens(usage["prompt_tokens"], usage["completion_tokens"]) + _metrics.record_cache_tokens(cache_creation, cache_read) _metrics.record_context_ratio(total_tok / context_window if context_window > 0 else 0.0) with self._ws_lock: self._ws_prompt_tokens += usage["prompt_tokens"] @@ -402,6 +405,8 @@ class WebUI: "context_window": context_window, "pct": round(pct, 1), "effort": effort, + "cache_creation_tokens": cache_creation, + "cache_read_tokens": cache_read, } ) # Record usage event for governance dashboard @@ -421,6 +426,8 @@ class WebUI: prompt_tokens=usage["prompt_tokens"], completion_tokens=usage["completion_tokens"], tool_calls_count=tool_count, + cache_creation_tokens=cache_creation, + cache_read_tokens=cache_read, ) except Exception: pass # Non-critical — never break the response pipeline