merge: update voice digest branch from main

* origin/main:
  fix(sqlite): skip slow-lock diagnostic when busyTimeoutMs is zero (#115990)
  fix(canvas): serve Content-Length on Canvas document HEAD responses (#117964)
  fix(workboard): diagnose archived cards still in an active status (#116359) (#117290)
  feat(diagnostics): configure OTEL metric name prefixes (#116687)
  fix(ui): respect UTC when filtering pending usage sessions (#118004)
This commit is contained in:
Vincent Koc
2026-08-02 22:41:10 +08:00
25 changed files with 555 additions and 108 deletions
+13
View File
@@ -90,6 +90,7 @@ stdout, or `both` for both.
logsEndpoint: "http://otel-collector:4318/v1/logs",
protocol: "http/protobuf", // grpc disables OTLP export
serviceName: "openclaw-gateway", // unset falls back to OTEL_SERVICE_NAME, then "openclaw"
metricNamePrefix: "acme.", // optional; include the separator
headers: { "x-collector-token": "..." },
traces: true,
metrics: true,
@@ -103,6 +104,18 @@ stdout, or `both` for both.
}
```
`metricNamePrefix` replaces the default `openclaw.` prefix only on
OpenClaw-owned metrics. For example, `"acme."` exports `openclaw.tokens` as
`acme.tokens`; set it to `""` to export `tokens` with no prefix. Non-empty
values must start with an ASCII letter, use only letters, digits, underscores,
dots, hyphens, and slashes, and contain at most 128 characters. Set it to
`"acme.openclaw."` if you want `acme.openclaw.tokens`. Standard
semantic-convention metrics such as
`gen_ai.client.token.usage` and `gen_ai.client.operation.duration` keep their
original names. Leave the option unset to preserve every current metric name.
Enabling or changing this option renames the affected metric series, so update
dashboards, alerts, and recording rules that query the old names.
### Environment variables
| Variable | Purpose |
+1
View File
@@ -375,6 +375,7 @@ Diagnostics are computed from local card metadata. Built-in checks flag:
| `repeated_failures` | Card's tracked failure count reaches 2 or more. |
| `missing_proof` | `done` card with no proof, artifacts, or attachments. |
| `orphaned_session` | `running` card with a `sessionKey` but no `execution` metadata. |
| `archived_but_active` | Archived card remains in any non-`done` lifecycle status. |
## Permissions
@@ -1,4 +1,4 @@
import type { Meter } from "@opentelemetry/api";
import type { Meter, MetricOptions } from "@opentelemetry/api";
import {
AGENT_DURATION_MS_BUCKETS,
CONTEXT_TOKENS_BUCKETS,
@@ -6,8 +6,20 @@ import {
GEN_AI_TOKEN_USAGE_BUCKETS,
} from "./service-constants.js";
export function createDiagnosticsMetrics(meter: Meter) {
const tokensCounter = meter.createCounter("openclaw.tokens", {
const DEFAULT_METRIC_NAME_PREFIX = "openclaw.";
export function createDiagnosticsMetrics(
meter: Meter,
metricNamePrefix = DEFAULT_METRIC_NAME_PREFIX,
) {
const resolveMetricName = (name: `openclaw.${string}`) =>
`${metricNamePrefix}${name.slice(DEFAULT_METRIC_NAME_PREFIX.length)}`;
const createCounter = (name: `openclaw.${string}`, options?: MetricOptions) =>
meter.createCounter(resolveMetricName(name), options);
const createHistogram = (name: `openclaw.${string}`, options?: MetricOptions) =>
meter.createHistogram(resolveMetricName(name), options);
const tokensCounter = createCounter("openclaw.tokens", {
unit: "1",
description: "Token usage by type",
});
@@ -28,266 +40,248 @@ export function createDiagnosticsMetrics(meter: Meter) {
},
},
);
const costCounter = meter.createCounter("openclaw.cost.usd", {
const costCounter = createCounter("openclaw.cost.usd", {
unit: "1",
description: "Estimated model cost (USD)",
});
const durationHistogram = meter.createHistogram("openclaw.run.duration_ms", {
const durationHistogram = createHistogram("openclaw.run.duration_ms", {
unit: "ms",
description: "Agent run duration",
advice: { explicitBucketBoundaries: AGENT_DURATION_MS_BUCKETS },
});
const harnessDurationHistogram = meter.createHistogram("openclaw.harness.duration_ms", {
const harnessDurationHistogram = createHistogram("openclaw.harness.duration_ms", {
unit: "ms",
description: "Agent harness lifecycle duration",
advice: { explicitBucketBoundaries: AGENT_DURATION_MS_BUCKETS },
});
const contextHistogram = meter.createHistogram("openclaw.context.tokens", {
const contextHistogram = createHistogram("openclaw.context.tokens", {
unit: "1",
description: "Context window size and usage",
advice: { explicitBucketBoundaries: CONTEXT_TOKENS_BUCKETS },
});
const webhookReceivedCounter = meter.createCounter("openclaw.webhook.received", {
const webhookReceivedCounter = createCounter("openclaw.webhook.received", {
unit: "1",
description: "Webhook requests received",
});
const webhookErrorCounter = meter.createCounter("openclaw.webhook.error", {
const webhookErrorCounter = createCounter("openclaw.webhook.error", {
unit: "1",
description: "Webhook processing errors",
});
const webhookDurationHistogram = meter.createHistogram("openclaw.webhook.duration_ms", {
const webhookDurationHistogram = createHistogram("openclaw.webhook.duration_ms", {
unit: "ms",
description: "Webhook processing duration",
});
const messageQueuedCounter = meter.createCounter("openclaw.message.queued", {
const messageQueuedCounter = createCounter("openclaw.message.queued", {
unit: "1",
description: "Messages queued for processing",
});
const messageReceivedCounter = meter.createCounter("openclaw.message.received", {
const messageReceivedCounter = createCounter("openclaw.message.received", {
unit: "1",
description: "Inbound messages received",
});
const messageDispatchStartedCounter = meter.createCounter("openclaw.message.dispatch.started", {
const messageDispatchStartedCounter = createCounter("openclaw.message.dispatch.started", {
unit: "1",
description: "Inbound message dispatch attempts started",
});
const messageDispatchCompletedCounter = meter.createCounter(
"openclaw.message.dispatch.completed",
{
unit: "1",
description: "Inbound message dispatch attempts completed",
},
);
const messageDispatchDurationHistogram = meter.createHistogram(
const messageDispatchCompletedCounter = createCounter("openclaw.message.dispatch.completed", {
unit: "1",
description: "Inbound message dispatch attempts completed",
});
const messageDispatchDurationHistogram = createHistogram(
"openclaw.message.dispatch.duration_ms",
{
unit: "ms",
description: "Inbound message dispatch duration",
},
);
const messageProcessedCounter = meter.createCounter("openclaw.message.processed", {
const messageProcessedCounter = createCounter("openclaw.message.processed", {
unit: "1",
description: "Messages processed by outcome",
});
const messageDurationHistogram = meter.createHistogram("openclaw.message.duration_ms", {
const messageDurationHistogram = createHistogram("openclaw.message.duration_ms", {
unit: "ms",
description: "Message processing duration",
});
const messageDeliveryStartedCounter = meter.createCounter("openclaw.message.delivery.started", {
const messageDeliveryStartedCounter = createCounter("openclaw.message.delivery.started", {
unit: "1",
description: "Outbound message delivery attempts started",
});
const messageDeliveryDurationHistogram = meter.createHistogram(
const messageDeliveryDurationHistogram = createHistogram(
"openclaw.message.delivery.duration_ms",
{
unit: "ms",
description: "Outbound message delivery duration",
},
);
const queueDepthHistogram = meter.createHistogram("openclaw.queue.depth", {
const queueDepthHistogram = createHistogram("openclaw.queue.depth", {
unit: "1",
description: "Queue depth on enqueue/dequeue",
});
const queueWaitHistogram = meter.createHistogram("openclaw.queue.wait_ms", {
const queueWaitHistogram = createHistogram("openclaw.queue.wait_ms", {
unit: "ms",
description: "Queue wait time before execution",
});
const laneEnqueueCounter = meter.createCounter("openclaw.queue.lane.enqueue", {
const laneEnqueueCounter = createCounter("openclaw.queue.lane.enqueue", {
unit: "1",
description: "Command queue lane enqueue events",
});
const laneDequeueCounter = meter.createCounter("openclaw.queue.lane.dequeue", {
const laneDequeueCounter = createCounter("openclaw.queue.lane.dequeue", {
unit: "1",
description: "Command queue lane dequeue events",
});
const sessionStateCounter = meter.createCounter("openclaw.session.state", {
const sessionStateCounter = createCounter("openclaw.session.state", {
unit: "1",
description: "Session state transitions",
});
const sessionTurnCreatedCounter = meter.createCounter("openclaw.session.turn.created", {
const sessionTurnCreatedCounter = createCounter("openclaw.session.turn.created", {
unit: "1",
description: "Agent session turns created",
});
const sessionStuckCounter = meter.createCounter("openclaw.session.stuck", {
const sessionStuckCounter = createCounter("openclaw.session.stuck", {
unit: "1",
description: "Sessions stuck in processing",
});
const sessionStuckAgeHistogram = meter.createHistogram("openclaw.session.stuck_age_ms", {
const sessionStuckAgeHistogram = createHistogram("openclaw.session.stuck_age_ms", {
unit: "ms",
description: "Age of stuck sessions",
});
const sessionRecoveryRequestedCounter = meter.createCounter(
"openclaw.session.recovery.requested",
{
unit: "1",
description: "Session recovery attempts requested",
},
);
const sessionRecoveryCompletedCounter = meter.createCounter(
"openclaw.session.recovery.completed",
{
unit: "1",
description: "Session recovery attempts completed",
},
);
const sessionRecoveryAgeHistogram = meter.createHistogram("openclaw.session.recovery.age_ms", {
const sessionRecoveryRequestedCounter = createCounter("openclaw.session.recovery.requested", {
unit: "1",
description: "Session recovery attempts requested",
});
const sessionRecoveryCompletedCounter = createCounter("openclaw.session.recovery.completed", {
unit: "1",
description: "Session recovery attempts completed",
});
const sessionRecoveryAgeHistogram = createHistogram("openclaw.session.recovery.age_ms", {
unit: "ms",
description: "Age of sessions selected for recovery",
});
const talkEventCounter = meter.createCounter("openclaw.talk.event", {
const talkEventCounter = createCounter("openclaw.talk.event", {
unit: "1",
description: "Talk events emitted by type",
});
const talkEventDurationHistogram = meter.createHistogram("openclaw.talk.event.duration_ms", {
const talkEventDurationHistogram = createHistogram("openclaw.talk.event.duration_ms", {
unit: "ms",
description: "Talk event duration when reported",
});
const talkAudioBytesHistogram = meter.createHistogram("openclaw.talk.audio.bytes", {
const talkAudioBytesHistogram = createHistogram("openclaw.talk.audio.bytes", {
unit: "By",
description: "Talk audio frame byte lengths",
});
const runAttemptCounter = meter.createCounter("openclaw.run.attempt", {
const runAttemptCounter = createCounter("openclaw.run.attempt", {
unit: "1",
description: "Run attempts",
});
const toolLoopCounter = meter.createCounter("openclaw.tool.loop", {
const toolLoopCounter = createCounter("openclaw.tool.loop", {
unit: "1",
description: "Detected repetitive tool-call loop events",
});
const skillUsedCounter = meter.createCounter("openclaw.skill.used", {
const skillUsedCounter = createCounter("openclaw.skill.used", {
unit: "1",
description: "Skills used by agent runs",
});
const modelCallDurationHistogram = meter.createHistogram("openclaw.model_call.duration_ms", {
const modelCallDurationHistogram = createHistogram("openclaw.model_call.duration_ms", {
unit: "ms",
description: "Model call duration",
});
const modelCallRequestBytesHistogram = meter.createHistogram(
"openclaw.model_call.request_bytes",
{
unit: "By",
description: "UTF-8 byte size of sanitized model request payloads",
},
);
const modelCallResponseBytesHistogram = meter.createHistogram(
"openclaw.model_call.response_bytes",
{
unit: "By",
description: "UTF-8 byte size of bounded streamed model response payloads",
},
);
const modelCallTimeToFirstByteHistogram = meter.createHistogram(
const modelCallRequestBytesHistogram = createHistogram("openclaw.model_call.request_bytes", {
unit: "By",
description: "UTF-8 byte size of sanitized model request payloads",
});
const modelCallResponseBytesHistogram = createHistogram("openclaw.model_call.response_bytes", {
unit: "By",
description: "UTF-8 byte size of bounded streamed model response payloads",
});
const modelCallTimeToFirstByteHistogram = createHistogram(
"openclaw.model_call.time_to_first_byte_ms",
{
unit: "ms",
description: "Elapsed time before the first streamed model response event",
},
);
const modelFailoverCounter = meter.createCounter("openclaw.model.failover", {
const modelFailoverCounter = createCounter("openclaw.model.failover", {
unit: "1",
description: "Model failovers by source, destination, lane, and reason",
});
const toolExecutionDurationHistogram = meter.createHistogram(
"openclaw.tool.execution.duration_ms",
{
unit: "ms",
description: "Tool execution duration",
},
);
const toolExecutionBlockedCounter = meter.createCounter("openclaw.tool.execution.blocked", {
const toolExecutionDurationHistogram = createHistogram("openclaw.tool.execution.duration_ms", {
unit: "ms",
description: "Tool execution duration",
});
const toolExecutionBlockedCounter = createCounter("openclaw.tool.execution.blocked", {
unit: "1",
description: "Tool executions blocked by policy or sandbox diagnostics",
});
const execProcessDurationHistogram = meter.createHistogram("openclaw.exec.duration_ms", {
const execProcessDurationHistogram = createHistogram("openclaw.exec.duration_ms", {
unit: "ms",
description: "Exec process duration",
});
const memoryRssHistogram = meter.createHistogram("openclaw.memory.rss_bytes", {
const memoryRssHistogram = createHistogram("openclaw.memory.rss_bytes", {
unit: "By",
description: "Resident set size reported by diagnostic memory samples",
});
const memoryHeapUsedHistogram = meter.createHistogram("openclaw.memory.heap_used_bytes", {
const memoryHeapUsedHistogram = createHistogram("openclaw.memory.heap_used_bytes", {
unit: "By",
description: "Heap used bytes reported by diagnostic memory samples",
});
const memoryHeapTotalHistogram = meter.createHistogram("openclaw.memory.heap_total_bytes", {
const memoryHeapTotalHistogram = createHistogram("openclaw.memory.heap_total_bytes", {
unit: "By",
description: "Heap total bytes reported by diagnostic memory samples",
});
const memoryExternalHistogram = meter.createHistogram("openclaw.memory.external_bytes", {
const memoryExternalHistogram = createHistogram("openclaw.memory.external_bytes", {
unit: "By",
description: "External memory bytes reported by diagnostic memory samples",
});
const memoryArrayBuffersHistogram = meter.createHistogram("openclaw.memory.array_buffers_bytes", {
const memoryArrayBuffersHistogram = createHistogram("openclaw.memory.array_buffers_bytes", {
unit: "By",
description: "ArrayBuffer bytes reported by diagnostic memory samples",
});
const memoryPressureCounter = meter.createCounter("openclaw.memory.pressure", {
const memoryPressureCounter = createCounter("openclaw.memory.pressure", {
unit: "1",
description: "Diagnostic memory pressure events",
});
const asyncQueueDroppedCounter = meter.createCounter("openclaw.diagnostic.async_queue.dropped", {
const asyncQueueDroppedCounter = createCounter("openclaw.diagnostic.async_queue.dropped", {
unit: "1",
description: "Async diagnostic queue drops by dropped event class",
});
const payloadLargeCounter = meter.createCounter("openclaw.payload.large", {
const payloadLargeCounter = createCounter("openclaw.payload.large", {
unit: "1",
description: "Oversized payload diagnostics by surface and action",
});
const payloadLargeBytesHistogram = meter.createHistogram("openclaw.payload.large_bytes", {
const payloadLargeBytesHistogram = createHistogram("openclaw.payload.large_bytes", {
unit: "By",
description: "Oversized payload byte sizes by surface and action",
});
const livenessWarningCounter = meter.createCounter("openclaw.liveness.warning", {
const livenessWarningCounter = createCounter("openclaw.liveness.warning", {
unit: "1",
description: "Diagnostic liveness warning events",
});
const livenessEventLoopDelayP99Histogram = meter.createHistogram(
const livenessEventLoopDelayP99Histogram = createHistogram(
"openclaw.liveness.event_loop_delay_p99_ms",
{
unit: "ms",
description: "P99 event-loop delay reported by diagnostic liveness warnings",
},
);
const livenessEventLoopDelayMaxHistogram = meter.createHistogram(
const livenessEventLoopDelayMaxHistogram = createHistogram(
"openclaw.liveness.event_loop_delay_max_ms",
{
unit: "ms",
description: "Maximum event-loop delay reported by diagnostic liveness warnings",
},
);
const livenessEventLoopUtilizationHistogram = meter.createHistogram(
const livenessEventLoopUtilizationHistogram = createHistogram(
"openclaw.liveness.event_loop_utilization",
{
unit: "1",
description: "Event-loop utilization reported by diagnostic liveness warnings",
},
);
const livenessCpuCoreRatioHistogram = meter.createHistogram("openclaw.liveness.cpu_core_ratio", {
const livenessCpuCoreRatioHistogram = createHistogram("openclaw.liveness.cpu_core_ratio", {
unit: "1",
description: "CPU core ratio reported by diagnostic liveness warnings",
});
const telemetryExporterCounter = meter.createCounter("openclaw.telemetry.exporter.events", {
const telemetryExporterCounter = createCounter("openclaw.telemetry.exporter.events", {
unit: "1",
description: "Diagnostic telemetry exporter lifecycle and failure events",
});
@@ -705,6 +705,49 @@ describe("diagnostics-otel service", () => {
}
});
test.each([
{
metricNamePrefix: undefined,
expectedTokenName: "openclaw.tokens",
expectedDurationName: "openclaw.run.duration_ms",
},
{
metricNamePrefix: "acme.",
expectedTokenName: "acme.tokens",
expectedDurationName: "acme.run.duration_ms",
},
{
metricNamePrefix: "",
expectedTokenName: "tokens",
expectedDurationName: "run.duration_ms",
},
{
metricNamePrefix: "acme.openclaw.",
expectedTokenName: "acme.openclaw.tokens",
expectedDurationName: "acme.openclaw.run.duration_ms",
},
])(
"replaces the default OpenClaw metric prefix with $metricNamePrefix",
async ({ metricNamePrefix, expectedTokenName, expectedDurationName }) => {
await startOtelService({
metrics: true,
configure: (ctx) => {
if (metricNamePrefix !== undefined) {
ctx.config.diagnostics!.otel!.metricNamePrefix = metricNamePrefix;
}
},
});
expect(telemetryState.counters.has(expectedTokenName)).toBe(true);
expect(telemetryState.histograms.has(expectedDurationName)).toBe(true);
expect(telemetryState.histograms.has("gen_ai.client.token.usage")).toBe(true);
expect(telemetryState.histograms.has("gen_ai.client.operation.duration")).toBe(true);
expect(telemetryState.counters.has("openclaw.tokens")).toBe(
expectedTokenName === "openclaw.tokens",
);
},
);
test("records message-flow metrics and spans", async () => {
await startOtelService({ traces: true, metrics: true, logs: true });
+1 -1
View File
@@ -257,7 +257,7 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
const tracer = trace.getTracer("openclaw");
const diagnosticsTrace = createDiagnosticsTraceRuntime(tracer);
stopActiveTrustedSpans = diagnosticsTrace.stopActiveTrustedSpans;
const diagnosticMetrics = createDiagnosticsMetrics(meter);
const diagnosticMetrics = createDiagnosticsMetrics(meter, otel.metricNamePrefix);
const diagnosticsLogs = createDiagnosticsLogExporter({
contentCapturePolicy,
+14
View File
@@ -140,6 +140,20 @@ describe("registerWorkboardCli", () => {
expect(defaultOutput).not.toContain("Archived card");
expect(includeOutput).toContain("Active card");
expect(includeOutput).toContain("Archived card");
expect(includeOutput).toContain("(archived)");
});
it("marks archived cards in show output", async () => {
const store = new WorkboardStore(createMemoryStore());
const archived = await store.create({ title: "Archived card", status: "ready" });
await store.archive(archived.id, true);
const program = createProgram(store);
const output = await captureStdout(async () => {
await program.parseAsync(["workboard", "show", archived.id], { from: "user" });
});
expect(output).toContain("Archived card (archived)");
});
it("preserves archived cards in JSON list output by default", async () => {
+2 -1
View File
@@ -69,7 +69,8 @@ function isWorkboardStatus(value: string): value is WorkboardStatus {
function formatCardLine(card: WorkboardCard): string {
const boardId = card.metadata?.automation?.boardId ?? "default";
const agent = card.agentId ? ` ${card.agentId}` : "";
return `${card.id.slice(0, 8)} ${card.status.padEnd(8)} ${card.priority.padEnd(6)} ${boardId}${agent} ${card.title}`;
const archived = card.metadata?.archivedAt ? " (archived)" : "";
return `${card.id.slice(0, 8)} ${card.status.padEnd(8)} ${card.priority.padEnd(6)} ${boardId}${agent} ${card.title}${archived}`;
}
function redactDispatchResult(result: WorkboardDispatchResult): WorkboardDispatchResult {
+13
View File
@@ -291,6 +291,19 @@ describe("handleWorkboardCommand", () => {
await expect(store.get(card.id)).resolves.toMatchObject({ status: "ready" });
});
it("shows when an archived card is excluded from dispatch", async () => {
const store = new WorkboardStore(createMemoryStore());
const api = createApi();
const card = await store.create({ title: "Archived slash card", status: "ready" });
await store.archive(card.id, true);
await expect(runWorkboardCommand({ api, store, args: `show ${card.id}` })).resolves.toEqual(
expect.objectContaining({
text: expect.stringContaining("archived: yes (excluded from dispatch)"),
}),
);
});
it("moves claimed cards for operators on slash-command surfaces", async () => {
const store = new WorkboardStore(createMemoryStore());
const api = createApi();
+3
View File
@@ -58,6 +58,9 @@ function formatCardDetails(card: WorkboardCard): string {
if (card.runId) {
lines.push(`run: ${card.runId}`);
}
if (card.metadata?.archivedAt) {
lines.push("archived: yes (excluded from dispatch)");
}
if (card.notes) {
lines.push("", card.notes);
}
@@ -395,6 +395,22 @@ export function mergeDiagnostics(
export function computeCardDiagnostics(card: WorkboardCard, now: number): WorkboardDiagnostic[] {
if (card.metadata?.archivedAt) {
// Archived cards intentionally skip automation. Keep nonterminal cards
// visible as a transient diagnostic without rewriting archived metadata.
if (card.status !== "done") {
return [
diagnostic(
{
kind: "archived_but_active",
severity: "warning",
title: "Archived card is still in an active status",
detail: `Card status is "${card.status}" but it is archived, so it is excluded from dispatch without any start failure or error. Unarchive it or move it to "done" to stop the silent skip.`,
actions: [],
},
now,
),
];
}
return [];
}
const diagnostics: WorkboardDiagnostic[] = [];
+54
View File
@@ -3,6 +3,7 @@ import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { DatabaseSync } from "node:sqlite";
import { WORKBOARD_STATUSES } from "@openclaw/workboard-contract";
import { MAX_DATE_TIMESTAMP_MS } from "openclaw/plugin-sdk/number-runtime";
import { describe, expect, it, vi } from "vitest";
import type {
@@ -2618,6 +2619,59 @@ describe("WorkboardStore", () => {
});
});
it.each(WORKBOARD_STATUSES)(
"reports archived %s cards according to terminal state",
async (status) => {
const store = new WorkboardStore(createMemoryStore());
const card = await store.create({ title: `Archived ${status}`, status });
await store.archive(card.id, true);
const result = await store.diagnostics(Date.now());
if (status === "done") {
expect(result).toEqual({ diagnostics: [], count: 0 });
return;
}
expect(result).toMatchObject({
diagnostics: [
expect.objectContaining({
card: expect.objectContaining({ id: card.id }),
diagnostics: [
expect.objectContaining({
kind: "archived_but_active",
severity: "warning",
actions: [],
}),
],
}),
],
count: 1,
});
},
);
it("keeps archived-card diagnostics transient across lifecycle changes", async () => {
const store = new WorkboardStore(createMemoryStore());
const card = await store.create({ title: "Archived but ready", status: "ready" });
const now = Date.now();
await store.archive(card.id, true);
await expect(store.refreshDiagnostics(now)).resolves.toEqual({ diagnostics: [], count: 0 });
await expect(store.get(card.id)).resolves.not.toHaveProperty("metadata.diagnostics");
await expect(store.diagnostics(now)).resolves.toMatchObject({
diagnostics: [expect.objectContaining({ card: expect.objectContaining({ id: card.id }) })],
count: 1,
});
await store.archive(card.id, false);
await expect(store.diagnostics(now + 1)).resolves.toEqual({ diagnostics: [], count: 0 });
await store.archive(card.id, true);
await store.move(card.id, "done", undefined);
await expect(store.diagnostics(now + 2)).resolves.toEqual({ diagnostics: [], count: 0 });
});
it("does not drop concurrent updates while refreshing diagnostics", async () => {
let proofPromise: Promise<unknown> | undefined;
let triggered = false;
+1
View File
@@ -71,6 +71,7 @@ export const WORKBOARD_DIAGNOSTIC_KINDS = [
"repeated_failures",
"missing_proof",
"orphaned_session",
"archived_but_active",
] as const;
export const WORKBOARD_DIAGNOSTIC_SEVERITIES = ["warning", "error", "critical"] as const;
export const WORKBOARD_NOTIFICATION_KINDS = ["completed", "failed", "stale"] as const;
+55 -2
View File
@@ -1,10 +1,10 @@
// Core Canvas document HTTP response coverage.
import { mkdtemp, rm } from "node:fs/promises";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import type { IncomingMessage, ServerResponse } from "node:http";
import { tmpdir } from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { createCanvasDocument } from "./documents.js";
import { createCanvasDocument, resolveCanvasDocumentsDir } from "./documents.js";
import { handleCanvasDocumentHttpRequest } from "./serve.runtime.js";
const tempDirs: string[] = [];
@@ -81,6 +81,59 @@ describe("core canvas document host", () => {
expect(response.headers["content-security-policy"]).toBeUndefined();
});
it("serves Content-Length on HEAD responses", async () => {
const stateDir = await createStateDir();
const html = "<html><body>widget</body></html>";
const document = await createCanvasDocument(
{
id: "widget-1",
kind: "html_bundle",
entrypoint: { type: "html", value: html },
cspSandbox: "scripts",
},
{ stateDir },
);
const css = "body { color: red; }";
await writeFile(
path.join(resolveCanvasDocumentsDir(stateDir), "widget-1", "style.css"),
css,
"utf8",
);
const cssUrl = document.entryUrl.replace(/index\.html$/, "style.css");
const getHtml = await capture(document.entryUrl);
const headHtml = await capture(document.entryUrl, "HEAD");
expect(headHtml.statusCode).toBe(200);
expect(headHtml.headers["content-length"]).toBe(String(getHtml.body.byteLength));
expect(headHtml.body.byteLength).toBe(0);
expect(headHtml.headers["content-type"]).toBe("text/html; charset=utf-8");
expect(headHtml.headers["content-security-policy"]).toBe("sandbox allow-scripts");
const getCss = await capture(cssUrl);
const headCss = await capture(cssUrl, "HEAD");
expect(getCss.statusCode).toBe(200);
expect(headCss.statusCode).toBe(200);
expect(headCss.headers["content-length"]).toBe(String(getCss.body.byteLength));
expect(headCss.body.byteLength).toBe(0);
// Invalid UTF-8 in a copied HTML file expands to U+FFFD when served, so the
// header must be measured from the decoded representation, not raw bytes.
const brokenBytes = Buffer.from([
0x3c, 0x68, 0x31, 0x3e, 0xff, 0xfe, 0x3c, 0x2f, 0x68, 0x31, 0x3e,
]);
await writeFile(
path.join(resolveCanvasDocumentsDir(stateDir), "widget-1", "broken.html"),
brokenBytes,
);
const brokenUrl = document.entryUrl.replace(/index\.html$/, "broken.html");
const getBroken = await capture(brokenUrl);
const headBroken = await capture(brokenUrl, "HEAD");
expect(getBroken.statusCode).toBe(200);
expect(getBroken.body.byteLength).toBeGreaterThan(brokenBytes.byteLength);
expect(headBroken.headers["content-length"]).toBe(String(getBroken.body.byteLength));
expect(headBroken.body.byteLength).toBe(0);
});
it("rejects unsupported methods and traversal paths", async () => {
await createStateDir();
const methodResponse = await capture(
+14 -1
View File
@@ -80,14 +80,27 @@ export async function handleCanvasDocumentHttpRequest(
: ((await detectMime({ filePath: opened.realPath })) ?? "application/octet-stream");
res.setHeader("Cache-Control", "no-store");
if (mime === "text/html") {
// Measure the decoded representation: toString("utf8") expands invalid
// bytes to U+FFFD, so the raw file length can differ from the body sent.
const body = opened.data.toString("utf8");
res.setHeader("Content-Type", "text/html; charset=utf-8");
res.setHeader("Content-Length", String(Buffer.byteLength(body)));
if ((await resolveDocumentSandbox(root, relativePath)) === "scripts") {
res.setHeader("Content-Security-Policy", "sandbox allow-scripts");
}
res.end(opened.data.toString("utf8"));
if (req.method === "HEAD") {
res.end();
return true;
}
res.end(body);
return true;
}
res.setHeader("Content-Type", mime);
res.setHeader("Content-Length", String(opened.data.byteLength));
if (req.method === "HEAD") {
res.end();
return true;
}
res.end(opened.data);
return true;
} catch (error) {
+18
View File
@@ -483,6 +483,24 @@ describe("diagnostics.otel.captureContent", () => {
});
});
describe("diagnostics.otel.metricNamePrefix", () => {
it("accepts valid metric name fragments and rejects invalid values", () => {
for (const metricNamePrefix of ["", "acme.", "Acme/team-1_"]) {
const result = OpenClawSchema.safeParse({
diagnostics: { otel: { metricNamePrefix } },
});
expect(result.success).toBe(true);
}
for (const metricNamePrefix of [42, " ", ".acme", "acme metrics.", "é.", "a".repeat(129)]) {
const result = OpenClawSchema.safeParse({
diagnostics: { otel: { metricNamePrefix } },
});
expect(result.success).toBe(false);
}
});
});
describe("ui.seamColor", () => {
it("accepts hex colors", () => {
const res = validateConfigObject({ ui: { seamColor: "#FF4500" } });
+2
View File
@@ -330,6 +330,8 @@ export const RUNTIME_FIELD_HELP: Record<string, string> = {
"Additional HTTP/gRPC metadata headers sent with OpenTelemetry export requests, often used for tenant auth or routing. Keep secrets in env-backed values and avoid unnecessary header sprawl.",
"diagnostics.otel.serviceName":
"Service name reported in telemetry resource attributes to identify this gateway instance in observability backends. Use stable names so dashboards and alerts remain consistent over deployments.",
"diagnostics.otel.metricNamePrefix":
'Replaces the default "openclaw." prefix on OpenClaw-owned metric names. Use an empty string to remove the prefix, or up to 128 ASCII letters, digits, underscores, dots, hyphens, and slashes starting with a letter. Include any separator you need, for example "acme."; standard gen_ai.* metric names are unchanged. Changing this value requires updating dashboards and alerts that query the old names.',
"diagnostics.otel.traces":
"Enable trace signal export to the configured OpenTelemetry collector endpoint. Keep enabled when latency/debug tracing is needed, and disable if you only want metrics/logs.",
"diagnostics.otel.metrics":
+1
View File
@@ -54,6 +54,7 @@ export const FIELD_LABELS: Record<string, string> = {
"diagnostics.otel.protocol": "OpenTelemetry Protocol",
"diagnostics.otel.headers": "OpenTelemetry Headers",
"diagnostics.otel.serviceName": "OpenTelemetry Service Name",
"diagnostics.otel.metricNamePrefix": "OpenTelemetry Metric Name Prefix",
"diagnostics.otel.traces": "OpenTelemetry Traces Enabled",
"diagnostics.otel.metrics": "OpenTelemetry Metrics Enabled",
"diagnostics.otel.logs": "OpenTelemetry Logs Enabled",
+2
View File
@@ -296,6 +296,8 @@ export type DiagnosticsOtelConfig = {
protocol?: "http/protobuf" | "grpc";
headers?: Record<string, string>;
serviceName?: string;
/** Replacement prefix for OpenClaw-owned metric names. Empty removes the prefix; defaults to "openclaw.". */
metricNamePrefix?: string;
traces?: boolean;
metrics?: boolean;
logs?: boolean;
+8
View File
@@ -32,6 +32,13 @@ import {
import { sensitive } from "./zod-schema.sensitive.js";
import { CommandsSchema, MessagesSchema, SessionSchema } from "./zod-schema.session.js";
// OpenTelemetry instrument names start with an ASCII letter and allow only these characters.
// The 128-character prefix cap leaves ample room within the dependency's 255-character name cap.
const MetricNamePrefixSchema = z
.string()
.max(128)
.regex(/^(?:[A-Za-z][A-Za-z0-9_./-]*)?$/);
export const OpenClawSchemaShape = {
$schema: z.string().optional(),
meta: z
@@ -83,6 +90,7 @@ export const OpenClawSchemaShape = {
protocol: z.union([z.literal("http/protobuf"), z.literal("grpc")]).optional(),
headers: z.record(z.string(), z.string()).optional(),
serviceName: z.string().optional(),
metricNamePrefix: MetricNamePrefixSchema.optional(),
traces: z.boolean().optional(),
metrics: z.boolean().optional(),
logs: z.boolean().optional(),
+54
View File
@@ -263,6 +263,60 @@ describe("runSqliteImmediateTransactionSync", () => {
);
});
it("does not warn for busyTimeoutMs: 0 with fast successful transactions (regression)", () => {
const logger = { warn: vi.fn() };
let now = 0;
vi.spyOn(Date, "now").mockImplementation(() => {
const value = now;
now += 5; // Fast: 5ms per step, well under the 1000ms default threshold
return value;
});
const db = {
exec() {},
} as unknown as import("node:sqlite").DatabaseSync;
runSqliteImmediateTransactionSync(db, () => "committed", {
busyTimeoutMs: 0,
databaseLabel: "agent.sqlite",
logger,
slowTransactionHoldMs: 0,
});
// busyTimeoutMs: 0 should NOT collapse threshold to 1ms.
// With the default 1000ms threshold, 5ms steps are not slow.
// Before the fix, this would have produced false-positive warnings.
expect(logger.warn).not.toHaveBeenCalledWith(
"slow SQLite transaction lock wait",
expect.anything(),
);
});
it("still warns for busyTimeoutMs: 0 when transaction crosses the default 1000ms threshold", () => {
const logger = { warn: vi.fn() };
let now = 0;
vi.spyOn(Date, "now").mockImplementation(() => {
const value = now;
now += 1_500; // Genuinely slow: 1500ms per step
return value;
});
const db = {
exec() {},
} as unknown as import("node:sqlite").DatabaseSync;
runSqliteImmediateTransactionSync(db, () => "committed", {
busyTimeoutMs: 0,
databaseLabel: "agent.sqlite",
logger,
slowTransactionHoldMs: 0,
});
// The 1000ms default threshold still catches genuinely slow transactions.
expect(logger.warn).toHaveBeenCalledWith(
"slow SQLite transaction lock wait",
expect.anything(),
);
});
it("logs slow successful transaction lock waits", () => {
const logger = { warn: vi.fn() };
let now = 0;
+2 -2
View File
@@ -79,10 +79,10 @@ export function isSqliteCorruptionError(error: unknown): boolean {
}
function slowBusyWaitThresholdMs(options: SqliteTransactionOptions | undefined): number {
if (options?.busyTimeoutMs === undefined) {
if (options?.busyTimeoutMs === undefined || options.busyTimeoutMs <= 0) {
return DEFAULT_SLOW_BUSY_WAIT_MS;
}
return Math.min(DEFAULT_SLOW_BUSY_WAIT_MS, Math.max(1, options.busyTimeoutMs));
return Math.min(DEFAULT_SLOW_BUSY_WAIT_MS, options.busyTimeoutMs);
}
function slowTransactionHoldThresholdMs(options: SqliteTransactionOptions | undefined): number {
@@ -77,6 +77,97 @@ describeControlUiE2e("Control UI usage cost analysis mocked Gateway E2E", () =>
await server?.close();
});
it("keeps pending sessions visible when their UTC activity day is selected", async () => {
const selectedDay = "2026-05-14";
const updatedAt = Date.parse("2026-05-14T00:30:00.000Z");
const pendingSessionKey = "agent:main:pending-cache";
const cachedSessionKey = "agent:main:cached-usage";
const context = await browser.newContext({
locale: "en-US",
serviceWorkers: "block",
timezoneId: "America/Los_Angeles",
viewport: { height: 1_000, width: 1_440 },
});
const page = await context.newPage();
const gateway = await installMockGateway(page, {
methodResponses: {
"sessions.usage": {
updatedAt,
startDate: selectedDay,
endDate: selectedDay,
sessions: [
{
key: cachedSessionKey,
label: "Cached session",
agentId: "main",
updatedAt,
usage: {
...totals,
activityDates: [selectedDay],
dailyBreakdown: [
{ date: selectedDay, cost: totals.totalCost, tokens: totals.totalTokens },
],
},
},
{
key: pendingSessionKey,
label: "Pending session",
agentId: "main",
updatedAt,
usage: null,
},
],
totals,
aggregates: {
messages: { total: 0, user: 0, assistant: 0, toolCalls: 0, toolResults: 0, errors: 0 },
tools: { totalCalls: 0, uniqueTools: 0, tools: [] },
byModel: [],
byProvider: [],
byAgent: [{ agentId: "main", totals }],
byChannel: [],
daily: [
{
date: selectedDay,
tokens: totals.totalTokens,
cost: totals.totalCost,
messages: 0,
toolCalls: 0,
errors: 0,
},
],
},
cacheStatus: { status: "refreshing", cachedFiles: 1, pendingFiles: 1, staleFiles: 0 },
},
"usage.cost": {
updatedAt,
days: 1,
daily: [{ ...totals, date: selectedDay }],
totals,
},
"usage.status": { updatedAt, providers: [] },
},
});
try {
await page.goto(`${server.baseUrl}usage`);
const pendingRow = page.locator(".session-bar-row").filter({ hasText: "Pending session" });
const cachedRow = page.locator(".session-bar-row").filter({ hasText: "Cached session" });
await expect.poll(() => pendingRow.count(), { timeout: 10_000 }).toBe(1);
await page.locator(".usage-select").selectOption("utc");
await expect
.poll(async () => (await gateway.getRequests("sessions.usage")).at(-1)?.params)
.toMatchObject({ mode: "utc" });
await expect.poll(() => cachedRow.count(), { timeout: 10_000 }).toBe(1);
await page.locator(".daily-bar-wrapper").click();
await expect.poll(() => cachedRow.count()).toBe(1);
await expect.poll(() => pendingRow.count()).toBe(1);
} finally {
await context.close();
}
});
it("renders cost analysis from Gateway usage data", async () => {
const context = await browser.newContext({
locale: "en-US",
+2 -2
View File
@@ -46,7 +46,7 @@ function dateBoundaryMs(date: string, timeZone: "local" | "utc", dayOffset: 0 |
return timeZone === "utc" ? Date.UTC(year, month, day) : new Date(year, month, day).getTime();
}
function dateKey(timestamp: number, timeZone: "local" | "utc"): string {
export function usageDateKey(timestamp: number, timeZone: "local" | "utc"): string {
const value = new Date(timestamp);
const year = timeZone === "utc" ? value.getUTCFullYear() : value.getFullYear();
const month = (timeZone === "utc" ? value.getUTCMonth() : value.getMonth()) + 1;
@@ -448,7 +448,7 @@ function renderTimeSeriesCompact(
return false;
}
if (selectedDaySet) {
return selectedDaySet.has(dateKey(p.timestamp, timeZone));
return selectedDaySet.has(usageDateKey(p.timestamp, timeZone));
}
return true;
});
+54
View File
@@ -154,6 +154,60 @@ function createUsageProps(overrides: Partial<UsageProps> = {}): UsageProps {
}
describe("renderUsage", () => {
it("keeps pending sessions on their selected local or UTC activity day", () => {
const localOffsetMs = -7 * 60 * 60 * 1000;
const localYear = vi
.spyOn(Date.prototype, "getFullYear")
.mockImplementation(function (this: Date) {
return new Date(this.getTime() + localOffsetMs).getUTCFullYear();
});
const localMonth = vi
.spyOn(Date.prototype, "getMonth")
.mockImplementation(function (this: Date) {
return new Date(this.getTime() + localOffsetMs).getUTCMonth();
});
const localDay = vi.spyOn(Date.prototype, "getDate").mockImplementation(function (this: Date) {
return new Date(this.getTime() + localOffsetMs).getUTCDate();
});
try {
const pendingSession = {
key: "agent:main:pending-cache",
label: "Pending cache",
agentId: "main",
updatedAt: Date.parse("2026-05-14T00:30:00.000Z"),
usage: null,
} satisfies UsageSessionEntry;
for (const { timeZone, selectedDay, visible } of [
{ timeZone: "utc", selectedDay: "2026-05-14", visible: true },
{ timeZone: "local", selectedDay: "2026-05-13", visible: true },
{ timeZone: "local", selectedDay: "2026-05-14", visible: false },
] as const) {
const container = document.createElement("div");
render(
renderUsage(
createUsageProps({
data: { ...createUsageProps().data, sessions: [pendingSession] },
filters: {
...createUsageProps().filters,
selectedDays: [selectedDay],
timeZone,
},
}),
),
container,
);
expect(container.querySelector(".session-bar-row") !== null).toBe(visible);
}
} finally {
localYear.mockRestore();
localMonth.mockRestore();
localDay.mockRestore();
}
});
it("keeps insight aggregates scoped to the selected agent", () => {
const container = document.createElement("div");
const sessions = [
+2 -4
View File
@@ -31,7 +31,7 @@ import {
setQueryTokensForKey,
} from "./query.ts";
import type { UsageFilterState, UsageProps, UsageSessionEntry, UsageTotals } from "./types.ts";
import { renderSessionDetailPanel } from "./view-details.ts";
import { renderSessionDetailPanel, usageDateKey } from "./view-details.ts";
import { renderUsageHeatmap } from "./view-heatmap.ts";
import {
renderCostBreakdownCompact,
@@ -220,9 +220,7 @@ export function renderUsage(props: UsageProps) {
if (!s.updatedAt) {
return false;
}
const d = new Date(s.updatedAt);
const sessionDate = `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`;
return selectedDaySet.has(sessionDate);
return selectedDaySet.has(usageDateKey(s.updatedAt, filters.timeZone));
})
: agentScopedSessions;