mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-28 05:16:23 -06:00
refactor: consolidate diagnostic stability field projection (#130845)
This commit is contained in:
committed by
GitHub
parent
d18b0a9661
commit
c52cf19bc6
@@ -3340,7 +3340,7 @@ src/logging/config.ts 2
|
||||
src/logging/console.ts 5
|
||||
src/logging/diagnostic-run-activity.ts 1
|
||||
src/logging/diagnostic-session-recovery-coordinator.ts 1
|
||||
src/logging/diagnostic-stability-bundle.ts 13
|
||||
src/logging/diagnostic-stability-bundle.ts 11
|
||||
src/logging/diagnostic-stability.ts 3
|
||||
src/logging/diagnostic-support-log-redaction.ts 1
|
||||
src/logging/diagnostic-support-redaction.ts 1
|
||||
|
||||
@@ -271,8 +271,15 @@ describe("diagnostic stability bundles", () => {
|
||||
evidence: {
|
||||
memoryPressure: {
|
||||
...retainedRuntimeEvidence,
|
||||
heapStatistics: {
|
||||
...retainedRuntimeEvidence.heapStatistics,
|
||||
totalHeapSizeBytes: 1536.75,
|
||||
},
|
||||
level: "critical",
|
||||
reason: "rss_threshold",
|
||||
thresholdBytes: 0,
|
||||
rssGrowthBytes: -1,
|
||||
windowMs: 0.5,
|
||||
memory: {
|
||||
rssBytes: 4096,
|
||||
heapTotalBytes: 2048,
|
||||
@@ -304,6 +311,7 @@ describe("diagnostic stability bundles", () => {
|
||||
});
|
||||
const snapshot = bundle.snapshot as Record<string, unknown>;
|
||||
Object.assign(snapshot, {
|
||||
count: 3,
|
||||
privateSnapshot: "snapshot-secret",
|
||||
events: [
|
||||
{
|
||||
@@ -324,11 +332,21 @@ describe("diagnostic stability bundles", () => {
|
||||
phase: "gateway_preflight",
|
||||
command: "raw command secret",
|
||||
},
|
||||
{
|
||||
seq: 3,
|
||||
ts: 3,
|
||||
type: "model.usage",
|
||||
costUsd: 0,
|
||||
durationMs: 0,
|
||||
usage: {},
|
||||
context: {},
|
||||
},
|
||||
],
|
||||
summary: {
|
||||
byType: {
|
||||
"webhook.error": 1,
|
||||
"exec.approval.followup_suppressed": 1,
|
||||
"model.usage": 1,
|
||||
"private summary type": 1,
|
||||
},
|
||||
privateSummary: "summary-secret",
|
||||
@@ -351,6 +369,17 @@ describe("diagnostic stability bundles", () => {
|
||||
"agents/<agent>/sessions/<session>.jsonl",
|
||||
);
|
||||
expect(result.bundle.evidence?.memoryPressure).toMatchObject(retainedRuntimeEvidence);
|
||||
expect(result.bundle.evidence?.memoryPressure).toMatchObject({
|
||||
thresholdBytes: 0,
|
||||
rssGrowthBytes: -1,
|
||||
windowMs: 0.5,
|
||||
heapStatistics: { totalHeapSizeBytes: 1536 },
|
||||
});
|
||||
expect(Object.keys(result.bundle.evidence?.memoryPressure?.heapStatistics ?? {})).toEqual([
|
||||
"totalHeapSizeBytes",
|
||||
"usedHeapSizeBytes",
|
||||
"heapSizeLimitBytes",
|
||||
]);
|
||||
expect(result.bundle.snapshot.events[0]).toEqual({
|
||||
seq: 1,
|
||||
ts: 1,
|
||||
@@ -365,9 +394,21 @@ describe("diagnostic stability bundles", () => {
|
||||
reason: "session_rebound",
|
||||
phase: "gateway_preflight",
|
||||
});
|
||||
expect(JSON.stringify(result.bundle.snapshot.events[2])).toBe(
|
||||
JSON.stringify({
|
||||
seq: 3,
|
||||
ts: 3,
|
||||
type: "model.usage",
|
||||
durationMs: 0,
|
||||
costUsd: 0,
|
||||
usage: {},
|
||||
context: {},
|
||||
}),
|
||||
);
|
||||
expect(result.bundle.snapshot.summary.byType).toEqual({
|
||||
"webhook.error": 1,
|
||||
"exec.approval.followup_suppressed": 1,
|
||||
"model.usage": 1,
|
||||
});
|
||||
const sanitized = JSON.stringify(result.bundle);
|
||||
for (const secret of [
|
||||
@@ -453,6 +494,45 @@ describe("diagnostic stability bundles", () => {
|
||||
},
|
||||
error: "snapshot.summary",
|
||||
},
|
||||
{
|
||||
name: "optional-code-before-number",
|
||||
bundle: {
|
||||
...baseBundle,
|
||||
snapshot: {
|
||||
...baseSnapshot,
|
||||
events: [{ seq: 1, ts: 1, type: "model.usage", channel: null, durationMs: null }],
|
||||
},
|
||||
},
|
||||
error: "snapshot.events[0].channel",
|
||||
},
|
||||
{
|
||||
name: "optional-usage-before-context",
|
||||
bundle: {
|
||||
...baseBundle,
|
||||
snapshot: {
|
||||
...baseSnapshot,
|
||||
events: [
|
||||
{ seq: 1, ts: 1, type: "model.usage", usage: { input: null }, context: false },
|
||||
],
|
||||
},
|
||||
},
|
||||
error: "snapshot.events[0].usage.input",
|
||||
},
|
||||
{
|
||||
name: "heap-statistics-before-required-memory",
|
||||
bundle: {
|
||||
...baseBundle,
|
||||
evidence: {
|
||||
memoryPressure: {
|
||||
level: "critical",
|
||||
reason: "rss_threshold",
|
||||
heapStatistics: { totalHeapSizeBytes: null },
|
||||
memory: null,
|
||||
},
|
||||
},
|
||||
},
|
||||
error: "evidence.memoryPressure.heapStatistics.totalHeapSizeBytes",
|
||||
},
|
||||
];
|
||||
|
||||
for (const testCase of cases) {
|
||||
|
||||
@@ -301,34 +301,19 @@ function readOptionalCodeString(value: unknown, label: string): string | undefin
|
||||
return SAFE_REASON_CODE.test(code) ? code : undefined;
|
||||
}
|
||||
|
||||
function assignOptionalNumber(target: object, key: string, value: unknown, label: string): void {
|
||||
const parsed = readOptionalNumber(value, label);
|
||||
if (parsed !== undefined) {
|
||||
(target as Record<string, unknown>)[key] = parsed;
|
||||
}
|
||||
}
|
||||
|
||||
function assignOptionalPositiveInteger(
|
||||
target: object,
|
||||
key: string,
|
||||
value: unknown,
|
||||
function assignOptionalFields<T extends object>(
|
||||
target: T,
|
||||
source: Record<string, unknown>,
|
||||
label: string,
|
||||
fields: readonly (keyof T & string)[],
|
||||
read: (value: unknown, label: string) => string | number | undefined,
|
||||
): void {
|
||||
const parsed = readOptionalPositiveInteger(value, label);
|
||||
if (parsed !== undefined) {
|
||||
(target as Record<string, unknown>)[key] = parsed;
|
||||
}
|
||||
}
|
||||
|
||||
function assignOptionalCodeString(
|
||||
target: object,
|
||||
key: string,
|
||||
value: unknown,
|
||||
label: string,
|
||||
): void {
|
||||
const parsed = readOptionalCodeString(value, label);
|
||||
if (parsed !== undefined) {
|
||||
(target as Record<string, unknown>)[key] = parsed;
|
||||
// The fixed order preserves serialized fields and the first failing validation label.
|
||||
for (const key of fields) {
|
||||
const parsed = read(source[key], `${label}.${key}`);
|
||||
if (parsed !== undefined) {
|
||||
(target as Record<string, unknown>)[key] = parsed;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -349,53 +334,21 @@ function readHeapStatistics(value: unknown): DiagnosticHeapStatisticsSummary | u
|
||||
}
|
||||
const source = readObject(value, "evidence.memoryPressure.heapStatistics");
|
||||
const result = {} as DiagnosticHeapStatisticsSummary;
|
||||
assignOptionalPositiveInteger(
|
||||
assignOptionalFields(
|
||||
result,
|
||||
"totalHeapSizeBytes",
|
||||
source.totalHeapSizeBytes,
|
||||
"evidence.memoryPressure.heapStatistics.totalHeapSizeBytes",
|
||||
);
|
||||
assignOptionalPositiveInteger(
|
||||
result,
|
||||
"totalHeapSizeExecutableBytes",
|
||||
source.totalHeapSizeExecutableBytes,
|
||||
"evidence.memoryPressure.heapStatistics.totalHeapSizeExecutableBytes",
|
||||
);
|
||||
assignOptionalPositiveInteger(
|
||||
result,
|
||||
"totalPhysicalSizeBytes",
|
||||
source.totalPhysicalSizeBytes,
|
||||
"evidence.memoryPressure.heapStatistics.totalPhysicalSizeBytes",
|
||||
);
|
||||
assignOptionalPositiveInteger(
|
||||
result,
|
||||
"totalAvailableSizeBytes",
|
||||
source.totalAvailableSizeBytes,
|
||||
"evidence.memoryPressure.heapStatistics.totalAvailableSizeBytes",
|
||||
);
|
||||
assignOptionalPositiveInteger(
|
||||
result,
|
||||
"usedHeapSizeBytes",
|
||||
source.usedHeapSizeBytes,
|
||||
"evidence.memoryPressure.heapStatistics.usedHeapSizeBytes",
|
||||
);
|
||||
assignOptionalPositiveInteger(
|
||||
result,
|
||||
"heapSizeLimitBytes",
|
||||
source.heapSizeLimitBytes,
|
||||
"evidence.memoryPressure.heapStatistics.heapSizeLimitBytes",
|
||||
);
|
||||
assignOptionalPositiveInteger(
|
||||
result,
|
||||
"mallocedMemoryBytes",
|
||||
source.mallocedMemoryBytes,
|
||||
"evidence.memoryPressure.heapStatistics.mallocedMemoryBytes",
|
||||
);
|
||||
assignOptionalPositiveInteger(
|
||||
result,
|
||||
"externalMemoryBytes",
|
||||
source.externalMemoryBytes,
|
||||
"evidence.memoryPressure.heapStatistics.externalMemoryBytes",
|
||||
source,
|
||||
"evidence.memoryPressure.heapStatistics",
|
||||
[
|
||||
"totalHeapSizeBytes",
|
||||
"totalHeapSizeExecutableBytes",
|
||||
"totalPhysicalSizeBytes",
|
||||
"totalAvailableSizeBytes",
|
||||
"usedHeapSizeBytes",
|
||||
"heapSizeLimitBytes",
|
||||
"mallocedMemoryBytes",
|
||||
"externalMemoryBytes",
|
||||
],
|
||||
readOptionalPositiveInteger,
|
||||
);
|
||||
return Object.keys(result).length > 0 ? result : undefined;
|
||||
}
|
||||
@@ -555,29 +508,20 @@ function readMemoryPressureEvidence(
|
||||
const cgroup = readCgroupMemorySummary(pressure.cgroup);
|
||||
const activeResources = readActiveResources(pressure.activeResources);
|
||||
const topSessionFiles = readSessionFiles(pressure.topSessionFiles);
|
||||
return {
|
||||
const result: DiagnosticMemoryPressureBundleEvidence = {
|
||||
level,
|
||||
reason,
|
||||
memory: readMemoryUsage(pressure.memory, "evidence.memoryPressure.memory"),
|
||||
...(pressure.thresholdBytes !== undefined
|
||||
? {
|
||||
thresholdBytes: readRequiredNumber(
|
||||
pressure.thresholdBytes,
|
||||
"evidence.memoryPressure.thresholdBytes",
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
...(pressure.rssGrowthBytes !== undefined
|
||||
? {
|
||||
rssGrowthBytes: readRequiredNumber(
|
||||
pressure.rssGrowthBytes,
|
||||
"evidence.memoryPressure.rssGrowthBytes",
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
...(pressure.windowMs !== undefined
|
||||
? { windowMs: readRequiredNumber(pressure.windowMs, "evidence.memoryPressure.windowMs") }
|
||||
: {}),
|
||||
};
|
||||
assignOptionalFields(
|
||||
result,
|
||||
pressure,
|
||||
"evidence.memoryPressure",
|
||||
["thresholdBytes", "rssGrowthBytes", "windowMs"],
|
||||
readOptionalNumber,
|
||||
);
|
||||
return {
|
||||
...result,
|
||||
...(heapStatistics ? { heapStatistics } : {}),
|
||||
...(heapSpaces ? { heapSpaces } : {}),
|
||||
...(cgroup ? { cgroup } : {}),
|
||||
@@ -676,97 +620,63 @@ function readStabilityEventRecord(
|
||||
) as DiagnosticStabilitySnapshot["events"][number]["type"],
|
||||
};
|
||||
|
||||
assignOptionalCodeString(sanitized, "channel", record.channel, `${label}.channel`);
|
||||
assignOptionalCodeString(sanitized, "pluginId", record.pluginId, `${label}.pluginId`);
|
||||
assignOptionalCodeString(sanitized, "source", record.source, `${label}.source`);
|
||||
assignOptionalCodeString(sanitized, "surface", record.surface, `${label}.surface`);
|
||||
assignOptionalCodeString(sanitized, "action", record.action, `${label}.action`);
|
||||
assignOptionalCodeString(sanitized, "reason", record.reason, `${label}.reason`);
|
||||
assignOptionalCodeString(sanitized, "outcome", record.outcome, `${label}.outcome`);
|
||||
assignOptionalCodeString(sanitized, "level", record.level, `${label}.level`);
|
||||
assignOptionalCodeString(sanitized, "phase", record.phase, `${label}.phase`);
|
||||
assignOptionalCodeString(sanitized, "approvalId", record.approvalId, `${label}.approvalId`);
|
||||
assignOptionalCodeString(sanitized, "detector", record.detector, `${label}.detector`);
|
||||
assignOptionalCodeString(sanitized, "toolName", record.toolName, `${label}.toolName`);
|
||||
assignOptionalCodeString(
|
||||
assignOptionalFields(
|
||||
sanitized,
|
||||
"activeWorkKind",
|
||||
record.activeWorkKind,
|
||||
`${label}.activeWorkKind`,
|
||||
record,
|
||||
label,
|
||||
[
|
||||
"channel",
|
||||
"pluginId",
|
||||
"source",
|
||||
"surface",
|
||||
"action",
|
||||
"reason",
|
||||
"outcome",
|
||||
"level",
|
||||
"phase",
|
||||
"approvalId",
|
||||
"detector",
|
||||
"toolName",
|
||||
"activeWorkKind",
|
||||
"pairedToolName",
|
||||
"provider",
|
||||
"model",
|
||||
],
|
||||
readOptionalCodeString,
|
||||
);
|
||||
assignOptionalCodeString(
|
||||
sanitized,
|
||||
"pairedToolName",
|
||||
record.pairedToolName,
|
||||
`${label}.pairedToolName`,
|
||||
);
|
||||
assignOptionalCodeString(sanitized, "provider", record.provider, `${label}.provider`);
|
||||
assignOptionalCodeString(sanitized, "model", record.model, `${label}.model`);
|
||||
|
||||
assignOptionalNumber(sanitized, "durationMs", record.durationMs, `${label}.durationMs`);
|
||||
assignOptionalNumber(sanitized, "requestBytes", record.requestBytes, `${label}.requestBytes`);
|
||||
assignOptionalNumber(sanitized, "responseBytes", record.responseBytes, `${label}.responseBytes`);
|
||||
assignOptionalNumber(
|
||||
assignOptionalFields(
|
||||
sanitized,
|
||||
"timeToFirstByteMs",
|
||||
record.timeToFirstByteMs,
|
||||
`${label}.timeToFirstByteMs`,
|
||||
);
|
||||
assignOptionalNumber(sanitized, "costUsd", record.costUsd, `${label}.costUsd`);
|
||||
assignOptionalNumber(sanitized, "count", record.count, `${label}.count`);
|
||||
assignOptionalNumber(sanitized, "bytes", record.bytes, `${label}.bytes`);
|
||||
assignOptionalNumber(sanitized, "limitBytes", record.limitBytes, `${label}.limitBytes`);
|
||||
assignOptionalNumber(
|
||||
sanitized,
|
||||
"thresholdBytes",
|
||||
record.thresholdBytes,
|
||||
`${label}.thresholdBytes`,
|
||||
);
|
||||
assignOptionalNumber(
|
||||
sanitized,
|
||||
"rssGrowthBytes",
|
||||
record.rssGrowthBytes,
|
||||
`${label}.rssGrowthBytes`,
|
||||
);
|
||||
assignOptionalNumber(sanitized, "windowMs", record.windowMs, `${label}.windowMs`);
|
||||
assignOptionalNumber(sanitized, "ageMs", record.ageMs, `${label}.ageMs`);
|
||||
assignOptionalNumber(sanitized, "queueDepth", record.queueDepth, `${label}.queueDepth`);
|
||||
assignOptionalNumber(sanitized, "queueSize", record.queueSize, `${label}.queueSize`);
|
||||
assignOptionalNumber(sanitized, "queueLength", record.queueLength, `${label}.queueLength`);
|
||||
assignOptionalNumber(sanitized, "waitMs", record.waitMs, `${label}.waitMs`);
|
||||
assignOptionalNumber(sanitized, "active", record.active, `${label}.active`);
|
||||
assignOptionalNumber(sanitized, "waiting", record.waiting, `${label}.waiting`);
|
||||
assignOptionalNumber(sanitized, "queued", record.queued, `${label}.queued`);
|
||||
assignOptionalNumber(sanitized, "droppedEvents", record.droppedEvents, `${label}.droppedEvents`);
|
||||
assignOptionalNumber(
|
||||
sanitized,
|
||||
"droppedTrustedEvents",
|
||||
record.droppedTrustedEvents,
|
||||
`${label}.droppedTrustedEvents`,
|
||||
);
|
||||
assignOptionalNumber(
|
||||
sanitized,
|
||||
"droppedUntrustedEvents",
|
||||
record.droppedUntrustedEvents,
|
||||
`${label}.droppedUntrustedEvents`,
|
||||
);
|
||||
assignOptionalNumber(
|
||||
sanitized,
|
||||
"droppedPriorityEvents",
|
||||
record.droppedPriorityEvents,
|
||||
`${label}.droppedPriorityEvents`,
|
||||
);
|
||||
assignOptionalNumber(
|
||||
sanitized,
|
||||
"maxQueueLength",
|
||||
record.maxQueueLength,
|
||||
`${label}.maxQueueLength`,
|
||||
);
|
||||
assignOptionalNumber(
|
||||
sanitized,
|
||||
"drainBatchSize",
|
||||
record.drainBatchSize,
|
||||
`${label}.drainBatchSize`,
|
||||
record,
|
||||
label,
|
||||
[
|
||||
"durationMs",
|
||||
"requestBytes",
|
||||
"responseBytes",
|
||||
"timeToFirstByteMs",
|
||||
"costUsd",
|
||||
"count",
|
||||
"bytes",
|
||||
"limitBytes",
|
||||
"thresholdBytes",
|
||||
"rssGrowthBytes",
|
||||
"windowMs",
|
||||
"ageMs",
|
||||
"queueDepth",
|
||||
"queueSize",
|
||||
"queueLength",
|
||||
"waitMs",
|
||||
"active",
|
||||
"waiting",
|
||||
"queued",
|
||||
"droppedEvents",
|
||||
"droppedTrustedEvents",
|
||||
"droppedUntrustedEvents",
|
||||
"droppedPriorityEvents",
|
||||
"maxQueueLength",
|
||||
"drainBatchSize",
|
||||
],
|
||||
readOptionalNumber,
|
||||
);
|
||||
|
||||
if (record.webhooks !== undefined) {
|
||||
@@ -782,37 +692,25 @@ function readStabilityEventRecord(
|
||||
}
|
||||
if (record.usage !== undefined) {
|
||||
const usage = readObject(record.usage, `${label}.usage`);
|
||||
sanitized.usage = {
|
||||
...(usage.input !== undefined
|
||||
? { input: readRequiredNumber(usage.input, `${label}.usage.input`) }
|
||||
: {}),
|
||||
...(usage.output !== undefined
|
||||
? { output: readRequiredNumber(usage.output, `${label}.usage.output`) }
|
||||
: {}),
|
||||
...(usage.cacheRead !== undefined
|
||||
? { cacheRead: readRequiredNumber(usage.cacheRead, `${label}.usage.cacheRead`) }
|
||||
: {}),
|
||||
...(usage.cacheWrite !== undefined
|
||||
? { cacheWrite: readRequiredNumber(usage.cacheWrite, `${label}.usage.cacheWrite`) }
|
||||
: {}),
|
||||
...(usage.promptTokens !== undefined
|
||||
? { promptTokens: readRequiredNumber(usage.promptTokens, `${label}.usage.promptTokens`) }
|
||||
: {}),
|
||||
...(usage.total !== undefined
|
||||
? { total: readRequiredNumber(usage.total, `${label}.usage.total`) }
|
||||
: {}),
|
||||
};
|
||||
sanitized.usage = {};
|
||||
assignOptionalFields(
|
||||
sanitized.usage,
|
||||
usage,
|
||||
`${label}.usage`,
|
||||
["input", "output", "cacheRead", "cacheWrite", "promptTokens", "total"],
|
||||
readOptionalNumber,
|
||||
);
|
||||
}
|
||||
if (record.context !== undefined) {
|
||||
const context = readObject(record.context, `${label}.context`);
|
||||
sanitized.context = {
|
||||
...(context.limit !== undefined
|
||||
? { limit: readRequiredNumber(context.limit, `${label}.context.limit`) }
|
||||
: {}),
|
||||
...(context.used !== undefined
|
||||
? { used: readRequiredNumber(context.used, `${label}.context.used`) }
|
||||
: {}),
|
||||
};
|
||||
sanitized.context = {};
|
||||
assignOptionalFields(
|
||||
sanitized.context,
|
||||
context,
|
||||
`${label}.context`,
|
||||
["limit", "used"],
|
||||
readOptionalNumber,
|
||||
);
|
||||
}
|
||||
|
||||
return sanitized;
|
||||
|
||||
@@ -258,24 +258,19 @@ function sanitizeDiagnosticEvent(event: DiagnosticEventPayload): DiagnosticStabi
|
||||
record.durationMs = event.durationMs;
|
||||
break;
|
||||
case "webhook.received":
|
||||
case "webhook.error":
|
||||
record.channel = event.channel;
|
||||
break;
|
||||
case "webhook.processed":
|
||||
record.channel = event.channel;
|
||||
record.durationMs = event.durationMs;
|
||||
break;
|
||||
case "webhook.error":
|
||||
record.channel = event.channel;
|
||||
break;
|
||||
case "message.queued":
|
||||
record.channel = event.channel;
|
||||
record.source = event.source;
|
||||
record.queueDepth = event.queueDepth;
|
||||
break;
|
||||
case "message.received":
|
||||
record.channel = event.channel;
|
||||
record.source = event.source;
|
||||
break;
|
||||
case "message.dispatch.started":
|
||||
record.channel = event.channel;
|
||||
record.source = event.source;
|
||||
@@ -398,7 +393,6 @@ function sanitizeDiagnosticEvent(event: DiagnosticEventPayload): DiagnosticStabi
|
||||
record.bytes = event.promptChars;
|
||||
record.context =
|
||||
event.contextTokenBudget !== undefined ? { limit: event.contextTokenBudget } : undefined;
|
||||
record.bytes = event.promptChars;
|
||||
break;
|
||||
case "diagnostic.heartbeat":
|
||||
record.webhooks = { ...event.webhooks };
|
||||
|
||||
Reference in New Issue
Block a user