fix(diagnostics): make memory pressure logs actionable

Log warning memory pressure at WARN level, add readable RSS/heap/threshold units, threshold ratio, and a concrete operator next step while preserving raw byte fields for diagnostics consumers.

Fixes #90783
This commit is contained in:
Vincent Koc
2026-06-07 19:23:23 +09:00
parent cfe5d24889
commit b00e1b2e7b
2 changed files with 119 additions and 6 deletions
+43
View File
@@ -369,6 +369,49 @@ describe("diagnostic memory", () => {
);
});
it("logs warning pressure with readable units and operator guidance", async () => {
setLoggerOverride({ level: "info", consoleLevel: "silent" });
const records: Array<Extract<DiagnosticEventPayload, { type: "log.record" }>> = [];
const stop = onInternalDiagnosticEvent((event) => {
if (event.type === "log.record") {
records.push(event);
}
});
try {
emitDiagnosticMemorySample({
now: Date.parse("2026-04-22T12:00:00.000Z"),
memoryUsage: memoryUsage({ rss: 2_012_905_472, heapUsed: 1_307_038_712 }),
thresholds: {
rssWarningBytes: 1_610_612_736,
rssCriticalBytes: 3_221_225_472,
pressureRepeatMs: 60_000,
},
});
await flushDiagnosticEvents();
} finally {
stop();
}
expect(records).toEqual(
expect.arrayContaining([
expect.objectContaining({
level: "WARN",
message: expect.stringContaining(
"memory pressure: level=warning reason=rss_threshold rss=1.87 GiB heap=1.22 GiB threshold=1.5 GiB thresholdRatio=125%",
),
attributes: expect.objectContaining({
subsystem: "gateway/diagnostics/memory",
}),
}),
]),
);
expect(records.at(-1)?.message).toContain("rssBytes=2012905472");
expect(records.at(-1)?.message).toContain("heapUsedBytes=1307038712");
expect(records.at(-1)?.message).toContain(
"nextStep=run openclaw gateway status --deep and openclaw gateway diagnostics export; restart gateway if pressure persists",
);
});
it("writes a stability bundle when critical pressure is emitted", () => {
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-memory-pressure-"));
const customRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-memory-custom-sessions-"));
+76 -6
View File
@@ -17,6 +17,7 @@ const DEFAULT_RSS_GROWTH_WARNING_BYTES = 512 * MB;
const DEFAULT_RSS_GROWTH_CRITICAL_BYTES = 1024 * MB;
const DEFAULT_GROWTH_WINDOW_MS = 10 * 60 * 1000;
const DEFAULT_PRESSURE_REPEAT_MS = 5 * 60 * 1000;
const BYTE_UNITS = ["B", "KiB", "MiB", "GiB", "TiB"] as const;
const log = createSubsystemLogger("gateway").child("diagnostics/memory");
@@ -168,6 +169,77 @@ function formatOptionalPressureMetric(label: string, value: number | undefined):
return typeof value === "number" && Number.isFinite(value) ? ` ${label}=${value}` : "";
}
function formatScaledNumber(value: number): string {
const fixed = value >= 10 ? value.toFixed(1) : value.toFixed(2);
return fixed.replace(/\.0+$/u, "").replace(/(\.\d*[1-9])0$/u, "$1");
}
function formatReadableBytes(value: number | undefined): string | undefined {
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
return undefined;
}
let scaled = value;
let unitIndex = 0;
while (scaled >= 1024 && unitIndex < BYTE_UNITS.length - 1) {
scaled /= 1024;
unitIndex++;
}
return unitIndex === 0
? `${Math.round(scaled)} ${BYTE_UNITS[unitIndex]}`
: `${formatScaledNumber(scaled)} ${BYTE_UNITS[unitIndex]}`;
}
function formatPressureRatio(params: {
pressure: Omit<DiagnosticMemoryPressureEvent, "seq" | "ts" | "type">;
thresholdBytes: number;
}): string | undefined {
const { pressure, thresholdBytes } = params;
if (!Number.isFinite(thresholdBytes) || thresholdBytes <= 0) {
return undefined;
}
const value =
pressure.reason === "heap_threshold"
? pressure.memory.heapUsedBytes
: pressure.reason === "rss_growth"
? pressure.rssGrowthBytes
: pressure.memory.rssBytes;
if (typeof value !== "number" || !Number.isFinite(value)) {
return undefined;
}
const ratio = (value / thresholdBytes) * 100;
return `${formatScaledNumber(ratio)}%`;
}
function formatPressureSummary(
pressure: Omit<DiagnosticMemoryPressureEvent, "seq" | "ts" | "type">,
): string {
const parts = [
`rss=${formatReadableBytes(pressure.memory.rssBytes)}`,
`heap=${formatReadableBytes(pressure.memory.heapUsedBytes)}`,
pressure.thresholdBytes !== undefined
? `threshold=${formatReadableBytes(pressure.thresholdBytes)}`
: "",
pressure.thresholdBytes !== undefined
? `thresholdRatio=${formatPressureRatio({
pressure,
thresholdBytes: pressure.thresholdBytes,
})}`
: "",
pressure.rssGrowthBytes !== undefined
? `rssGrowth=${formatReadableBytes(pressure.rssGrowthBytes)}`
: "",
];
return parts.filter((part): part is string => Boolean(part)).join(" ");
}
function formatPressureNextStep(
pressure: Omit<DiagnosticMemoryPressureEvent, "seq" | "ts" | "type">,
): string {
return pressure.level === "critical"
? "nextStep=inspect latest stability bundle or run openclaw gateway diagnostics export; restart gateway if process is unstable"
: "nextStep=run openclaw gateway status --deep and openclaw gateway diagnostics export; restart gateway if pressure persists";
}
function logMemoryPressure(params: {
pressure: Omit<DiagnosticMemoryPressureEvent, "seq" | "ts" | "type">;
writeCriticalBundle: boolean;
@@ -175,6 +247,7 @@ function logMemoryPressure(params: {
const { pressure } = params;
const message =
`memory pressure: level=${pressure.level} reason=${pressure.reason}` +
` ${formatPressureSummary(pressure)}` +
` rssBytes=${pressure.memory.rssBytes}` +
` heapUsedBytes=${pressure.memory.heapUsedBytes}` +
formatOptionalPressureMetric("thresholdBytes", pressure.thresholdBytes) +
@@ -182,12 +255,9 @@ function logMemoryPressure(params: {
formatOptionalPressureMetric("windowMs", pressure.windowMs) +
(pressure.level === "critical"
? ` memoryPressureSnapshot=${params.writeCriticalBundle ? "enabled" : "disabled"}`
: "");
if (pressure.level === "critical") {
log.warn(message);
} else {
log.info(message);
}
: "") +
` ${formatPressureNextStep(pressure)}`;
log.warn(message);
}
export function emitDiagnosticMemorySample(options?: {