mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-27 12:56:01 -06:00
fix(logging): scale heap pressure with V8 limit (#115153)
Fixes #104631.
This commit is contained in:
committed by
GitHub
parent
edea5287dc
commit
20eda756fa
@@ -156,6 +156,75 @@ describe("diagnostic memory", () => {
|
||||
expect(events.map((event) => event.type)).toEqual(["diagnostic.memory.pressure"]);
|
||||
});
|
||||
|
||||
it("scales default heap pressure thresholds with enlarged V8 limits", () => {
|
||||
const events: DiagnosticEventPayload[] = [];
|
||||
const stop = onDiagnosticEvent((event) => events.push(event));
|
||||
const gb = 1024 ** 3;
|
||||
|
||||
emitDiagnosticMemorySample({
|
||||
now: 1000,
|
||||
heapSizeLimitBytes: 8 * gb,
|
||||
memoryUsage: memoryUsage({ heapUsed: 2.1 * gb }),
|
||||
});
|
||||
expect(events.filter((event) => event.type === "diagnostic.memory.pressure")).toEqual([]);
|
||||
|
||||
emitDiagnosticMemorySample({
|
||||
now: 2000,
|
||||
heapSizeLimitBytes: 8 * gb,
|
||||
memoryUsage: memoryUsage({ heapUsed: 4.1 * gb }),
|
||||
});
|
||||
emitDiagnosticMemorySample({
|
||||
now: 3000,
|
||||
heapSizeLimitBytes: 8 * gb,
|
||||
memoryUsage: memoryUsage({ heapUsed: 6.1 * gb }),
|
||||
});
|
||||
stop();
|
||||
|
||||
expect(
|
||||
events
|
||||
.filter((event) => event.type === "diagnostic.memory.pressure")
|
||||
.map((event) => ({
|
||||
level: event.level,
|
||||
reason: event.reason,
|
||||
threshold: event.thresholdBytes,
|
||||
})),
|
||||
).toEqual([
|
||||
{ level: "warning", reason: "heap_threshold", threshold: 4 * gb },
|
||||
{ level: "critical", reason: "heap_threshold", threshold: 6 * gb },
|
||||
]);
|
||||
});
|
||||
|
||||
it("scales default heap pressure thresholds down for constrained V8 limits", () => {
|
||||
const events: DiagnosticEventPayload[] = [];
|
||||
const stop = onDiagnosticEvent((event) => events.push(event));
|
||||
const mb = 1024 ** 2;
|
||||
|
||||
emitDiagnosticMemorySample({
|
||||
now: 1000,
|
||||
heapSizeLimitBytes: 1024 * mb,
|
||||
memoryUsage: memoryUsage({ heapUsed: 600 * mb }),
|
||||
});
|
||||
emitDiagnosticMemorySample({
|
||||
now: 2000,
|
||||
heapSizeLimitBytes: 1024 * mb,
|
||||
memoryUsage: memoryUsage({ heapUsed: 800 * mb }),
|
||||
});
|
||||
stop();
|
||||
|
||||
expect(
|
||||
events
|
||||
.filter((event) => event.type === "diagnostic.memory.pressure")
|
||||
.map((event) => ({
|
||||
level: event.level,
|
||||
reason: event.reason,
|
||||
threshold: event.thresholdBytes,
|
||||
})),
|
||||
).toEqual([
|
||||
{ level: "warning", reason: "heap_threshold", threshold: 512 * mb },
|
||||
{ level: "critical", reason: "heap_threshold", threshold: 768 * mb },
|
||||
]);
|
||||
});
|
||||
|
||||
it("emits pressure when RSS grows quickly", () => {
|
||||
const events: DiagnosticEventPayload[] = [];
|
||||
const stop = onDiagnosticEvent((event) => events.push(event));
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Diagnostic memory helpers capture process memory facts for support diagnostics.
|
||||
import { getHeapStatistics } from "node:v8";
|
||||
import {
|
||||
emitInternalDiagnosticEvent as emitDiagnosticEvent,
|
||||
type DiagnosticMemoryPressureEvent,
|
||||
@@ -9,10 +10,15 @@ import { createSubsystemLogger } from "./subsystem.js";
|
||||
|
||||
// Diagnostic memory sampler with threshold/growth pressure detection and repeat suppression.
|
||||
const MB = 1024 * 1024;
|
||||
const GB = 1024 * MB;
|
||||
const DEFAULT_RSS_WARNING_BYTES = 1536 * MB;
|
||||
const DEFAULT_RSS_CRITICAL_BYTES = 3072 * MB;
|
||||
const DEFAULT_HEAP_WARNING_BYTES = 1024 * MB;
|
||||
const DEFAULT_HEAP_CRITICAL_BYTES = 2048 * MB;
|
||||
const DEFAULT_HEAP_WARNING_RATIO = 0.5;
|
||||
const DEFAULT_HEAP_CRITICAL_RATIO = 0.75;
|
||||
const DEFAULT_HEAP_WARNING_MAX_BYTES = 4 * GB;
|
||||
const DEFAULT_HEAP_CRITICAL_MAX_BYTES = 6 * GB;
|
||||
const DEFAULT_RSS_GROWTH_WARNING_BYTES = 512 * MB;
|
||||
const DEFAULT_RSS_GROWTH_CRITICAL_BYTES = 1024 * MB;
|
||||
const DEFAULT_GROWTH_WINDOW_MS = 10 * 60 * 1000;
|
||||
@@ -60,12 +66,31 @@ function normalizeMemoryUsage(memory: NodeJS.MemoryUsage): DiagnosticMemoryUsage
|
||||
|
||||
function resolveThresholds(
|
||||
thresholds?: DiagnosticMemoryThresholds,
|
||||
heapSizeLimitBytes?: number,
|
||||
): Required<DiagnosticMemoryThresholds> {
|
||||
const hasHeapLimit =
|
||||
typeof heapSizeLimitBytes === "number" &&
|
||||
Number.isFinite(heapSizeLimitBytes) &&
|
||||
heapSizeLimitBytes > 0;
|
||||
// Scale both directions with V8's effective limit, but keep a warning/critical
|
||||
// ceiling so very large heaps still surface actionable pressure diagnostics.
|
||||
const heapWarningBytes = hasHeapLimit
|
||||
? Math.min(
|
||||
Math.floor(heapSizeLimitBytes * DEFAULT_HEAP_WARNING_RATIO),
|
||||
DEFAULT_HEAP_WARNING_MAX_BYTES,
|
||||
)
|
||||
: DEFAULT_HEAP_WARNING_BYTES;
|
||||
const heapCriticalBytes = hasHeapLimit
|
||||
? Math.min(
|
||||
Math.floor(heapSizeLimitBytes * DEFAULT_HEAP_CRITICAL_RATIO),
|
||||
DEFAULT_HEAP_CRITICAL_MAX_BYTES,
|
||||
)
|
||||
: DEFAULT_HEAP_CRITICAL_BYTES;
|
||||
return {
|
||||
rssWarningBytes: thresholds?.rssWarningBytes ?? DEFAULT_RSS_WARNING_BYTES,
|
||||
rssCriticalBytes: thresholds?.rssCriticalBytes ?? DEFAULT_RSS_CRITICAL_BYTES,
|
||||
heapUsedWarningBytes: thresholds?.heapUsedWarningBytes ?? DEFAULT_HEAP_WARNING_BYTES,
|
||||
heapUsedCriticalBytes: thresholds?.heapUsedCriticalBytes ?? DEFAULT_HEAP_CRITICAL_BYTES,
|
||||
heapUsedWarningBytes: thresholds?.heapUsedWarningBytes ?? heapWarningBytes,
|
||||
heapUsedCriticalBytes: thresholds?.heapUsedCriticalBytes ?? heapCriticalBytes,
|
||||
rssGrowthWarningBytes: thresholds?.rssGrowthWarningBytes ?? DEFAULT_RSS_GROWTH_WARNING_BYTES,
|
||||
rssGrowthCriticalBytes: thresholds?.rssGrowthCriticalBytes ?? DEFAULT_RSS_GROWTH_CRITICAL_BYTES,
|
||||
growthWindowMs: thresholds?.growthWindowMs ?? DEFAULT_GROWTH_WINDOW_MS,
|
||||
@@ -263,6 +288,7 @@ function logMemoryPressure(params: {
|
||||
export function emitDiagnosticMemorySample(options?: {
|
||||
now?: number;
|
||||
memoryUsage?: NodeJS.MemoryUsage;
|
||||
heapSizeLimitBytes?: number;
|
||||
uptimeMs?: number;
|
||||
thresholds?: DiagnosticMemoryThresholds;
|
||||
emitSample?: boolean;
|
||||
@@ -274,7 +300,10 @@ export function emitDiagnosticMemorySample(options?: {
|
||||
const now = options?.now ?? Date.now();
|
||||
const memory = normalizeMemoryUsage(options?.memoryUsage ?? process.memoryUsage());
|
||||
const current = { ts: now, memory };
|
||||
const thresholds = resolveThresholds(options?.thresholds);
|
||||
const thresholds = resolveThresholds(
|
||||
options?.thresholds,
|
||||
options?.heapSizeLimitBytes ?? getHeapStatistics().heap_size_limit,
|
||||
);
|
||||
const shouldEmitSample = options?.emitSample !== false;
|
||||
|
||||
if (shouldEmitSample) {
|
||||
|
||||
Reference in New Issue
Block a user