mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-28 05:16:23 -06:00
fix(scripts): clamp parallels update timeouts
This commit is contained in:
@@ -6,6 +6,11 @@ import { appendFileSync, existsSync, readFileSync, writeFileSync } from "node:fs
|
||||
import { copyFile, readFile, rm } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import {
|
||||
addTimerTimeoutGraceMs,
|
||||
clampTimerTimeoutMs,
|
||||
finiteSecondsToTimerSafeMilliseconds,
|
||||
} from "@openclaw/normalization-core/number-coercion";
|
||||
import {
|
||||
die,
|
||||
ensureValue,
|
||||
@@ -121,8 +126,24 @@ interface NpmUpdateSummary {
|
||||
const macosVmDefault = "macOS Tahoe";
|
||||
const windowsVm = "Windows 11";
|
||||
const linuxVmDefault = "Ubuntu 26.04";
|
||||
|
||||
function resolveRequiredTimerMs(timeoutMs: number): number {
|
||||
return clampTimerTimeoutMs(timeoutMs) ?? 1;
|
||||
}
|
||||
|
||||
function resolveOptionalTimerMs(timeoutMs: number | undefined): number | undefined {
|
||||
return timeoutMs === undefined || timeoutMs <= 0 ? undefined : resolveRequiredTimerMs(timeoutMs);
|
||||
}
|
||||
|
||||
function resolveSecondsTimerMs(timeoutSeconds: number): number {
|
||||
return finiteSecondsToTimerSafeMilliseconds(timeoutSeconds) ?? 1;
|
||||
}
|
||||
|
||||
const updateTimeoutSeconds = readPositiveIntEnv("OPENCLAW_PARALLELS_NPM_UPDATE_TIMEOUT_S", 1200);
|
||||
const updateCleanupBackstopMs = 60_000;
|
||||
const updateTimeoutMs = resolveSecondsTimerMs(updateTimeoutSeconds);
|
||||
const updateWithCleanupTimeoutMs =
|
||||
addTimerTimeoutGraceMs(updateTimeoutMs, updateCleanupBackstopMs) ?? 1;
|
||||
const freshLaneTimeoutKillGraceMs = readPositiveIntEnv(
|
||||
"OPENCLAW_PARALLELS_NPM_UPDATE_FRESH_TIMEOUT_KILL_GRACE_MS",
|
||||
2_000,
|
||||
@@ -133,7 +154,9 @@ let loggedExitCleanupInstalled = false;
|
||||
|
||||
export function freshLaneTimeoutMs(platform: Platform): number {
|
||||
const defaultSeconds = platform === "windows" ? 90 * 60 : 75 * 60;
|
||||
return readPositiveIntEnv("OPENCLAW_PARALLELS_NPM_UPDATE_FRESH_TIMEOUT_S", defaultSeconds) * 1000;
|
||||
return resolveSecondsTimerMs(
|
||||
readPositiveIntEnv("OPENCLAW_PARALLELS_NPM_UPDATE_FRESH_TIMEOUT_S", defaultSeconds),
|
||||
);
|
||||
}
|
||||
|
||||
export function spawnLoggedCommand(
|
||||
@@ -161,19 +184,23 @@ export function spawnLoggedCommand(
|
||||
appendFileSync(logPath, text, "utf8");
|
||||
onOutput(text);
|
||||
};
|
||||
const timeoutMs = options.timeoutMs ?? 0;
|
||||
const timeoutMs = resolveOptionalTimerMs(options.timeoutMs);
|
||||
const timeoutKillGraceMs =
|
||||
options.timeoutKillGraceMs === undefined
|
||||
? resolveRequiredTimerMs(freshLaneTimeoutKillGraceMs)
|
||||
: resolveRequiredTimerMs(options.timeoutKillGraceMs);
|
||||
const timeoutTimer =
|
||||
timeoutMs > 0
|
||||
timeoutMs !== undefined
|
||||
? setTimeout(() => {
|
||||
timedOut = true;
|
||||
append(
|
||||
`\n[${options.timeoutLabel ?? `${command} ${args.join(" ")}`} timed out after ${timeoutMs}ms]\n`,
|
||||
);
|
||||
signalLoggedChild(child, "SIGTERM");
|
||||
forceKillAt = Date.now() + (options.timeoutKillGraceMs ?? freshLaneTimeoutKillGraceMs);
|
||||
forceKillAt = Date.now() + timeoutKillGraceMs;
|
||||
forceKillTimer = setTimeout(
|
||||
() => signalLoggedChild(child, "SIGKILL"),
|
||||
options.timeoutKillGraceMs ?? freshLaneTimeoutKillGraceMs,
|
||||
timeoutKillGraceMs,
|
||||
);
|
||||
}, timeoutMs)
|
||||
: undefined;
|
||||
@@ -208,7 +235,7 @@ export function spawnLoggedCommand(
|
||||
if (timedOut) {
|
||||
void finishTimedOutLoggedProcessTree(child, {
|
||||
forceKillAt,
|
||||
timeoutKillGraceMs: options.timeoutKillGraceMs ?? freshLaneTimeoutKillGraceMs,
|
||||
timeoutKillGraceMs,
|
||||
}).then(finish, finish);
|
||||
return;
|
||||
}
|
||||
@@ -926,7 +953,7 @@ export class NpmUpdateSmoke {
|
||||
label,
|
||||
run: ({ signal }) => fn({ append, logPath, signal }),
|
||||
timeoutDescription: `${updateTimeoutSeconds}s plus cleanup backstop`,
|
||||
timeoutMs: updateTimeoutSeconds * 1000 + updateCleanupBackstopMs,
|
||||
timeoutMs: updateWithCleanupTimeoutMs,
|
||||
writeLog: async () => undefined,
|
||||
});
|
||||
})().finally(() => {
|
||||
@@ -937,15 +964,15 @@ export class NpmUpdateSmoke {
|
||||
}
|
||||
|
||||
private async runMacosUpdate(ctx: UpdateJobContext): Promise<void> {
|
||||
await this.guestMacos(this.updateScript("macos"), updateTimeoutSeconds * 1000, ctx);
|
||||
await this.guestMacos(this.updateScript("macos"), updateTimeoutMs, ctx);
|
||||
}
|
||||
|
||||
private runWindowsUpdate(ctx: UpdateJobContext): Promise<void> {
|
||||
return this.guestWindows(this.updateScript("windows"), updateTimeoutSeconds * 1000, ctx);
|
||||
return this.guestWindows(this.updateScript("windows"), updateTimeoutMs, ctx);
|
||||
}
|
||||
|
||||
private async runLinuxUpdate(ctx: UpdateJobContext): Promise<void> {
|
||||
await this.guestLinux(this.updateScript("linux"), updateTimeoutSeconds * 1000, ctx);
|
||||
await this.guestLinux(this.updateScript("linux"), updateTimeoutMs, ctx);
|
||||
}
|
||||
|
||||
private updateScript(platform: Platform): string {
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
// Update Job Timeout script supports OpenClaw repository automation.
|
||||
import { resolveTimerTimeoutMs } from "@openclaw/normalization-core/number-coercion";
|
||||
|
||||
interface TimedUpdateJobOptions {
|
||||
abortSettleMs?: number;
|
||||
append(this: void, chunk: string): void;
|
||||
@@ -21,6 +23,8 @@ export async function runTimedUpdateJob({
|
||||
let timedOut = false;
|
||||
const controller = new AbortController();
|
||||
const timeoutMessage = `${label} update timed out after ${timeoutDescription}`;
|
||||
const resolvedAbortSettleMs = resolveTimerTimeoutMs(abortSettleMs, 0, 0);
|
||||
const resolvedTimeoutMs = resolveTimerTimeoutMs(timeoutMs, 1);
|
||||
let timeout: NodeJS.Timeout | undefined;
|
||||
const runOutcome = Promise.resolve()
|
||||
.then(() => run({ signal: controller.signal }))
|
||||
@@ -34,13 +38,13 @@ export async function runTimedUpdateJob({
|
||||
append(`${timeoutMessage}\n`);
|
||||
controller.abort(new Error(timeoutMessage));
|
||||
resolve("timeout");
|
||||
}, timeoutMs);
|
||||
}, resolvedTimeoutMs);
|
||||
});
|
||||
|
||||
try {
|
||||
const outcome = await Promise.race([runOutcome, timeoutPromise]);
|
||||
if (outcome === "timeout") {
|
||||
await waitForAbortSettle(runOutcome, abortSettleMs);
|
||||
await waitForAbortSettle(runOutcome, resolvedAbortSettleMs);
|
||||
await writeLog();
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -2,6 +2,10 @@
|
||||
import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import {
|
||||
MAX_TIMER_TIMEOUT_MS,
|
||||
MAX_TIMER_TIMEOUT_SECONDS,
|
||||
} from "@openclaw/normalization-core/number-coercion";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { runWindowsBackgroundPowerShell } from "../../scripts/e2e/parallels/guest-transports.ts";
|
||||
import { run as hostCommandRun } from "../../scripts/e2e/parallels/host-command.ts";
|
||||
@@ -405,6 +409,29 @@ exit 1
|
||||
withEnv({ OPENCLAW_PARALLELS_NPM_UPDATE_FRESH_TIMEOUT_S: "3" }, () => {
|
||||
expect(freshLaneTimeoutMs("macos")).toBe(3000);
|
||||
});
|
||||
|
||||
withEnv(
|
||||
{ OPENCLAW_PARALLELS_NPM_UPDATE_FRESH_TIMEOUT_S: String(MAX_TIMER_TIMEOUT_SECONDS + 1) },
|
||||
() => {
|
||||
expect(freshLaneTimeoutMs("linux")).toBe(MAX_TIMER_TIMEOUT_MS);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("clamps oversized fresh lane command timeouts before scheduling", async () => {
|
||||
const root = makeTempDir();
|
||||
const logPath = path.join(root, "fresh.log");
|
||||
|
||||
const code = await spawnLoggedCommand(
|
||||
process.execPath,
|
||||
["-e", "setTimeout(() => process.exit(0), 25);"],
|
||||
logPath,
|
||||
{},
|
||||
undefined,
|
||||
{ timeoutMs: MAX_TIMER_TIMEOUT_MS + 1 },
|
||||
);
|
||||
|
||||
expect(code).toBe(0);
|
||||
});
|
||||
|
||||
it.runIf(process.platform !== "win32")("times out fresh lane process groups", async () => {
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { runTimedUpdateJob } from "../../scripts/e2e/parallels/update-job-timeout.ts";
|
||||
|
||||
@@ -29,6 +30,28 @@ describe("Parallels update job timeout", () => {
|
||||
expect(writeLog).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("clamps oversized update job timers before scheduling", async () => {
|
||||
const chunks: string[] = [];
|
||||
const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout");
|
||||
const writeLog = vi.fn(async () => undefined);
|
||||
|
||||
try {
|
||||
await expect(
|
||||
runTimedUpdateJob({
|
||||
append: (chunk) => chunks.push(chunk),
|
||||
label: "Linux",
|
||||
run: async () => undefined,
|
||||
timeoutDescription: "oversized",
|
||||
timeoutMs: MAX_TIMER_TIMEOUT_MS + 1,
|
||||
writeLog,
|
||||
}),
|
||||
).resolves.toBe(0);
|
||||
expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), MAX_TIMER_TIMEOUT_MS);
|
||||
} finally {
|
||||
setTimeoutSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("records update failures and writes the job log", async () => {
|
||||
const chunks: string[] = [];
|
||||
const writeLog = vi.fn(async () => undefined);
|
||||
|
||||
Reference in New Issue
Block a user