mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-23 10:55:31 -06:00
fix(snapshot): survive cold PowerShell starts in Windows staging gates (#123633)
* fix(snapshot): survive cold PowerShell starts in Windows staging gates CI run 31775262530, checks-windows-node-test-1 attempt 1, showed the fail-closed ACL probe timing out during PowerShell first-use module preparation. Centralize encoded one-shot spawning, budget 60 seconds for cold starts, and preserve the underlying probe failure as the error cause. * fix(snapshot): sanitize PowerShell failure causes in Windows staging gates * fix(secrets): explain the sanitized plan-file failure cause suppression check-lint-core-2 flagged preserve-caught-error at the private plan file catch; retaining the raw error would re-leak the -EncodedCommand argv the sanitization contract strips, so the suppression is intentional (same idiom as setup-inference-activate.ts). * test(lint): register the private-plan-file suppression in the inventory * test(infra): give the LAN-host real PowerShell spawn a cold-start budget checks-windows-node-test-2 (run 31804325922) hit the same cold-start flake class this PR fixes: the codepage-proof test spawns real powershell.exe bounded at 3s, which a cold runner cannot meet. Production keeps its fail-open 3s route-hint probe; only the test's real-spawn verification uses the shared cold-spawn budget.
This commit is contained in:
committed by
GitHub
parent
7c579abf06
commit
b4f91fadf3
@@ -3,12 +3,10 @@ import { describe, expect, it, vi } from "vitest";
|
||||
import { runCommandWithTimeout } from "../process/exec.js";
|
||||
import { resolveAdvertisedLanHostCore } from "./advertised-lan-host.js";
|
||||
import type { NetworkInterfacesSnapshot } from "./network-interfaces.js";
|
||||
import { WINDOWS_POWERSHELL_COLD_SPAWN_TIMEOUT_MS } from "./windows-powershell-spawn.js";
|
||||
|
||||
type ResolveOptions = NonNullable<Parameters<typeof resolveAdvertisedLanHostCore>[0]>;
|
||||
type RouteRunner = NonNullable<ResolveOptions["runCommandWithTimeout"]>;
|
||||
// This native probe validates PowerShell encoding, not the product's optional route-hint latency budget.
|
||||
const POWERSHELL_ENCODING_PROBE_TIMEOUT_MS = 10_000;
|
||||
|
||||
function ipv4(address: string) {
|
||||
return {
|
||||
address,
|
||||
@@ -50,7 +48,9 @@ describe.runIf(process.platform === "win32")("advertised LAN host PowerShell con
|
||||
"-Command",
|
||||
`[Console]::OutputEncoding=[Text.Encoding]::GetEncoding(437); ${outputPrefix}[pscustomobject]@{InterfaceAlias='réseau-网卡';RouteMetric=1;InterfaceMetric=1} | ConvertTo-Json -Compress`,
|
||||
],
|
||||
{ timeoutMs: POWERSHELL_ENCODING_PROBE_TIMEOUT_MS, maxOutputBytes: 16 * 1024 },
|
||||
// Real spawn: cold PowerShell first-use can exceed the production 3s
|
||||
// fail-open probe budget; only production keeps the short bound.
|
||||
{ timeoutMs: WINDOWS_POWERSHELL_COLD_SPAWN_TIMEOUT_MS, maxOutputBytes: 16 * 1024 },
|
||||
);
|
||||
expect(result).toMatchObject({ code: 0 });
|
||||
expect(JSON.parse(result.stdout)).toMatchObject({ InterfaceAlias: "réseau-网卡" });
|
||||
|
||||
@@ -4,9 +4,14 @@ import { randomUUID } from "node:crypto";
|
||||
import fsSync from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
|
||||
import { resolveSystemBin } from "./resolve-system-bin.js";
|
||||
import { decodeWindowsOutputBuffer } from "./windows-encoding.js";
|
||||
import {
|
||||
buildEncodedPowerShellArgs,
|
||||
buildPowerShellFailureCause,
|
||||
sanitizePowerShellOutputText,
|
||||
WINDOWS_POWERSHELL_COLD_SPAWN_TIMEOUT_MS,
|
||||
} from "./windows-powershell-spawn.js";
|
||||
|
||||
const SQLITE_DIRECTORY_MODE = 0o700;
|
||||
const WINDOWS_DIRECTORY_EXISTS_MARKER = "OPENCLAW_SQLITE_DIRECTORY_EXISTS";
|
||||
@@ -79,14 +84,7 @@ function failureText(value: unknown): string {
|
||||
: typeof value === "string"
|
||||
? value
|
||||
: "";
|
||||
return truncateUtf16Safe(
|
||||
text
|
||||
.split(/\r?\n/u)
|
||||
.filter((line) => !line.toLowerCase().includes("encodedcommand"))
|
||||
.join("\n")
|
||||
.trim(),
|
||||
1000,
|
||||
);
|
||||
return sanitizePowerShellOutputText(text);
|
||||
}
|
||||
|
||||
function privateDirectoryError(
|
||||
@@ -105,22 +103,14 @@ function privateDirectoryError(
|
||||
(existsError as NodeJS.ErrnoException).code = "EEXIST";
|
||||
return existsError;
|
||||
}
|
||||
const status = [
|
||||
typeof failure.status === "number" ? `status=${failure.status}` : "",
|
||||
typeof failure.code === "number"
|
||||
? `exit=${failure.code}`
|
||||
: typeof failure.code === "string"
|
||||
? `code=${failure.code}`
|
||||
: "",
|
||||
typeof failure.killed === "boolean" ? `killed=${failure.killed}` : "",
|
||||
typeof failure.signal === "string" ? `signal=${failure.signal}` : "",
|
||||
].filter(Boolean);
|
||||
const stderrText = failureText(stderr) || failureText(failure.stderr);
|
||||
const stdoutText = failureText(stdout) || failureText(failure.stdout);
|
||||
const detail = stderrText ? `stderr: ${stderrText}` : stdoutText ? `stdout: ${stdoutText}` : "";
|
||||
const cause = new Error(
|
||||
`PowerShell failed${status.length ? ` (${status.join(", ")})` : ""}${detail ? `; ${detail}` : ""}`,
|
||||
);
|
||||
const cause = buildPowerShellFailureCause({
|
||||
status: failure.status,
|
||||
code: failure.code,
|
||||
killed: failure.killed,
|
||||
signal: failure.signal,
|
||||
stderr: failureText(stderr) || failureText(failure.stderr),
|
||||
stdout: failureText(stdout) || failureText(failure.stdout),
|
||||
});
|
||||
return new Error(`Unable to create private Windows SQLite directory: ${directoryPath}`, {
|
||||
cause,
|
||||
});
|
||||
@@ -129,16 +119,16 @@ function privateDirectoryError(
|
||||
function runPrivateDirectoryPowerShell(
|
||||
directoryPath: string,
|
||||
powershell: string,
|
||||
encodedCommand: string,
|
||||
args: string[],
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
execFile(
|
||||
powershell,
|
||||
["-NoLogo", "-NoProfile", "-NonInteractive", "-EncodedCommand", encodedCommand],
|
||||
args,
|
||||
{
|
||||
encoding: "buffer",
|
||||
maxBuffer: 64 * 1024,
|
||||
timeout: 10_000,
|
||||
timeout: WINDOWS_POWERSHELL_COLD_SPAWN_TIMEOUT_MS,
|
||||
windowsHide: true,
|
||||
},
|
||||
(error, stdout, stderr) => {
|
||||
@@ -153,7 +143,7 @@ function runPrivateDirectoryPowerShell(
|
||||
}
|
||||
|
||||
function resolvePrivateDirectoryPowerShell(directoryPath: string): {
|
||||
encodedCommand: string;
|
||||
args: string[];
|
||||
powershell: string;
|
||||
} {
|
||||
const nativeDirectoryPath = path.toNamespacedPath(path.resolve(directoryPath));
|
||||
@@ -185,7 +175,7 @@ function resolvePrivateDirectoryPowerShell(directoryPath: string): {
|
||||
}
|
||||
return {
|
||||
powershell,
|
||||
encodedCommand: Buffer.from(command, "utf16le").toString("base64"),
|
||||
args: buildEncodedPowerShellArgs(command),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -195,8 +185,8 @@ export async function createPrivateSqliteDirectory(directoryPath: string): Promi
|
||||
return;
|
||||
}
|
||||
// This raw Win32 call bypasses Node's automatic long-path normalization.
|
||||
const { encodedCommand, powershell } = resolvePrivateDirectoryPowerShell(directoryPath);
|
||||
await runPrivateDirectoryPowerShell(directoryPath, powershell, encodedCommand);
|
||||
const { args, powershell } = resolvePrivateDirectoryPowerShell(directoryPath);
|
||||
await runPrivateDirectoryPowerShell(directoryPath, powershell, args);
|
||||
}
|
||||
|
||||
function createPrivateSqliteDirectorySync(directoryPath: string): void {
|
||||
@@ -204,17 +194,13 @@ function createPrivateSqliteDirectorySync(directoryPath: string): void {
|
||||
fsSync.mkdirSync(directoryPath, { mode: SQLITE_DIRECTORY_MODE });
|
||||
return;
|
||||
}
|
||||
const { encodedCommand, powershell } = resolvePrivateDirectoryPowerShell(directoryPath);
|
||||
const { args, powershell } = resolvePrivateDirectoryPowerShell(directoryPath);
|
||||
try {
|
||||
execFileSync(
|
||||
powershell,
|
||||
["-NoLogo", "-NoProfile", "-NonInteractive", "-EncodedCommand", encodedCommand],
|
||||
{
|
||||
maxBuffer: 64 * 1024,
|
||||
timeout: 10_000,
|
||||
windowsHide: true,
|
||||
},
|
||||
);
|
||||
execFileSync(powershell, args, {
|
||||
maxBuffer: 64 * 1024,
|
||||
timeout: WINDOWS_POWERSHELL_COLD_SPAWN_TIMEOUT_MS,
|
||||
windowsHide: true,
|
||||
});
|
||||
} catch (error) {
|
||||
throw privateDirectoryError(directoryPath, error);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
|
||||
|
||||
// Windows PowerShell one-shots pay cold first-use costs: module analysis ("Preparing modules
|
||||
// for first use") and NGEN image compilation exceeded 10 seconds on loaded CI runners.
|
||||
// Fail-closed security gates must out-wait cold starts; this bound only stops true hangs.
|
||||
export const WINDOWS_POWERSHELL_COLD_SPAWN_TIMEOUT_MS = 60_000;
|
||||
|
||||
export function sanitizePowerShellOutputText(text: string): string {
|
||||
return truncateUtf16Safe(
|
||||
text
|
||||
.split(/\r?\n/u)
|
||||
.filter((line) => !line.toLowerCase().includes("encodedcommand"))
|
||||
.join("\n")
|
||||
.trim(),
|
||||
1000,
|
||||
);
|
||||
}
|
||||
|
||||
export function buildPowerShellFailureCause(error: unknown): Error {
|
||||
const failure = error && typeof error === "object" ? (error as Record<string, unknown>) : {};
|
||||
const status = [
|
||||
typeof failure.status === "number" ? `status=${failure.status}` : "",
|
||||
typeof failure.code === "number"
|
||||
? `exit=${failure.code}`
|
||||
: typeof failure.code === "string"
|
||||
? `code=${failure.code}`
|
||||
: "",
|
||||
typeof failure.killed === "boolean" ? `killed=${failure.killed}` : "",
|
||||
typeof failure.signal === "string" ? `signal=${failure.signal}` : "",
|
||||
].filter(Boolean);
|
||||
const stderr =
|
||||
typeof failure.stderr === "string" ? sanitizePowerShellOutputText(failure.stderr) : "";
|
||||
const stdout =
|
||||
typeof failure.stdout === "string" ? sanitizePowerShellOutputText(failure.stdout) : "";
|
||||
const detail = stderr ? `stderr: ${stderr}` : stdout ? `stdout: ${stdout}` : "";
|
||||
return new Error(
|
||||
`PowerShell failed${status.length ? ` (${status.join(", ")})` : ""}${detail ? `; ${detail}` : ""}`,
|
||||
);
|
||||
}
|
||||
|
||||
export function buildEncodedPowerShellArgs(command: string): string[] {
|
||||
const encodedCommand = Buffer.from(command, "utf16le").toString("base64");
|
||||
// Canonical argv for non-interactive encoded one-shots.
|
||||
return ["-NoLogo", "-NoProfile", "-NonInteractive", "-EncodedCommand", encodedCommand];
|
||||
}
|
||||
@@ -1,6 +1,11 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import path from "node:path";
|
||||
import { resolveSystemBin } from "../infra/resolve-system-bin.js";
|
||||
import {
|
||||
buildEncodedPowerShellArgs,
|
||||
buildPowerShellFailureCause,
|
||||
WINDOWS_POWERSHELL_COLD_SPAWN_TIMEOUT_MS,
|
||||
} from "../infra/windows-powershell-spawn.js";
|
||||
import { runExec } from "../process/exec.js";
|
||||
import {
|
||||
resolveTrustedPlanDirectoryPath,
|
||||
@@ -301,35 +306,28 @@ export async function createPrivateWindowsPlanFile(
|
||||
"utf8",
|
||||
).toString("base64");
|
||||
try {
|
||||
await run(
|
||||
powershell,
|
||||
[
|
||||
"-NoLogo",
|
||||
"-NoProfile",
|
||||
"-NonInteractive",
|
||||
"-EncodedCommand",
|
||||
Buffer.from(command, "utf16le").toString("base64"),
|
||||
],
|
||||
{
|
||||
baseEnv: {},
|
||||
env: {
|
||||
SYSTEMROOT: systemRoot,
|
||||
TEMP: compilerTempDir,
|
||||
TMP: compilerTempDir,
|
||||
WINDIR: systemRoot,
|
||||
},
|
||||
input,
|
||||
logOutput: false,
|
||||
maxBuffer: 64 * 1024,
|
||||
timeoutMs: 10_000,
|
||||
await run(powershell, buildEncodedPowerShellArgs(command), {
|
||||
baseEnv: {},
|
||||
env: {
|
||||
SYSTEMROOT: systemRoot,
|
||||
TEMP: compilerTempDir,
|
||||
TMP: compilerTempDir,
|
||||
WINDIR: systemRoot,
|
||||
},
|
||||
);
|
||||
input,
|
||||
logOutput: false,
|
||||
maxBuffer: 64 * 1024,
|
||||
timeoutMs: WINDOWS_POWERSHELL_COLD_SPAWN_TIMEOUT_MS,
|
||||
});
|
||||
} catch (error) {
|
||||
if (String(error).includes(WINDOWS_PLAN_FILE_EXISTS_MARKER)) {
|
||||
const existsError = new Error(`Private plan file already exists: ${filePath}`);
|
||||
(existsError as NodeJS.ErrnoException).code = "EEXIST";
|
||||
throw existsError;
|
||||
}
|
||||
throw new Error(`Unable to create private Windows plan file: ${filePath}`, { cause: error });
|
||||
// oxlint-disable-next-line preserve-caught-error -- The raw error carries the -EncodedCommand argv; only the sanitized bounded diagnostic may escape.
|
||||
throw new Error(`Unable to create private Windows plan file: ${filePath}`, {
|
||||
cause: buildPowerShellFailureCause(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,6 +33,11 @@ import {
|
||||
} from "../infra/sqlite-private-directory.js";
|
||||
import { publishVerifiedSqliteFile } from "../infra/sqlite-snapshot.js";
|
||||
import { readSqliteUserVersion } from "../infra/sqlite-user-version.js";
|
||||
import {
|
||||
buildEncodedPowerShellArgs,
|
||||
buildPowerShellFailureCause,
|
||||
WINDOWS_POWERSHELL_COLD_SPAWN_TIMEOUT_MS,
|
||||
} from "../infra/windows-powershell-spawn.js";
|
||||
import { runExec } from "../process/exec.js";
|
||||
import {
|
||||
copySnapshotArtifact,
|
||||
@@ -1415,8 +1420,10 @@ async function assertTrustedWindowsStagingPath(rootPath: string): Promise<void>
|
||||
let security: z.infer<typeof WINDOWS_PATH_SECURITY_SCHEMA>;
|
||||
try {
|
||||
security = await inspectWindowsPathSecurity(paths);
|
||||
} catch {
|
||||
throw new Error(`Unable to verify private Windows ACL for SQLite staging: ${rootPath}`);
|
||||
} catch (error) {
|
||||
throw new Error(`Unable to verify private Windows ACL for SQLite staging: ${rootPath}`, {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
if (security.paths.length !== paths.length) {
|
||||
throw new Error(`Unable to verify private Windows ACL for SQLite staging: ${rootPath}`);
|
||||
@@ -1531,16 +1538,15 @@ async function runEncodedWindowsPowerShell(command: string, maxBuffer: number):
|
||||
if (!powershell) {
|
||||
throw new Error("Unable to resolve PowerShell for Windows SQLite path security.");
|
||||
}
|
||||
const encodedCommand = Buffer.from(command, "utf16le").toString("base64");
|
||||
const { stdout } = await runExec(
|
||||
powershell,
|
||||
["-NoLogo", "-NoProfile", "-NonInteractive", "-EncodedCommand", encodedCommand],
|
||||
{
|
||||
timeoutMs: 10_000,
|
||||
try {
|
||||
const { stdout } = await runExec(powershell, buildEncodedPowerShellArgs(command), {
|
||||
timeoutMs: WINDOWS_POWERSHELL_COLD_SPAWN_TIMEOUT_MS,
|
||||
maxBuffer,
|
||||
},
|
||||
);
|
||||
return stdout;
|
||||
});
|
||||
return stdout;
|
||||
} catch (error) {
|
||||
throw buildPowerShellFailureCause(error);
|
||||
}
|
||||
}
|
||||
|
||||
async function removePublishedSnapshotDirectoryIfOwned(
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import { WINDOWS_POWERSHELL_COLD_SPAWN_TIMEOUT_MS } from "../infra/windows-powershell-spawn.js";
|
||||
|
||||
const execMocks = vi.hoisted(() => ({
|
||||
runExec: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../process/exec.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../process/exec.js")>()),
|
||||
runExec: execMocks.runExec,
|
||||
}));
|
||||
vi.mock("../infra/resolve-system-bin.js", () => ({
|
||||
resolveSystemBin: () => "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe",
|
||||
}));
|
||||
|
||||
import { ensurePrivateSnapshotRepositoryRoot } from "./local-repository.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
describe("fail-closed Windows ACL probe", () => {
|
||||
it("budgets for cold PowerShell startup and sanitizes the spawn failure", async () => {
|
||||
const tempDir = tempDirs.make("openclaw-snapshot-windows-acl-probe-");
|
||||
vi.spyOn(process, "platform", "get").mockReturnValue("win32");
|
||||
const encodedPayload = Buffer.from("private PowerShell script bytes").toString("base64");
|
||||
const command = `powershell.exe -EncodedCommand ${encodedPayload}`;
|
||||
const spawnError = Object.assign(
|
||||
new Error(`Command timed out after 60000 milliseconds: ${command}`),
|
||||
{
|
||||
code: "ETIMEDOUT",
|
||||
command,
|
||||
escapedCommand: command,
|
||||
killed: true,
|
||||
stderr: "boring stderr line\n-EncodedCommand secret",
|
||||
},
|
||||
);
|
||||
execMocks.runExec.mockRejectedValue(spawnError);
|
||||
|
||||
const error = await ensurePrivateSnapshotRepositoryRoot(tempDir).catch(
|
||||
(cause: unknown) => cause,
|
||||
);
|
||||
|
||||
expect(error).toMatchObject({
|
||||
message: expect.stringContaining("Unable to verify private Windows ACL for SQLite staging"),
|
||||
});
|
||||
const causes: Error[] = [];
|
||||
let current: unknown = error;
|
||||
while (current instanceof Error) {
|
||||
causes.push(current);
|
||||
current = (current as Error & { cause?: unknown }).cause;
|
||||
}
|
||||
expect(causes.map((cause) => cause.message).join("\n")).toContain(
|
||||
"Unable to verify private Windows ACL",
|
||||
);
|
||||
expect(causes.at(-1)?.message).toContain("code=ETIMEDOUT, killed=true");
|
||||
expect(causes.at(-1)?.message).toContain("stderr: boring stderr line");
|
||||
expect(causes).not.toContain(spawnError);
|
||||
for (const cause of causes) {
|
||||
const retainedText = Object.getOwnPropertyNames(cause)
|
||||
.map((key) => String((cause as unknown as Record<string, unknown>)[key]))
|
||||
.join("\n");
|
||||
expect(retainedText).not.toMatch(/encodedcommand/iu);
|
||||
expect(retainedText).not.toContain(encodedPayload);
|
||||
}
|
||||
expect(execMocks.runExec).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.any(Array),
|
||||
expect.objectContaining({ timeoutMs: WINDOWS_POWERSHELL_COLD_SPAWN_TIMEOUT_MS }),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -231,6 +231,8 @@ describe("production lint suppressions", () => {
|
||||
"src/plugins/runtime/runtime-plugin-boundary.ts|typescript/no-unnecessary-type-parameters|1",
|
||||
"src/plugins/runtime/types-channel.ts|typescript/no-unnecessary-type-parameters|1",
|
||||
"src/plugins/trusted-tool-policy.ts|typescript/no-unnecessary-type-parameters|1",
|
||||
// Raw PowerShell errors carry the -EncodedCommand argv; only the sanitized cause may escape.
|
||||
"src/secrets/private-plan-file.ts|preserve-caught-error|1",
|
||||
"src/state/config-machine-state.ts|typescript/no-unnecessary-type-parameters|1",
|
||||
"src/system-agent/setup-inference-activate.ts|no-unsafe-finally|1",
|
||||
"src/system-agent/setup-inference-activate.ts|preserve-caught-error|1",
|
||||
|
||||
Reference in New Issue
Block a user