mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-25 11:55:47 -06:00
fix(crabbox): preserve Machine0 provisioning budget (#128185)
* fix(crabbox): bound Machine0 lifecycle backoff recovery * fix(crabbox): reserve full Machine0 cleanup budget * fix(crabbox): allow paced Machine0 inspection * fix(crabbox): reserve Machine0 readiness retry * fix(crabbox): pace Machine0 readiness checks * fix(crabbox): use readiness-aware Machine0 status * feat(crabbox): forward explicit setup environment * fix(crabbox): preserve bounded command failure context * fix(crabbox): enforce exact setup environment
This commit is contained in:
committed by
GitHub
parent
32f49a4937
commit
044c0feb7c
@@ -24,4 +24,15 @@ contracts: `workerProviders`
|
||||
|
||||
See [Cloud worker environments](/gateway/configuration-reference#crabbox-profile) for the profile schema and lifecycle notes.
|
||||
|
||||
Forward Gateway environment variables to an operator-provided setup script by listing their names in the Crabbox profile settings:
|
||||
|
||||
```json5 validate=false
|
||||
{
|
||||
setup: 'install-worker "$OPENCLAW_WORKER_ARTIFACT_TOKEN"',
|
||||
setupEnv: ["OPENCLAW_WORKER_ARTIFACT_TOKEN"],
|
||||
}
|
||||
```
|
||||
|
||||
`setupEnv` explicitly forwards up to 16 unique environment variable names to the setup command only. Values are read from the Gateway process environment and are never stored in the profile configuration. Missing variables fail before a machine is allocated.
|
||||
|
||||
<!-- openclaw-plugin-reference:manual-end -->
|
||||
|
||||
@@ -1,23 +1,33 @@
|
||||
import { redactSensitiveText } from "openclaw/plugin-sdk/logging-core";
|
||||
import { WorkerProviderError } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import type { SpawnResult } from "openclaw/plugin-sdk/process-runtime";
|
||||
import { sliceUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
|
||||
import { sliceUtf16Safe, truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
|
||||
|
||||
const MAX_COMMAND_DETAIL_CHARS = 512;
|
||||
|
||||
function crabboxCommandDetail(result: SpawnResult): string {
|
||||
const raw = (result.stderr || result.stdout).trim();
|
||||
const raw = [result.stderr, result.stdout].filter(Boolean).join("\n").trim();
|
||||
if (!raw) {
|
||||
return "";
|
||||
}
|
||||
const compressed = redactSensitiveText(raw).replace(/\s+/gu, " ");
|
||||
const redacted = sliceUtf16Safe(compressed, -MAX_COMMAND_DETAIL_CHARS);
|
||||
const omitted = " ... ";
|
||||
const remaining = MAX_COMMAND_DETAIL_CHARS - omitted.length;
|
||||
const redacted =
|
||||
compressed.length <= MAX_COMMAND_DETAIL_CHARS
|
||||
? compressed
|
||||
: `${truncateUtf16Safe(compressed, Math.ceil(remaining / 2))}${omitted}${sliceUtf16Safe(
|
||||
compressed,
|
||||
-Math.floor(remaining / 2),
|
||||
)}`;
|
||||
return redacted ? `: ${redacted}` : "";
|
||||
}
|
||||
|
||||
export function crabboxCommandError(action: string, result: SpawnResult): Error {
|
||||
if (result.termination !== "exit") {
|
||||
return new Error(`Crabbox ${action} did not exit normally (${result.termination})`);
|
||||
return new Error(
|
||||
`Crabbox ${action} did not exit normally (${result.termination})${crabboxCommandDetail(result)}`,
|
||||
);
|
||||
}
|
||||
const exitCode = result.code === null ? "unknown" : String(result.code);
|
||||
return new Error(
|
||||
|
||||
@@ -18,6 +18,7 @@ const PROFILE_KEYS = new Set([
|
||||
"idleTimeout",
|
||||
"provider",
|
||||
"setup",
|
||||
"setupEnv",
|
||||
"ttl",
|
||||
]);
|
||||
const GO_DURATION_PATTERN = /^\+?(?:(?:\d+(?:\.\d*)?|\.\d+)(?:ns|us|µs|μs|ms|s|m|h))+$/u;
|
||||
@@ -45,6 +46,7 @@ type CrabboxProfile = {
|
||||
provider: string;
|
||||
ttl: string;
|
||||
setup?: string;
|
||||
setupEnv?: string[];
|
||||
};
|
||||
|
||||
const CRABBOX_FALLBACK_MACHINE_CLASSES = ["standard", "fast", "large", "beast"] as const;
|
||||
@@ -144,6 +146,32 @@ export function parseCrabboxProfile(profile: WorkerProfile): CrabboxProfile {
|
||||
if (setupValue !== undefined && !setup) {
|
||||
throw new WorkerProviderError("Crabbox profile setup must be a non-empty command string");
|
||||
}
|
||||
let setupEnv: string[] | undefined;
|
||||
if (profile.setupEnv !== undefined) {
|
||||
if (!Array.isArray(profile.setupEnv)) {
|
||||
throw new WorkerProviderError("Crabbox profile setupEnv must be an array");
|
||||
}
|
||||
if (profile.setupEnv.length > 16) {
|
||||
throw new WorkerProviderError("Crabbox profile setupEnv must contain at most 16 names");
|
||||
}
|
||||
setupEnv = profile.setupEnv.map((name) => {
|
||||
if (typeof name !== "string" || !/^[A-Za-z_][A-Za-z0-9_]*$/u.test(name)) {
|
||||
throw new WorkerProviderError(
|
||||
"Crabbox profile setupEnv must contain only valid POSIX environment variable names",
|
||||
);
|
||||
}
|
||||
if (name === "CRABBOX_ENV_ALLOW") {
|
||||
throw new WorkerProviderError(`Crabbox profile setupEnv name ${name} is reserved`);
|
||||
}
|
||||
return name;
|
||||
});
|
||||
if (new Set(setupEnv).size !== setupEnv.length) {
|
||||
throw new WorkerProviderError("Crabbox profile setupEnv must not contain duplicate names");
|
||||
}
|
||||
if (setupEnv.length > 0 && !setup) {
|
||||
throw new WorkerProviderError("Crabbox profile setupEnv requires setup");
|
||||
}
|
||||
}
|
||||
const desktop = profile.desktop;
|
||||
if (desktop !== undefined && typeof desktop !== "boolean") {
|
||||
throw new WorkerProviderError("Crabbox profile desktop must be a boolean");
|
||||
@@ -165,10 +193,46 @@ export function parseCrabboxProfile(profile: WorkerProfile): CrabboxProfile {
|
||||
idleTimeout,
|
||||
provider,
|
||||
setup,
|
||||
setupEnv,
|
||||
ttl,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveCrabboxProfileSetupEnv(
|
||||
setupEnv: readonly string[] | undefined,
|
||||
): Record<string, string> | undefined {
|
||||
if (!setupEnv?.length) {
|
||||
return undefined;
|
||||
}
|
||||
return Object.fromEntries(
|
||||
setupEnv.map((name) => {
|
||||
const value = process.env[name];
|
||||
if (!Object.hasOwn(process.env, name) || value === undefined) {
|
||||
throw new WorkerProviderError(`Crabbox profile setupEnv variable is missing: ${name}`);
|
||||
}
|
||||
return [name, value];
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveCrabboxProvisionProfile(
|
||||
profile: WorkerProfile,
|
||||
requestedClassValue: unknown,
|
||||
): { profile: CrabboxProfile; forwardedEnv?: Record<string, string> } {
|
||||
const configured = parseCrabboxProfile(profile);
|
||||
const requestedClass = nonEmptyString(requestedClassValue);
|
||||
if (
|
||||
requestedClassValue !== undefined &&
|
||||
(!requestedClass || requestedClass.length > MAX_CRABBOX_MACHINE_CLASS_LENGTH)
|
||||
) {
|
||||
throw new WorkerProviderError(
|
||||
"Crabbox machine class must be a non-empty string of at most 128 characters",
|
||||
);
|
||||
}
|
||||
const resolved = requestedClass ? { ...configured, class: requestedClass } : configured;
|
||||
return { profile: resolved, forwardedEnv: resolveCrabboxProfileSetupEnv(resolved.setupEnv) };
|
||||
}
|
||||
|
||||
export function listCrabboxMachineOptions(
|
||||
configuredClass: string,
|
||||
shapes: readonly CrabboxMachineShape[] | undefined,
|
||||
|
||||
@@ -14,6 +14,7 @@ import * as doctorRuntime from "./crabbox-worker-doctor-runtime.js";
|
||||
import {
|
||||
findCrabboxBinary,
|
||||
operationLeaseId,
|
||||
parseCrabboxProfile,
|
||||
resolveCrabboxBinary,
|
||||
} from "./crabbox-worker-profile.js";
|
||||
import { createCrabboxWorkerProvider, resolveOpenClawRoot } from "./crabbox-worker-provider.js";
|
||||
@@ -39,6 +40,7 @@ const PROFILE = {
|
||||
idleTimeout: "60m",
|
||||
};
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
afterEach(() => vi.unstubAllEnvs());
|
||||
|
||||
type CrabboxWorkerProviderDependencies = NonNullable<
|
||||
Parameters<typeof createCrabboxWorkerProvider>[0]
|
||||
@@ -79,13 +81,14 @@ function lifecycleLease(leaseId = LEASE_ID, profile: WorkerProfile = PROFILE) {
|
||||
function providerWithRawRunner(
|
||||
runCommand: CrabboxCommandRunner,
|
||||
warn?: (message: string) => void,
|
||||
sleep: (milliseconds: number) => Promise<void> = async () => {},
|
||||
): WorkerProvider {
|
||||
const provider = createCrabboxWorkerProvider({
|
||||
runCommand,
|
||||
openclawRoot: OPENCLAW_ROOT,
|
||||
pathEnv: "",
|
||||
isExecutable: (candidate) => candidate === SIBLING_BINARY,
|
||||
sleep: async () => {},
|
||||
sleep,
|
||||
wallpaperPath: WORKER_WALLPAPER_PATH,
|
||||
...(warn ? { warn } : {}),
|
||||
});
|
||||
@@ -109,13 +112,21 @@ function providerWithRawRunner(
|
||||
};
|
||||
}
|
||||
|
||||
function providerWithRunner(runCommand: CrabboxCommandRunner, warn?: (message: string) => void) {
|
||||
return providerWithRawRunner(async (argv, options) => {
|
||||
if (argv[1] === "config" && argv[2] === "show") {
|
||||
return commandResult({ stdout: JSON.stringify({ aws: { instanceProfile: "" } }) });
|
||||
}
|
||||
return runCommand(argv, options);
|
||||
}, warn);
|
||||
function providerWithRunner(
|
||||
runCommand: CrabboxCommandRunner,
|
||||
warn?: (message: string) => void,
|
||||
sleep?: (milliseconds: number) => Promise<void>,
|
||||
) {
|
||||
return providerWithRawRunner(
|
||||
async (argv, options) => {
|
||||
if (argv[1] === "config" && argv[2] === "show") {
|
||||
return commandResult({ stdout: JSON.stringify({ aws: { instanceProfile: "" } }) });
|
||||
}
|
||||
return runCommand(argv, options);
|
||||
},
|
||||
warn,
|
||||
sleep,
|
||||
);
|
||||
}
|
||||
|
||||
function failedNodeEnrollment(
|
||||
@@ -625,42 +636,93 @@ describe("Crabbox worker provider", () => {
|
||||
expect(calls.flatMap((call) => call.argv)).not.toContain("rsync");
|
||||
});
|
||||
|
||||
it("runs the profile setup command on the ready lease and keeps it", async () => {
|
||||
const calls: Array<{ argv: string[]; options: Parameters<CrabboxCommandRunner>[1] }> = [];
|
||||
let warmed = false;
|
||||
const provider = providerWithRunner(async (argv, options) => {
|
||||
calls.push({ argv, options });
|
||||
if (argv[1] === "warmup") {
|
||||
warmed = true;
|
||||
return commandResult({ stdout: `leased ${LEASE_ID} slug=test\n` });
|
||||
it.each([
|
||||
{ name: "without forwarded environment", setupEnv: undefined, forwardedEnv: undefined },
|
||||
{
|
||||
name: "with only explicitly forwarded Gateway environment",
|
||||
setupEnv: ["OPENCLAW_WORKER_ARTIFACT_TOKEN", "CRABBOX_EMPTY_VALUE"],
|
||||
forwardedEnv: {
|
||||
OPENCLAW_WORKER_ARTIFACT_TOKEN: "fixture-artifact-token",
|
||||
CRABBOX_EMPTY_VALUE: "",
|
||||
},
|
||||
},
|
||||
])(
|
||||
"runs profile setup $name without widening node enrollment",
|
||||
async ({ setupEnv, forwardedEnv }) => {
|
||||
vi.stubEnv("CRABBOX_ENV_ALLOW", "OPENCLAW_UNSELECTED_SECRET");
|
||||
vi.stubEnv("OPENCLAW_UNSELECTED_SECRET", "unselected-secret");
|
||||
for (const [name, value] of Object.entries(forwardedEnv ?? {})) {
|
||||
vi.stubEnv(name, value);
|
||||
}
|
||||
if (argv[1] === "run") {
|
||||
return commandResult();
|
||||
}
|
||||
return warmed || argv.includes(LEASE_ID)
|
||||
? commandResult({ stdout: inspectJson({ sshHostKey: HOST_KEY }) })
|
||||
: commandResult({ code: 4, stderr: `lease/server not found: ${argv.at(-2)}` });
|
||||
});
|
||||
const calls: Array<{ argv: string[]; options: Parameters<CrabboxCommandRunner>[1] }> = [];
|
||||
let warmed = false;
|
||||
const provider = providerWithRunner(async (argv, options) => {
|
||||
calls.push({ argv, options });
|
||||
if (argv[1] === "warmup") {
|
||||
warmed = true;
|
||||
return commandResult({ stdout: `leased ${LEASE_ID} slug=test\n` });
|
||||
}
|
||||
if (argv[1] === "run") {
|
||||
return commandResult();
|
||||
}
|
||||
return warmed || argv.includes(LEASE_ID)
|
||||
? commandResult({ stdout: inspectJson({ sshHostKey: HOST_KEY }) })
|
||||
: commandResult({ code: 4, stderr: `lease/server not found: ${argv.at(-2)}` });
|
||||
});
|
||||
|
||||
const setup = "command -v node || install-node";
|
||||
await expect(provider.provision({ ...PROFILE, setup }, OPERATION_ID)).resolves.toMatchObject({
|
||||
leaseId: LEASE_ID,
|
||||
const setup = "command -v node || install-node";
|
||||
const profile = { ...PROFILE, setup, ...(setupEnv ? { setupEnv } : {}) };
|
||||
await expect(provider.provision(profile, OPERATION_ID)).resolves.toMatchObject({
|
||||
leaseId: LEASE_ID,
|
||||
});
|
||||
const [setupCall, enrollmentCall] = calls.filter((call) => call.argv[1] === "run");
|
||||
expect(setupCall?.argv.slice(1)).toEqual([
|
||||
"run",
|
||||
"--provider",
|
||||
"aws",
|
||||
"--network",
|
||||
"public",
|
||||
"--tailscale=false",
|
||||
"--id",
|
||||
LEASE_ID,
|
||||
"--keep=true",
|
||||
"--no-sync",
|
||||
...Object.keys(forwardedEnv ?? {}).flatMap((name) => ["--allow-env", name]),
|
||||
"--script-stdin",
|
||||
]);
|
||||
expect(setupCall?.options.env).toEqual({
|
||||
...forwardedEnv,
|
||||
CRABBOX_ENV_ALLOW: setupEnv?.join(",") || ",",
|
||||
});
|
||||
expect(setupCall?.options.input).toBe(setup);
|
||||
expect(enrollmentCall?.options.env).toEqual({
|
||||
CRABBOX_WORKER_SETUP_CODE: "secret-setup-value",
|
||||
});
|
||||
expect(
|
||||
enrollmentCall?.argv.filter((argument, index, argv) => argv[index - 1] === "--allow-env"),
|
||||
).toEqual(["CRABBOX_WORKER_SETUP_CODE"]);
|
||||
expect(
|
||||
calls.filter((call) => call.argv[1] !== "run").every((call) => !call.options.env),
|
||||
).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects a missing profile setup environment variable before invoking Crabbox", async () => {
|
||||
const missingName = "OPENCLAW_MISSING_WORKER_ARTIFACT_TOKEN";
|
||||
vi.stubEnv(missingName, undefined);
|
||||
const runCommand = vi.fn<CrabboxCommandRunner>();
|
||||
const provider = providerWithRawRunner(runCommand);
|
||||
|
||||
await expect(
|
||||
provider.provision(
|
||||
{ ...PROFILE, setup: "install-node", setupEnv: [missingName] },
|
||||
OPERATION_ID,
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
code: "invalid_profile",
|
||||
message: expect.stringContaining(missingName),
|
||||
});
|
||||
const runCall = calls.find((call) => call.argv[1] === "run");
|
||||
expect(runCall?.argv.slice(1)).toEqual([
|
||||
"run",
|
||||
"--provider",
|
||||
"aws",
|
||||
"--network",
|
||||
"public",
|
||||
"--tailscale=false",
|
||||
"--id",
|
||||
LEASE_ID,
|
||||
"--keep=true",
|
||||
"--no-sync",
|
||||
"--script-stdin",
|
||||
]);
|
||||
expect(runCall?.options.input).toBe(setup);
|
||||
expect(runCommand).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("waits for post-setup SSH readiness and returns the final endpoint", async () => {
|
||||
@@ -830,7 +892,7 @@ describe("Crabbox worker provider", () => {
|
||||
it("preserves the allocated lease and both failures when setup cleanup times out", async () => {
|
||||
let releaseCommitted = false;
|
||||
const provider = providerWithRunner(async (argv) => {
|
||||
if (argv[1] === "inspect") {
|
||||
if (argv[1] === "inspect" || argv[1] === "status") {
|
||||
return commandResult({ stdout: inspectJson({ sshHostKey: HOST_KEY }) });
|
||||
}
|
||||
if (argv[1] === "run") {
|
||||
@@ -1277,6 +1339,16 @@ describe("Crabbox worker provider", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["OPENCLAW_WORKER_ARTIFACT_TOKEN"],
|
||||
["OPENCLAW_WORKER_ARTIFACT_TOKEN", "_SECOND_VALUE2"],
|
||||
])("accepts valid profile setup environment names %j", (...setupEnv) => {
|
||||
expect(parseCrabboxProfile({ ...PROFILE, setup: "install-node", setupEnv })).toMatchObject({
|
||||
setup: "install-node",
|
||||
setupEnv,
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
[`provision:v2:${"0".repeat(64)}`, "cbx_6071fc2062a6"],
|
||||
[`provision:v2:${"a".repeat(64)}`, "cbx_d75d2e596dde"],
|
||||
@@ -1390,7 +1462,7 @@ describe("Crabbox worker provider", () => {
|
||||
if (argv[1] === "config" && argv[2] === "show") {
|
||||
return commandResult({ stdout: JSON.stringify(config) });
|
||||
}
|
||||
if (argv[1] === "inspect") {
|
||||
if (argv[1] === "inspect" || argv[1] === "status") {
|
||||
return commandResult({ stdout: inspectJson({ sshHostKey: HOST_KEY }) });
|
||||
}
|
||||
if (argv[1] === "run" && String(options.input).includes("openclaw-worker-browser")) {
|
||||
@@ -1506,7 +1578,7 @@ describe("Crabbox worker provider", () => {
|
||||
const pairingSecret = "pairing-secret-value-0123456789";
|
||||
const provider = providerWithRunner(async (argv, options) => {
|
||||
calls.push({ argv, options });
|
||||
if (argv[1] === "inspect") {
|
||||
if (argv[1] === "inspect" || argv[1] === "status") {
|
||||
return commandResult({ stdout: inspectJson({ sshHostKey: HOST_KEY }) });
|
||||
}
|
||||
if (argv[1] === "run" && String(options.input).includes("node.log tail:")) {
|
||||
@@ -1579,7 +1651,7 @@ describe("Crabbox worker provider", () => {
|
||||
const calls: string[][] = [];
|
||||
const provider = providerWithRunner(async (argv, options) => {
|
||||
calls.push(argv);
|
||||
if (argv[1] === "inspect") {
|
||||
if (argv[1] === "inspect" || argv[1] === "status") {
|
||||
return commandResult({ stdout: inspectJson({ sshHostKey: HOST_KEY }) });
|
||||
}
|
||||
if (argv[1] === "run" && String(options.input).includes("node.log tail:")) {
|
||||
@@ -1614,7 +1686,7 @@ describe("Crabbox worker provider", () => {
|
||||
},
|
||||
])("bounds enrollment evidence for $name", async ({ output, expected }) => {
|
||||
const provider = providerWithRunner(async (argv, options) => {
|
||||
if (argv[1] === "inspect") {
|
||||
if (argv[1] === "inspect" || argv[1] === "status") {
|
||||
return commandResult({ stdout: inspectJson({ sshHostKey: HOST_KEY }) });
|
||||
}
|
||||
if (argv[1] === "run" && String(options.input).includes("node.log tail:")) {
|
||||
@@ -1686,79 +1758,234 @@ describe("Crabbox worker provider", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("runs one fixed warmup, ignores its output, and inspects only the canonical id", async () => {
|
||||
const calls: Array<{ argv: string[]; options: Parameters<CrabboxCommandRunner>[1] }> = [];
|
||||
it.each([
|
||||
{
|
||||
providerId: "aws",
|
||||
warmupTimeoutMs: 50 * 60_000,
|
||||
lifecycleTimeoutMs: 60_000,
|
||||
provisionTimeoutMs: 67 * 60_000,
|
||||
},
|
||||
{
|
||||
providerId: "hetzner",
|
||||
warmupTimeoutMs: 50 * 60_000,
|
||||
lifecycleTimeoutMs: 60_000,
|
||||
provisionTimeoutMs: 67 * 60_000,
|
||||
},
|
||||
{
|
||||
providerId: "machine0",
|
||||
warmupTimeoutMs: 50 * 60_000,
|
||||
lifecycleTimeoutMs: 5 * 60_000,
|
||||
provisionTimeoutMs: 80 * 60_000,
|
||||
},
|
||||
])(
|
||||
"runs one fixed $providerId warmup, ignores its output, and inspects only the canonical id",
|
||||
async ({ providerId, warmupTimeoutMs, lifecycleTimeoutMs, provisionTimeoutMs }) => {
|
||||
const calls: Array<{ argv: string[]; options: Parameters<CrabboxCommandRunner>[1] }> = [];
|
||||
const provider = providerWithRunner(async (argv, options) => {
|
||||
calls.push({ argv, options });
|
||||
return argv[1] === "warmup"
|
||||
? commandResult({ stdout: "warmup completed without a lease token\n" })
|
||||
: commandResult({ stdout: inspectJson({ sshHostKey: HOST_KEY }) });
|
||||
});
|
||||
const profile = { ...PROFILE, provider: providerId };
|
||||
const readinessAction = providerId === "machine0" ? "status" : "inspect";
|
||||
|
||||
await expect(provider.provision(profile, OPERATION_ID)).resolves.toMatchObject({
|
||||
leaseId: LEASE_ID,
|
||||
});
|
||||
expect(calls).toHaveLength(4);
|
||||
expect(calls[0]?.argv).toEqual([
|
||||
SIBLING_BINARY,
|
||||
"warmup",
|
||||
"--provider",
|
||||
providerId,
|
||||
"--network",
|
||||
"public",
|
||||
"--tailscale=false",
|
||||
"--class",
|
||||
"standard",
|
||||
"--ttl",
|
||||
"24h",
|
||||
"--idle-timeout",
|
||||
"60m",
|
||||
"--lease-id",
|
||||
LEASE_ID,
|
||||
"--slug",
|
||||
expect.stringMatching(/^openclaw-[a-f0-9]{32}$/u),
|
||||
"--keep=true",
|
||||
]);
|
||||
expect({
|
||||
warmupOptions: calls[0]?.options,
|
||||
provisionTimeoutMs: provider.resolveProvisionTimeoutMs?.(profile),
|
||||
provisionTimeoutWithSetupMs: provider.resolveProvisionTimeoutMs?.({
|
||||
...profile,
|
||||
setup: "install-node",
|
||||
}),
|
||||
}).toEqual({
|
||||
warmupOptions: {
|
||||
timeoutMs: warmupTimeoutMs,
|
||||
maxOutputBytes: 65_536,
|
||||
killProcessTree: true,
|
||||
},
|
||||
provisionTimeoutMs,
|
||||
provisionTimeoutWithSetupMs: provisionTimeoutMs + 15 * 60_000,
|
||||
});
|
||||
expect(calls[1]?.argv).toEqual([
|
||||
SIBLING_BINARY,
|
||||
readinessAction,
|
||||
"--provider",
|
||||
providerId,
|
||||
"--network",
|
||||
"public",
|
||||
"--id",
|
||||
LEASE_ID,
|
||||
...(providerId === "machine0" ? ["--wait", "--wait-timeout", "4m"] : []),
|
||||
"--json",
|
||||
]);
|
||||
expect(calls[1]?.options.timeoutMs).toBe(lifecycleTimeoutMs);
|
||||
expect(calls[2]?.argv[1]).toBe("run");
|
||||
expect(String(calls[2]?.options.input)).toContain("openclaw@2026.8.1");
|
||||
expect(String(calls[2]?.options.input)).toContain(
|
||||
"'OpenClaw 2026.8.1'|'OpenClaw 2026.8.1 '*",
|
||||
);
|
||||
expect(String(calls[2]?.options.input)).toContain(
|
||||
'npx --yes --package "$package_spec" -- openclaw',
|
||||
);
|
||||
expect(String(calls[2]?.options.input)).toContain(
|
||||
"OpenClaw worker bootstrap could not install Gateway version 2026.8.1",
|
||||
);
|
||||
expect(String(calls[2]?.options.input)).toContain(
|
||||
'connect --target-file "$setup_code_file" --ephemeral',
|
||||
);
|
||||
expect(String(calls[2]?.options.input)).toContain("setsid -f sh -c");
|
||||
expect(String(calls[2]?.options.input)).not.toContain(
|
||||
"config set nodeHost.workerRuns.enabled",
|
||||
);
|
||||
expect(String(calls[2]?.options.input)).not.toContain("nohup");
|
||||
expect(String(calls[2]?.options.input)).not.toContain("secret-setup-value");
|
||||
expect(calls[2]?.options.env).toMatchObject({
|
||||
CRABBOX_WORKER_SETUP_CODE: "secret-setup-value",
|
||||
});
|
||||
expect(calls[2]?.argv).toEqual(
|
||||
expect.arrayContaining(["--allow-env", "CRABBOX_WORKER_SETUP_CODE"]),
|
||||
);
|
||||
expect(calls[2]?.argv.join(" ")).not.toContain("setup-code");
|
||||
expect(calls[3]?.argv[1]).toBe(readinessAction);
|
||||
expect(calls[3]?.options.timeoutMs).toBe(lifecycleTimeoutMs);
|
||||
|
||||
const lease = lifecycleLease(LEASE_ID, profile);
|
||||
await expect(provider.inspect(lease)).resolves.toEqual({ status: "active" });
|
||||
await expect(provider.destroy(lease)).resolves.toBeUndefined();
|
||||
expect(calls.slice(4).map(({ argv, options }) => [argv[1], options.timeoutMs])).toEqual([
|
||||
["inspect", lifecycleTimeoutMs],
|
||||
["stop", lifecycleTimeoutMs],
|
||||
]);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ providerId: "aws", expectedIntervalMs: 2_000 },
|
||||
{ providerId: "hetzner", expectedIntervalMs: 2_000 },
|
||||
{ providerId: "machine0", expectedIntervalMs: 60_000 },
|
||||
])(
|
||||
"paces $providerId readiness re-inspection at $expectedIntervalMs ms",
|
||||
async ({ providerId, expectedIntervalMs }) => {
|
||||
let inspections = 0;
|
||||
const delays: number[] = [];
|
||||
const provider = providerWithRunner(
|
||||
async (argv) => {
|
||||
if (argv[1] === "inspect" || argv[1] === "status") {
|
||||
inspections += 1;
|
||||
return commandResult({
|
||||
stdout: inspectJson({ ready: inspections > 1, sshHostKey: HOST_KEY }),
|
||||
});
|
||||
}
|
||||
return commandResult();
|
||||
},
|
||||
undefined,
|
||||
async (milliseconds) => {
|
||||
delays.push(milliseconds);
|
||||
},
|
||||
);
|
||||
|
||||
await expect(
|
||||
provider.provision({ ...PROFILE, provider: providerId }, OPERATION_ID),
|
||||
).resolves.toMatchObject({ leaseId: LEASE_ID });
|
||||
expect(delays).toEqual([expectedIntervalMs]);
|
||||
},
|
||||
);
|
||||
|
||||
it("reserves separate Machine0 inspection and readiness windows after a near-max warmup", async () => {
|
||||
const profile = { ...PROFILE, provider: "machine0" };
|
||||
let elapsedMs = 0;
|
||||
const inspectTimeouts: number[] = [];
|
||||
const now = vi.spyOn(Date, "now").mockImplementation(() => elapsedMs);
|
||||
const provider = providerWithRunner(async (argv, options) => {
|
||||
calls.push({ argv, options });
|
||||
return argv[1] === "warmup"
|
||||
? commandResult({ stdout: "warmup completed without a lease token\n" })
|
||||
: commandResult({ stdout: inspectJson({ sshHostKey: HOST_KEY }) });
|
||||
if (argv[1] === "warmup") {
|
||||
elapsedMs = 50 * 60_000;
|
||||
return commandResult();
|
||||
}
|
||||
if (argv[1] === "inspect" || argv[1] === "status") {
|
||||
inspectTimeouts.push(options.timeoutMs);
|
||||
if (inspectTimeouts.length <= 2) {
|
||||
elapsedMs += 4 * 60_000;
|
||||
}
|
||||
return commandResult({
|
||||
stdout: inspectJson({ ready: inspectTimeouts.length > 1, sshHostKey: HOST_KEY }),
|
||||
});
|
||||
}
|
||||
return commandResult();
|
||||
});
|
||||
|
||||
await expect(provider.provision(PROFILE, OPERATION_ID)).resolves.toMatchObject({
|
||||
leaseId: LEASE_ID,
|
||||
try {
|
||||
await expect(provider.provision(profile, OPERATION_ID)).resolves.toMatchObject({
|
||||
leaseId: LEASE_ID,
|
||||
});
|
||||
expect(inspectTimeouts.slice(0, 2)).toEqual([5 * 60_000, 5 * 60_000]);
|
||||
} finally {
|
||||
now.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("reserves the full Machine0 cleanup budget after late node enrollment failure", async () => {
|
||||
const profile = { ...PROFILE, provider: "machine0" };
|
||||
let elapsedMs = 0;
|
||||
let cleanupTimeoutMs = 0;
|
||||
const now = vi.spyOn(Date, "now").mockImplementation(() => elapsedMs);
|
||||
const provider = providerWithRunner(async (argv, options) => {
|
||||
if (argv[1] === "inspect" || argv[1] === "status") {
|
||||
return commandResult({ stdout: inspectJson({ sshHostKey: HOST_KEY }) });
|
||||
}
|
||||
if (argv[1] === "stop") {
|
||||
cleanupTimeoutMs = options.timeoutMs;
|
||||
elapsedMs += options.timeoutMs;
|
||||
return commandResult({ code: null, killed: true, termination: "timeout" });
|
||||
}
|
||||
return commandResult();
|
||||
});
|
||||
expect(calls).toHaveLength(4);
|
||||
expect(calls[0]?.argv).toEqual([
|
||||
SIBLING_BINARY,
|
||||
"warmup",
|
||||
"--provider",
|
||||
"aws",
|
||||
"--network",
|
||||
"public",
|
||||
"--tailscale=false",
|
||||
"--class",
|
||||
"standard",
|
||||
"--ttl",
|
||||
"24h",
|
||||
"--idle-timeout",
|
||||
"60m",
|
||||
"--lease-id",
|
||||
LEASE_ID,
|
||||
"--slug",
|
||||
expect.stringMatching(/^openclaw-[a-f0-9]{32}$/u),
|
||||
"--keep=true",
|
||||
]);
|
||||
expect(calls[0]?.options).toEqual({
|
||||
timeoutMs: 50 * 60_000,
|
||||
maxOutputBytes: 65_536,
|
||||
killProcessTree: true,
|
||||
});
|
||||
expect(calls[1]?.argv).toEqual([
|
||||
SIBLING_BINARY,
|
||||
"inspect",
|
||||
"--provider",
|
||||
"aws",
|
||||
"--network",
|
||||
"public",
|
||||
"--id",
|
||||
LEASE_ID,
|
||||
"--json",
|
||||
]);
|
||||
expect(calls[2]?.argv[1]).toBe("run");
|
||||
expect(String(calls[2]?.options.input)).toContain("openclaw@2026.8.1");
|
||||
expect(String(calls[2]?.options.input)).toContain("'OpenClaw 2026.8.1'|'OpenClaw 2026.8.1 '*");
|
||||
expect(String(calls[2]?.options.input)).toContain(
|
||||
'npx --yes --package "$package_spec" -- openclaw',
|
||||
);
|
||||
expect(String(calls[2]?.options.input)).toContain(
|
||||
"OpenClaw worker bootstrap could not install Gateway version 2026.8.1",
|
||||
);
|
||||
expect(String(calls[2]?.options.input)).toContain(
|
||||
'connect --target-file "$setup_code_file" --ephemeral',
|
||||
);
|
||||
expect(String(calls[2]?.options.input)).toContain("setsid -f sh -c");
|
||||
expect(String(calls[2]?.options.input)).not.toContain("config set nodeHost.workerRuns.enabled");
|
||||
expect(String(calls[2]?.options.input)).not.toContain("nohup");
|
||||
expect(String(calls[2]?.options.input)).not.toContain("secret-setup-value");
|
||||
expect(calls[2]?.options.env).toMatchObject({
|
||||
CRABBOX_WORKER_SETUP_CODE: "secret-setup-value",
|
||||
});
|
||||
expect(calls[2]?.argv).toEqual(
|
||||
expect.arrayContaining(["--allow-env", "CRABBOX_WORKER_SETUP_CODE"]),
|
||||
);
|
||||
expect(calls[2]?.argv.join(" ")).not.toContain("setup-code");
|
||||
expect(calls[3]?.argv[1]).toBe("inspect");
|
||||
|
||||
try {
|
||||
await expect(
|
||||
provider.provision(profile, OPERATION_ID, {
|
||||
beginNodeEnrollment: async () => ({
|
||||
mode: "resume" as const,
|
||||
deviceId: "device-bound",
|
||||
openclawVersion: "2026.8.1",
|
||||
packageSpecs: ["openclaw@2026.8.1"],
|
||||
displayName: "Bound worker",
|
||||
waitForDeviceId: async () => {
|
||||
elapsedMs = 75 * 60_000;
|
||||
throw new Error("node enrollment expired");
|
||||
},
|
||||
}),
|
||||
}),
|
||||
).rejects.toMatchObject({ code: "cleanup_indeterminate", leaseId: LEASE_ID });
|
||||
|
||||
expect(cleanupTimeoutMs).toBe(5 * 60_000);
|
||||
expect(provider.resolveProvisionTimeoutMs?.(profile)).toBe(elapsedMs);
|
||||
} finally {
|
||||
now.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("overrides the configured class for one provision operation", async () => {
|
||||
@@ -2036,7 +2263,7 @@ describe("Crabbox worker provider", () => {
|
||||
pathEnv: "",
|
||||
isExecutable: (candidate) => candidate === SIBLING_BINARY,
|
||||
sleep: async () => {
|
||||
nowMs += resolveCrabboxProvisionBaseTimeoutMs({}) + 1;
|
||||
nowMs += resolveCrabboxProvisionBaseTimeoutMs(PROFILE) + 1;
|
||||
},
|
||||
wallpaperPath: WORKER_WALLPAPER_PATH,
|
||||
});
|
||||
@@ -2086,6 +2313,29 @@ describe("Crabbox worker provider", () => {
|
||||
{ profile: { ...PROFILE, idleTimeout: "0s" }, message: "positive Go duration" },
|
||||
{ profile: { ...PROFILE, binary: " " }, message: "binary" },
|
||||
{ profile: { ...PROFILE, binary: "crabbox" }, message: "absolute path" },
|
||||
{ profile: { ...PROFILE, setup: "install-node", setupEnv: "TOKEN" }, message: "array" },
|
||||
{ profile: { ...PROFILE, setup: "install-node", setupEnv: null }, message: "array" },
|
||||
{ profile: { ...PROFILE, setup: "install-node", setupEnv: [4] }, message: "valid" },
|
||||
{ profile: { ...PROFILE, setup: "install-node", setupEnv: [""] }, message: "valid" },
|
||||
{ profile: { ...PROFILE, setup: "install-node", setupEnv: ["1TOKEN"] }, message: "valid" },
|
||||
{ profile: { ...PROFILE, setup: "install-node", setupEnv: ["BAD-NAME"] }, message: "valid" },
|
||||
{
|
||||
profile: { ...PROFILE, setup: "install-node", setupEnv: ["CRABBOX_ENV_ALLOW"] },
|
||||
message: "CRABBOX_ENV_ALLOW is reserved",
|
||||
},
|
||||
{
|
||||
profile: { ...PROFILE, setup: "install-node", setupEnv: ["TOKEN", "TOKEN"] },
|
||||
message: "duplicate",
|
||||
},
|
||||
{
|
||||
profile: {
|
||||
...PROFILE,
|
||||
setup: "install-node",
|
||||
setupEnv: Array.from({ length: 17 }, (_, index) => `TOKEN_${index}`),
|
||||
},
|
||||
message: "at most 16",
|
||||
},
|
||||
{ profile: { ...PROFILE, setupEnv: ["TOKEN"] }, message: "requires setup" },
|
||||
{ profile: { ...PROFILE, typo: true }, message: "unknown" },
|
||||
])("rejects an invalid profile ($message)", async ({ profile, message }) => {
|
||||
let invoked = false;
|
||||
@@ -2479,56 +2729,67 @@ describe("Crabbox worker provider", () => {
|
||||
await expect(cliMissing.inspect(lease)).rejects.toThrow("inspect could not start");
|
||||
});
|
||||
|
||||
it("retains the redacted terminal provider failure after verbose provisioning progress", async () => {
|
||||
const secret = ["sk", "abcdefghijklmnop"].join("-");
|
||||
const terminalCause = "machine0 provider rejected authenticated provisioning";
|
||||
const failurePrefix = "Crabbox warmup failed with exit code 5: ";
|
||||
const provider = providerWithRunner(async () =>
|
||||
commandResult({
|
||||
code: 5,
|
||||
stderr: [
|
||||
"coordinator lease class=standard preferred_type=machine0",
|
||||
"provisioning progress ".repeat(100),
|
||||
`token=${secret}`,
|
||||
terminalCause,
|
||||
].join("\n"),
|
||||
stdout: "stdout must not replace stderr",
|
||||
}),
|
||||
);
|
||||
|
||||
const error = await provider
|
||||
.provision({ ...PROFILE, provider: "machine0" }, OPERATION_ID)
|
||||
.catch((cause: unknown) => cause);
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
const message = error instanceof Error ? error.message : "";
|
||||
expect(message).toContain(terminalCause);
|
||||
expect(message).not.toContain(secret);
|
||||
expect(message).not.toContain("stdout must not replace stderr");
|
||||
expect(message).toHaveLength(failurePrefix.length + 512);
|
||||
expect(message.startsWith(failurePrefix)).toBe(true);
|
||||
});
|
||||
|
||||
it("preserves UTF-16 boundaries at the start of terminal provider failure details", async () => {
|
||||
const suffix = "x".repeat(511);
|
||||
const provider = providerWithRunner(async () =>
|
||||
commandResult({ code: 2, stderr: `progress😀${suffix}` }),
|
||||
);
|
||||
|
||||
const error = await provider.inspect(lifecycleLease()).catch((cause: unknown) => cause);
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
const message = error instanceof Error ? error.message : "";
|
||||
expect(message).toBe(`${INSPECT_FAILURE_PREFIX}${suffix}`);
|
||||
expect(hasLoneSurrogate(message)).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ name: "exactly at the bound", prefix: "" },
|
||||
{ name: "beyond the bound", prefix: "earlier progress " },
|
||||
])("keeps a complete stdout boundary pair $name", async ({ prefix }) => {
|
||||
{ action: "warmup", termination: "exit", code: 5 },
|
||||
{ action: "inspect", termination: "timeout", code: null },
|
||||
] as const)(
|
||||
"preserves bounded, redacted terminal diagnostics for $action $termination failures",
|
||||
async ({ action, termination, code }) => {
|
||||
const secret = ["sk", "abcdefghijklmnop"].join("-");
|
||||
const terminalStderr = "Machine0 terminal stderr: provider quota exhausted";
|
||||
const terminalStdout = "Machine0 terminal stdout: quota window has not reset";
|
||||
const provider = providerWithRunner(async () =>
|
||||
commandResult({
|
||||
code,
|
||||
termination,
|
||||
killed: termination !== "exit",
|
||||
stderr: `provider warning ${secret} ${"provider progress ".repeat(90)}${terminalStderr}`,
|
||||
stdout: terminalStdout,
|
||||
}),
|
||||
);
|
||||
const operation =
|
||||
action === "warmup"
|
||||
? provider.provision({ ...PROFILE, provider: "machine0" }, OPERATION_ID)
|
||||
: provider.inspect(lifecycleLease());
|
||||
|
||||
const error = await operation.catch((cause: unknown) => cause);
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
const message = error instanceof Error ? error.message : "";
|
||||
const failurePrefix =
|
||||
action === "warmup"
|
||||
? "Crabbox warmup failed with exit code 5: "
|
||||
: "Crabbox inspect did not exit normally (timeout): ";
|
||||
expect(message).toContain("provider warning");
|
||||
expect(message).toContain(terminalStderr);
|
||||
expect(message).toContain(terminalStdout);
|
||||
expect(message).not.toContain(secret);
|
||||
expect(message.length).toBeLessThanOrEqual(failurePrefix.length + 512);
|
||||
expect(hasLoneSurrogate(message)).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["stderr", "stdout"] as const)(
|
||||
"preserves UTF-16 boundaries and terminal detail from %s",
|
||||
async (stream) => {
|
||||
const provider = providerWithRunner(async () =>
|
||||
commandResult({
|
||||
code: 2,
|
||||
[stream]: `${"x".repeat(253)}😀${"y".repeat(300)}😀 terminal failure`,
|
||||
}),
|
||||
);
|
||||
|
||||
const error = await provider.inspect(lifecycleLease()).catch((cause: unknown) => cause);
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
const message = error instanceof Error ? error.message : "";
|
||||
expect(message).toContain("😀 terminal failure");
|
||||
expect(message.length).toBeLessThanOrEqual(INSPECT_FAILURE_PREFIX.length + 512);
|
||||
expect(hasLoneSurrogate(message)).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps a complete stdout boundary pair exactly at the bound", async () => {
|
||||
const detail = `${"x".repeat(510)}😀`;
|
||||
const provider = providerWithRunner(async () =>
|
||||
commandResult({ code: 2, stdout: `${prefix}${detail}` }),
|
||||
);
|
||||
const provider = providerWithRunner(async () => commandResult({ code: 2, stdout: detail }));
|
||||
|
||||
const error = await provider.inspect(lifecycleLease()).catch((cause: unknown) => cause);
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
|
||||
@@ -40,22 +40,25 @@ import {
|
||||
operationSlug,
|
||||
parseCrabboxProfile,
|
||||
resolveCrabboxBinary,
|
||||
resolveCrabboxProvisionProfile,
|
||||
} from "./crabbox-worker-profile.js";
|
||||
import {
|
||||
countCrabboxProvisionSetupPhases,
|
||||
CRABBOX_DESKTOP_WARMUP_TIMEOUT_MS,
|
||||
CRABBOX_LIFECYCLE_TIMEOUT_MS,
|
||||
CRABBOX_MACHINE0_READY_WAIT_TIMEOUT,
|
||||
CRABBOX_NODE_ENROLLMENT_TIMEOUT_MS,
|
||||
CRABBOX_SETUP_TIMEOUT_MS,
|
||||
CRABBOX_WARMUP_TIMEOUT_MS,
|
||||
resolveCrabboxLifecycleTimeoutMs,
|
||||
resolveCrabboxProvisionBaseTimeoutMs,
|
||||
resolveCrabboxProvisionCallTimeoutMs,
|
||||
resolveCrabboxReadyPollIntervalMs,
|
||||
} from "./crabbox-worker-timeouts.js";
|
||||
import { loadCrabboxWorkerWallpaperBase64 } from "./crabbox-worker-wallpaper.js";
|
||||
|
||||
export { resolveOpenClawRoot } from "./crabbox-worker-profile.js";
|
||||
|
||||
const READY_POLL_INTERVAL_MS = 2_000;
|
||||
const MAX_ERROR_DETAIL_CHARS = 512;
|
||||
// Only states that prove the resource is gone or stopped map to `destroyed`. Crabbox also
|
||||
// treats `deleting` and `failed` as unable to become ready, but those can retain resources
|
||||
@@ -155,22 +158,27 @@ async function inspectWithContext(params: {
|
||||
id: string;
|
||||
runCommand: CrabboxCommandRunner;
|
||||
timeoutMs?: number;
|
||||
waitForReady?: boolean;
|
||||
}): Promise<InspectCommandResult> {
|
||||
const action = params.waitForReady ? "status" : "inspect";
|
||||
const result = await runCrabboxCommand({
|
||||
action: "inspect",
|
||||
action,
|
||||
args: [
|
||||
"inspect",
|
||||
action,
|
||||
"--provider",
|
||||
params.context.provider,
|
||||
"--network",
|
||||
"public",
|
||||
"--id",
|
||||
params.id,
|
||||
...(params.waitForReady
|
||||
? ["--wait", "--wait-timeout", CRABBOX_MACHINE0_READY_WAIT_TIMEOUT]
|
||||
: []),
|
||||
"--json",
|
||||
],
|
||||
binary: params.context.binary,
|
||||
runCommand: params.runCommand,
|
||||
timeoutMs: params.timeoutMs ?? CRABBOX_LIFECYCLE_TIMEOUT_MS,
|
||||
timeoutMs: params.timeoutMs ?? resolveCrabboxLifecycleTimeoutMs(params.context.provider),
|
||||
});
|
||||
if (result.termination === "exit" && result.code === 0) {
|
||||
// A successful but malformed response cannot attest the fixed lease. Command failures and
|
||||
@@ -191,7 +199,7 @@ async function inspectWithContext(params: {
|
||||
if (result.termination === "exit" && isAuthoritativeLeaseAbsence(result, params.id)) {
|
||||
return { status: "unknown" };
|
||||
}
|
||||
throw crabboxCommandError("inspect", result);
|
||||
throw crabboxCommandError(action, result);
|
||||
}
|
||||
|
||||
function remainingProvisionTimeout(deadline: number, maximum: number): number {
|
||||
@@ -254,7 +262,11 @@ async function waitForProvisionReady(
|
||||
expectedLeaseId: inspect.id,
|
||||
id: inspect.id,
|
||||
runCommand: params.runCommand,
|
||||
timeoutMs: remainingProvisionTimeout(params.deadline, CRABBOX_LIFECYCLE_TIMEOUT_MS),
|
||||
timeoutMs: remainingProvisionTimeout(
|
||||
params.deadline,
|
||||
resolveCrabboxLifecycleTimeoutMs(params.provider),
|
||||
),
|
||||
waitForReady: params.provider === "machine0",
|
||||
});
|
||||
if (replay.status === "unknown") {
|
||||
throw new Error("Crabbox operation lease disappeared while waiting for SSH readiness");
|
||||
@@ -267,7 +279,7 @@ async function waitForProvisionReady(
|
||||
assertProvisionSecurityPolicy({ inspect, provider: params.provider });
|
||||
while (inspect.ready !== true && !isUnusableProvisionState(inspect.state)) {
|
||||
const remaining = remainingProvisionTimeout(params.deadline, CRABBOX_LIFECYCLE_TIMEOUT_MS);
|
||||
await params.sleep(Math.min(READY_POLL_INTERVAL_MS, remaining));
|
||||
await params.sleep(Math.min(resolveCrabboxReadyPollIntervalMs(params.provider), remaining));
|
||||
inspect = await inspectAgain();
|
||||
assertProvisionSecurityPolicy({ inspect, provider: params.provider });
|
||||
}
|
||||
@@ -293,6 +305,7 @@ async function runProvisionSetup(
|
||||
setup: string;
|
||||
timeoutMs?: number;
|
||||
forwardedEnv?: Record<string, string>;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
},
|
||||
): Promise<void> {
|
||||
let result: SpawnResult;
|
||||
@@ -301,7 +314,7 @@ async function runProvisionSetup(
|
||||
action: "setup",
|
||||
args: crabboxLeaseRunArgs({ ...params, id: params.inspect.id }, params.forwardedEnv),
|
||||
binary: params.binary,
|
||||
env: params.forwardedEnv,
|
||||
env: params.env ?? params.forwardedEnv,
|
||||
input: params.setup,
|
||||
runCommand: params.runCommand,
|
||||
timeoutMs: remainingProvisionTimeout(
|
||||
@@ -320,10 +333,7 @@ async function runProvisionSetup(
|
||||
}
|
||||
|
||||
async function runProvisionSetupAndWaitReady(
|
||||
params: ProvisionInspectContext & {
|
||||
setup: string;
|
||||
timeoutMs?: number;
|
||||
forwardedEnv?: Record<string, string>;
|
||||
params: Parameters<typeof runProvisionSetup>[0] & {
|
||||
sleep: (milliseconds: number) => Promise<void>;
|
||||
},
|
||||
): Promise<ParsedInspect> {
|
||||
@@ -345,7 +355,7 @@ async function stopProvisionId(params: {
|
||||
provider: params.provider,
|
||||
runCommand: params.runCommand,
|
||||
// Cleanup gets its own budget so an exhausted provision deadline cannot leak a lease.
|
||||
timeoutMs: CRABBOX_LIFECYCLE_TIMEOUT_MS,
|
||||
timeoutMs: resolveCrabboxLifecycleTimeoutMs(params.provider),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -504,14 +514,10 @@ export function createCrabboxWorkerProvider(
|
||||
) {
|
||||
throw new WorkerProviderError("Crabbox execution mode is unsupported");
|
||||
}
|
||||
const configured = parseCrabboxProfile(profile);
|
||||
const requestedClass = nonEmptyString(options?.machineClass);
|
||||
if (options?.machineClass !== undefined && (!requestedClass || requestedClass.length > 128)) {
|
||||
throw new WorkerProviderError(
|
||||
"Crabbox machine class must be a non-empty string of at most 128 characters",
|
||||
);
|
||||
}
|
||||
const parsed = requestedClass ? { ...configured, class: requestedClass } : configured;
|
||||
const { profile: parsed, forwardedEnv } = resolveCrabboxProvisionProfile(
|
||||
profile,
|
||||
options?.machineClass,
|
||||
);
|
||||
const warmupTimeoutMs = parsed.desktop
|
||||
? CRABBOX_DESKTOP_WARMUP_TIMEOUT_MS
|
||||
: CRABBOX_WARMUP_TIMEOUT_MS;
|
||||
@@ -571,7 +577,11 @@ export function createCrabboxWorkerProvider(
|
||||
expectedLeaseId: leaseId,
|
||||
id: leaseId,
|
||||
runCommand,
|
||||
timeoutMs: remainingProvisionTimeout(deadline, CRABBOX_LIFECYCLE_TIMEOUT_MS),
|
||||
timeoutMs: remainingProvisionTimeout(
|
||||
deadline,
|
||||
resolveCrabboxLifecycleTimeoutMs(parsed.provider),
|
||||
),
|
||||
waitForReady: parsed.provider === "machine0",
|
||||
});
|
||||
} catch (error) {
|
||||
// Transport failure after warmup is indeterminate; preserve the lease for durable replay.
|
||||
@@ -606,6 +616,8 @@ export function createCrabboxWorkerProvider(
|
||||
inspectedParams.inspect = await runProvisionSetupAndWaitReady({
|
||||
...inspectedParams,
|
||||
setup: parsed.setup,
|
||||
forwardedEnv,
|
||||
env: { ...forwardedEnv, CRABBOX_ENV_ALLOW: parsed.setupEnv?.join(",") || "," },
|
||||
sleep,
|
||||
});
|
||||
}
|
||||
@@ -707,7 +719,11 @@ export function createCrabboxWorkerProvider(
|
||||
const context = resolveLeaseContext(lease);
|
||||
// Fence the provider keepalive before teardown so an in-flight touch cannot reschedule.
|
||||
heartbeats.stop(context.id);
|
||||
await stopCrabboxLease({ ...context, runCommand });
|
||||
await stopCrabboxLease({
|
||||
...context,
|
||||
runCommand,
|
||||
timeoutMs: resolveCrabboxLifecycleTimeoutMs(context.provider),
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
type CrabboxProvisionTimeoutProfile = {
|
||||
provider: string;
|
||||
desktop?: boolean;
|
||||
setup?: string;
|
||||
};
|
||||
@@ -23,18 +24,36 @@ export const CRABBOX_HEARTBEAT_TIMEOUT_MS = 150_000;
|
||||
// lifecycle budget — a hung binary must fall back to label-only choices
|
||||
// promptly instead of stalling the whole cloud picker.
|
||||
export const CRABBOX_MACHINE_CATALOG_TIMEOUT_MS = 5_000;
|
||||
// Fixed-lease inspection can follow warmup's final read; allow four one-minute retries.
|
||||
const CRABBOX_MACHINE0_LIFECYCLE_TIMEOUT_MS = 5 * 60_000;
|
||||
// Setup gets its own budget on top of provision so a slow warmup cannot starve it.
|
||||
// Setup may install an exact candidate CLI and official plugins on a minimal cloud image.
|
||||
export const CRABBOX_SETUP_TIMEOUT_MS = 15 * 60_000;
|
||||
export const CRABBOX_NODE_ENROLLMENT_TIMEOUT_MS = 15 * 60_000;
|
||||
|
||||
// Leave one minute inside the lifecycle cap for process startup and cleanup handoff.
|
||||
export const CRABBOX_MACHINE0_READY_WAIT_TIMEOUT = "4m";
|
||||
|
||||
// Match Machine0's provider-read cadence; fast re-inspection can exhaust its hourly API budget.
|
||||
export function resolveCrabboxReadyPollIntervalMs(provider: string): number {
|
||||
return provider === "machine0" ? 60_000 : 2_000;
|
||||
}
|
||||
|
||||
export function resolveCrabboxLifecycleTimeoutMs(provider: string): number {
|
||||
return provider === "machine0"
|
||||
? CRABBOX_MACHINE0_LIFECYCLE_TIMEOUT_MS
|
||||
: CRABBOX_LIFECYCLE_TIMEOUT_MS;
|
||||
}
|
||||
|
||||
export function resolveCrabboxProvisionBaseTimeoutMs(
|
||||
profile: CrabboxProvisionTimeoutProfile,
|
||||
): number {
|
||||
const warmupTimeoutMs = profile.desktop
|
||||
? CRABBOX_DESKTOP_WARMUP_TIMEOUT_MS
|
||||
: CRABBOX_WARMUP_TIMEOUT_MS;
|
||||
return warmupTimeoutMs + CRABBOX_LIFECYCLE_TIMEOUT_MS;
|
||||
const lifecycleTimeoutMs = resolveCrabboxLifecycleTimeoutMs(profile.provider);
|
||||
// Machine0 needs separate windows for authoritative inspection and readiness retry.
|
||||
return warmupTimeoutMs + lifecycleTimeoutMs * (profile.provider === "machine0" ? 2 : 1);
|
||||
}
|
||||
|
||||
export function countCrabboxProvisionSetupPhases(profile: CrabboxProvisionTimeoutProfile): number {
|
||||
@@ -48,6 +67,6 @@ export function resolveCrabboxProvisionCallTimeoutMs(
|
||||
resolveCrabboxProvisionBaseTimeoutMs(profile) +
|
||||
countCrabboxProvisionSetupPhases(profile) * CRABBOX_SETUP_TIMEOUT_MS +
|
||||
CRABBOX_NODE_ENROLLMENT_TIMEOUT_MS +
|
||||
CRABBOX_LIFECYCLE_TIMEOUT_MS
|
||||
resolveCrabboxLifecycleTimeoutMs(profile.provider)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -361,11 +361,9 @@ describe("worker environment service provision replay", () => {
|
||||
expect(destroy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("preserves an allocated lease after indeterminate provision cleanup across restart", async () => {
|
||||
it.each([true, false])("recovers indeterminate cleanup (released: %s)", async (released) => {
|
||||
const leaseId = "lease:worker-provision-cleanup";
|
||||
let releaseCommitted = false;
|
||||
const provision = vi.fn(async () => {
|
||||
releaseCommitted = true;
|
||||
throw WorkerProviderError.cleanupIndeterminate(
|
||||
leaseId,
|
||||
new Error("worker enrollment failed"),
|
||||
@@ -373,7 +371,7 @@ describe("worker environment service provision replay", () => {
|
||||
);
|
||||
});
|
||||
const inspect = vi.fn(async () => ({
|
||||
status: releaseCommitted ? ("destroyed" as const) : ("active" as const),
|
||||
status: released ? ("destroyed" as const) : ("active" as const),
|
||||
}));
|
||||
const destroy = vi.fn(async () => {});
|
||||
const provider = support.createProvider({ provision, inspect, destroy });
|
||||
@@ -416,7 +414,10 @@ describe("worker environment service provision replay", () => {
|
||||
|
||||
expect(provision).toHaveBeenCalledTimes(1);
|
||||
expect(inspect).toHaveBeenCalledWith({ leaseId, profile: { region: "test" } });
|
||||
expect(destroy).not.toHaveBeenCalled();
|
||||
expect(destroy).toHaveBeenCalledTimes(released ? 0 : 1);
|
||||
if (!released) {
|
||||
expect(destroy).toHaveBeenCalledWith({ leaseId, profile: { region: "test" } });
|
||||
}
|
||||
expect(support.testState.store.get(pending.environmentId)).toMatchObject({
|
||||
state: "failed",
|
||||
leaseId: null,
|
||||
|
||||
Reference in New Issue
Block a user