diff --git a/src/gateway/server-kernel.test.ts b/src/gateway/server-kernel.test.ts index 1e0c556fb33c..adace9ba16e4 100644 --- a/src/gateway/server-kernel.test.ts +++ b/src/gateway/server-kernel.test.ts @@ -18,6 +18,87 @@ import { createGatewayKernel } from "./server-kernel.js"; import { createSyntheticPluginRuntimeClient } from "./server-plugin-runtime-client.js"; describe("createGatewayKernel", () => { + it("keeps readiness red until deferred startup unlocks chat dispatch", async () => { + const port = await getFreePort(); + const state = await createOpenClawTestState({ + label: "gateway-kernel-deferred-readiness", + layout: "home", + env: { + OPENCLAW_GATEWAY_PASSWORD: undefined, + OPENCLAW_GATEWAY_TOKEN: undefined, + OPENCLAW_SKIP_BROWSER_CONTROL_SERVER: "1", + OPENCLAW_SKIP_CANVAS_HOST: "1", + OPENCLAW_SKIP_CHANNELS: "1", + OPENCLAW_SKIP_CRON: "1", + OPENCLAW_SKIP_GMAIL_WATCHER: "1", + OPENCLAW_SKIP_PROVIDERS: "1", + OPENCLAW_TEST_MINIMAL_GATEWAY: "0", + VITEST: "1", + }, + }); + const token = "gateway-kernel-deferred-readiness-token"; + let kernel: Awaited> | undefined; + try { + await state.writeConfig({ + gateway: { + auth: { mode: "token", token }, + controlUi: { enabled: false }, + port, + }, + }); + state.applyEnv(); + kernel = await createGatewayKernel(port, { + auth: { mode: "token", token }, + bind: "loopback", + controlUiEnabled: false, + sidecarStartup: "defer", + }); + + const client = createSyntheticPluginRuntimeClient({ + scopes: [...CLI_DEFAULT_OPERATOR_SCOPES], + }); + const dispatchOptions = { + client, + context: kernel.gatewayRequestContext, + methodRegistry: kernel.getAttachedGatewayMethodRegistry(), + }; + const runId = "deferred-readiness-chat"; + const chatParams = { + sessionKey: "agent:main:deferred-readiness", + message: "readiness truth", + idempotencyKey: runId, + }; + + const getReadiness = kernel.createHttpTransportOptions().getReadiness; + expect(getReadiness()).toMatchObject({ + ready: false, + failing: ["startup-sidecars"], + }); + await expect( + dispatchGatewayRequestInProcess("chat.send", chatParams, dispatchOptions), + ).rejects.toThrow("chat.send unavailable during gateway startup"); + + kernel.dedupe.set(`chat:${runId}`, { + ts: Date.now(), + ok: true, + payload: { runId, status: "ok" }, + }); + kernel.kernel.unlockStartupMethods(); + kernel.kernel.markSidecarsReady(); + + expect(getReadiness()).toMatchObject({ ready: true, failing: [] }); + await expect( + dispatchGatewayRequestInProcess("chat.send", chatParams, dispatchOptions), + ).resolves.toEqual({ runId, status: "ok" }); + } finally { + try { + await kernel?.closeOnStartupFailure(); + } finally { + await state.cleanup(); + } + } + }); + it("dispatches health and an agent turn without creating a transport", async () => { const port = await getFreePort(); const state = await createOpenClawTestState({ diff --git a/src/gateway/server-runtime-state-prepare.ts b/src/gateway/server-runtime-state-prepare.ts index 2aadc9dfdab8..5bffbe94fada 100644 --- a/src/gateway/server-runtime-state-prepare.ts +++ b/src/gateway/server-runtime-state-prepare.ts @@ -399,7 +399,7 @@ export async function prepareGatewayKernelState(params: { }); channelManager.setAutostartSuppression(opts.channelAutostartSuppression ?? null); const sidecarStartup = opts.sidecarStartup ?? "start"; - const isGatewayStartupPending = () => !startupState.sidecarsReady && sidecarStartup === "start"; + const isGatewayStartupPending = () => !startupState.sidecarsReady; const startupCheckerDeps = { startedAt: serverStartedAt, getStartupPending: isGatewayStartupPending, diff --git a/src/gateway/worker-environments/worker-turn-admission.ts b/src/gateway/worker-environments/worker-turn-admission.ts index b20411117357..777d47303325 100644 --- a/src/gateway/worker-environments/worker-turn-admission.ts +++ b/src/gateway/worker-environments/worker-turn-admission.ts @@ -105,12 +105,16 @@ export function resolvePlacementIdentity( export function requireActivePlacement( placement: WorkerSessionPlacementRecord, ): ActiveWorkerPlacement { + const failureDetail = + placement.state === "failed" + ? `: ${placement.terminalReason ?? placement.recoveryError}; redispatch the session so its worker can bootstrap the current build before retrying.` + : ""; if ( placement.state !== "active" || !placement.remoteWorkspaceDir || !placement.workerBundleHash ) { - throw new Error(`Worker turn rejected in placement ${placement.state}`); + throw new Error(`Worker turn rejected in placement ${placement.state}${failureDetail}`); } return placement; } diff --git a/src/gateway/worker-environments/worker-turn-launcher-reclaimed-placement.test.ts b/src/gateway/worker-environments/worker-turn-launcher-reclaimed-placement.test.ts index 419ba5fab4f2..27f45835e469 100644 --- a/src/gateway/worker-environments/worker-turn-launcher-reclaimed-placement.test.ts +++ b/src/gateway/worker-environments/worker-turn-launcher-reclaimed-placement.test.ts @@ -466,4 +466,38 @@ describe("worker turn launcher reclaimed placement", () => { expect(runLocal).not.toHaveBeenCalled(); expect(placements.get(SESSION_ID)?.turnClaim).toBeNull(); }); + + it("projects a failed placement cause with current-build recovery guidance", async () => { + placements.startDispatch({ + sessionId: SESSION_ID, + sessionKey: SESSION_KEY, + agentId: "main", + }); + placements.fail({ + sessionId: SESSION_ID, + recoveryError: "cloud worker disappeared: environment state destroyed", + }); + const provider = createWorkerSessionTurnPlacementProvider({ + environments: unusedEnvironments(), + placements, + }); + const runLocal = vi.fn(async () => ({ meta: { durationMs: 1 } })); + + await expect( + provider.executeTurn( + { + sessionId: SESSION_ID, + sessionKey: SESSION_KEY, + agentId: "main", + runId: "run-failed", + }, + turn("run-failed"), + runLocal, + ), + ).rejects.toThrow( + "Worker turn rejected in placement failed: cloud worker disappeared: environment state destroyed; redispatch the session so its worker can bootstrap the current build before retrying.", + ); + expect(runLocal).not.toHaveBeenCalled(); + expect(placements.get(SESSION_ID)?.turnClaim).toBeNull(); + }); });