From 7d95cff39d61a583e976b9d2492462bcbb48aab0 Mon Sep 17 00:00:00 2001 From: ClawSweeper Date: Sat, 22 Aug 2026 15:51:04 -0700 Subject: [PATCH] fix: macOS onboarding waits for Gateway restart (#127713) * fix(onboarding): wait for inference gateway restart Co-authored-by: hannesrudolph <49103247+hannesrudolph@users.noreply.github.com> * fix(onboarding): preserve custodian handoff after restart Co-authored-by: hannesrudolph <49103247+hannesrudolph@users.noreply.github.com> * refactor(macos): share activation restart finalization Co-authored-by: hannesrudolph <49103247+hannesrudolph@users.noreply.github.com> * style(macos): format restart reconciliation Co-authored-by: hannesrudolph <49103247+hannesrudolph@users.noreply.github.com> * fix(macos): compile restart finalization Co-authored-by: hannesrudolph <49103247+hannesrudolph@users.noreply.github.com> * test(macos): sequence onboarding restart proof Co-authored-by: hannesrudolph <49103247+hannesrudolph@users.noreply.github.com> * test(macos): finish onboarding after activation Co-authored-by: Tak Hoffman <781889+Takhoffman@users.noreply.github.com> Co-authored-by: hannesrudolph <49103247+hannesrudolph@users.noreply.github.com> * test(macos): reuse managed restart proof Co-authored-by: Tak Hoffman <781889+Takhoffman@users.noreply.github.com> Co-authored-by: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> * test(macos): assert receipt before handoff cleanup Co-authored-by: Tak Hoffman <781889+Takhoffman@users.noreply.github.com> Co-authored-by: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> * fix(onboarding): keep restart verification bounded Co-authored-by: Tak Hoffman <781889+Takhoffman@users.noreply.github.com> Co-authored-by: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> --------- Co-authored-by: RoboClaw <309084314+roboclaw-bot@users.noreply.github.com> Co-authored-by: hannesrudolph <49103247+hannesrudolph@users.noreply.github.com> --- .../Sources/OpenClaw/OnboardingAISetup.swift | 155 +++++++++--------- .../OpenClaw/OnboardingAISetupSupport.swift | 31 ++++ .../OnboardingAISetupTests.swift | 52 ++++-- .../OpenClawProtocol/GatewayModels.swift | 4 + .../gateway-protocol/src/schema/openclaw.ts | 2 + src/commands/onboard-remote-gateway.test.ts | 125 ++++++++++++++ src/commands/onboard-remote-gateway.ts | 60 ++++++- .../setup-inference-activate-persist.ts | 8 +- src/system-agent/setup-inference-activate.ts | 12 +- src/system-agent/setup-inference-core.ts | 8 +- src/system-agent/setup-inference.test.ts | 31 +++- 11 files changed, 378 insertions(+), 110 deletions(-) diff --git a/apps/macos/Sources/OpenClaw/OnboardingAISetup.swift b/apps/macos/Sources/OpenClaw/OnboardingAISetup.swift index 5e7c195f6c6d..8efd51d50498 100644 --- a/apps/macos/Sources/OpenClaw/OnboardingAISetup.swift +++ b/apps/macos/Sources/OpenClaw/OnboardingAISetup.swift @@ -905,28 +905,27 @@ extension OnboardingAISetupModel { context: context) let result = try JSONDecoder().decode(ActivateResult.self, from: data) guard self.isCurrentAttempt(context), !Task.isCancelled else { return } - guard await self.gateway.isCurrentServerLease(lease) else { - if result.ok, - OnboardingSystemAgentResumeStore.markCompleted( - ifOwnedBy: context.routeIdentity, - activationOwner: activationOwner, - defaults: self.defaults) + if result.ok { + if let failure = await finishSuccessfulActivation( + kind: kind, + expectedModel: modelRef, + context: context, + activationOwner: activationOwner, + before: persistedStateBeforeActivation, + originalServerLease: lease, + gatewayRestartRequired: result.gatewayRestartRequired == true) { - self.pendingActivationVerification = true - self.phase = .detecting - _ = await self.verifyPendingConfiguredInference() - } else { + self.statuses[kind] = .failed(failure) + self.exposeActivationFailure(failure, whenTerminal: !tryNextCandidateOnFailure) + } + } else { + guard await self.gateway.isCurrentServerLease(lease) else { self.pendingActivationVerification = false self.clearPendingHandoff(ifOwnedBy: context, activationOwner: activationOwner) requireFreshDetection(after: Self.transportFailure( "The Gateway connection changed while AI setup was finishing. Check again.")) + return } - return - } - guard self.isCurrentAttempt(context), !Task.isCancelled else { return } - if result.ok { - finishConnected(kind: kind, activationOwner: activationOwner) - } else { self.pendingActivationVerification = false self.clearPendingHandoff(ifOwnedBy: context, activationOwner: activationOwner) let failure = Self.failure( @@ -1034,17 +1033,20 @@ extension OnboardingAISetupModel { guard self.isCurrentAttempt(context), !Task.isCancelled else { return false } let leaseTimeoutMs = deadline.remainingMilliseconds(cappedAt: 3000) guard leaseTimeoutMs > 0 else { return false } - if let replacementLease = try? await gateway.acquireServerLease( - ifSameRouteAs: originalServerLease, - timeoutMs: Double(leaseTimeoutMs)), - await reconcilePersistedActivation( - kind: kind, - expectedModel: expectedModel, - context: context, - activationOwner: activationOwner, - before: before, - serverLease: replacementLease, - deadline: deadline) + // A successful activation reply can precede its deferred restart. + // Never verify or hand off on the physical socket that scheduled it. + if await !(self.gateway.isCurrentServerLease(originalServerLease)), + let replacementLease = try? await self.gateway.acquireServerLease( + ifSameRouteAs: originalServerLease, + timeoutMs: Double(leaseTimeoutMs)), + await self.reconcilePersistedActivation( + kind: kind, + expectedModel: expectedModel, + context: context, + activationOwner: activationOwner, + before: before, + serverLease: replacementLease, + deadline: deadline) { self.serverLease = replacementLease return true @@ -1061,6 +1063,40 @@ extension OnboardingAISetupModel { return false } + private func finishSuccessfulActivation( + kind: String, + expectedModel: String, + context: AttemptContext, + activationOwner: OnboardingSystemAgentResumeStore.ActivationOwner, + before: PersistedActivationState?, + originalServerLease: GatewayConnection.ServerLease, + gatewayRestartRequired: Bool) async -> Failure? + { + let originalLeaseWasReplaced = await !(self.gateway.isCurrentServerLease(originalServerLease)) + let restartRequired = gatewayRestartRequired || originalLeaseWasReplaced + guard self.isCurrentAttempt(context), !Task.isCancelled else { return nil } + guard restartRequired else { + self.finishConnected(kind: kind, activationOwner: activationOwner) + return nil + } + self.pendingActivationVerification = true + self.phase = .detecting + if await self.reconcileActivationAfterGatewayRestart( + kind: kind, + expectedModel: expectedModel, + context: context, + activationOwner: activationOwner, + before: before, + originalServerLease: originalServerLease) + { + return nil + } + guard self.isCurrentAttempt(context), !Task.isCancelled else { return nil } + self.phase = .ready + return Self.transportFailure( + "The Gateway did not finish restarting after AI setup. Try again once it is available.") + } + private func reconcilePersistedActivation( kind: String, expectedModel: String, @@ -1119,26 +1155,7 @@ extension OnboardingAISetupModel { } extension OnboardingAISetupModel { - func startProviderAuth(_ option: AuthOption) { - self.startProviderWizard(option, kind: .auth) - } - - func startProviderPrepare(_ option: PrepareOption) { - self.startProviderWizard( - AuthOption( - id: option.id, - brandId: option.brandId, - label: option.label, - hint: option.hint, - groupLabel: nil, - icon: option.icon, - website: option.website, - kind: "prepare", - featured: false), - kind: .prepare) - } - - private func startProviderWizard(_ option: AuthOption, kind: ProviderWizardKind) { + func startProviderWizard(_ option: AuthOption, kind: ProviderWizardKind) { guard !self.isBusy, self.activeAuthOption == nil, let serverLease else { return } self.activeAuthOption = option self.providerWizardKind = kind @@ -1258,16 +1275,6 @@ extension OnboardingAISetupModel { } } - var authWizardOptions: [WizardOption] { - parseWizardOptions(self.authStep?.options) - } - - var selectedAuthWizardOption: WizardOption? { - let options = self.authWizardOptions - guard options.indices.contains(self.authSelection) else { return options.first } - return options[self.authSelection] - } - private func advanceProviderAuth(stepID: String?, value: AnyCodable?) { guard let sessionID = authSessionID, let serverLease else { return } self.authBusy = true @@ -1510,6 +1517,7 @@ extension OnboardingAISetupModel { ifCurrentServerLease: lease) guard self.isCurrentAttempt(context), !Task.isCancelled else { return } let requestTimeoutMs = Self.activationRequestTimeoutMs(for: "api-key") + let persistedStateBeforeActivation = self.lastDetectedActivationState // Same keychain-unavailable degradation as detected candidates: an // unbound lease keeps the ambiguity window without a resume receipt. let activationOwner = routeFingerprint.map { fingerprint in @@ -1547,31 +1555,24 @@ extension OnboardingAISetupModel { ifCurrentServerLease: lease) let result = try JSONDecoder().decode(ActivateResult.self, from: data) guard self.isCurrentAttempt(context), !Task.isCancelled else { return } - guard await self.gateway.isCurrentServerLease(lease) else { - if result.ok, - OnboardingSystemAgentResumeStore.markCompleted( - ifOwnedBy: context.routeIdentity, - activationOwner: activationOwner, - defaults: self.defaults) - { - self.pendingActivationVerification = true - self.phase = .detecting - _ = await self.verifyPendingConfiguredInference() - } else { + if result.ok { + self.manualKey = "" + self.manualError = await self.finishSuccessfulActivation( + kind: "api-key", + expectedModel: result.modelRef ?? "", + context: context, + activationOwner: activationOwner, + before: persistedStateBeforeActivation, + originalServerLease: lease, + gatewayRestartRequired: result.gatewayRestartRequired == true) + } else { + guard await self.gateway.isCurrentServerLease(lease) else { self.pendingActivationVerification = false self.clearPendingHandoff(ifOwnedBy: context, activationOwner: activationOwner) self.requireFreshDetection(after: Self.transportFailure( "The Gateway connection changed while AI setup was finishing. Check again.")) + return } - return - } - guard self.isCurrentAttempt(context), !Task.isCancelled else { return } - if result.ok { - self.manualKey = "" - self.finishConnected( - kind: "api-key", - activationOwner: activationOwner) - } else { self.pendingActivationVerification = false self.clearPendingHandoff(ifOwnedBy: context, activationOwner: activationOwner) self.manualError = Self.failure( diff --git a/apps/macos/Sources/OpenClaw/OnboardingAISetupSupport.swift b/apps/macos/Sources/OpenClaw/OnboardingAISetupSupport.swift index ae61a1d80f39..a4017bfbeb0f 100644 --- a/apps/macos/Sources/OpenClaw/OnboardingAISetupSupport.swift +++ b/apps/macos/Sources/OpenClaw/OnboardingAISetupSupport.swift @@ -1,6 +1,7 @@ import Foundation import OpenClawChatUI import OpenClawKit +import OpenClawProtocol extension OnboardingAISetupModel { struct PersistedActivationState: Equatable { @@ -81,6 +82,7 @@ extension OnboardingAISetupModel { let modelRef: String? let status: String? let error: String? + let gatewayRestartRequired: Bool? } struct Candidate: Identifiable, Equatable { @@ -205,6 +207,16 @@ extension OnboardingAISetupModel { self.providerWizardKind == .prepare } + var authWizardOptions: [WizardOption] { + parseWizardOptions(self.authStep?.options) + } + + var selectedAuthWizardOption: WizardOption? { + let options = self.authWizardOptions + guard options.indices.contains(self.authSelection) else { return options.first } + return options[self.authSelection] + } + var connected: Bool { self.phase == .connected } @@ -219,6 +231,25 @@ extension OnboardingAISetupModel { return !self.isBusy || (self.phase == .testing && self.selectedKind != kind) } + func startProviderAuth(_ option: AuthOption) { + self.startProviderWizard(option, kind: .auth) + } + + func startProviderPrepare(_ option: PrepareOption) { + self.startProviderWizard( + AuthOption( + id: option.id, + brandId: option.brandId, + label: option.label, + hint: option.hint, + groupLabel: nil, + icon: option.icon, + website: option.website, + kind: "prepare", + featured: false), + kind: .prepare) + } + /// True when setup live-verified an already-configured route instead of /// activating a new one. The custodian first-run handoff belongs only to /// fresh activations; verified reopens land on the normal dashboard. diff --git a/apps/macos/Tests/OpenClawIPCTests/OnboardingAISetupTests.swift b/apps/macos/Tests/OpenClawIPCTests/OnboardingAISetupTests.swift index 6b0d91b69afe..085e9c234e00 100644 --- a/apps/macos/Tests/OpenClawIPCTests/OnboardingAISetupTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/OnboardingAISetupTests.swift @@ -660,7 +660,8 @@ private func makeRestartingAISetupSession( suiteName: String, recorder: AISetupRequestRecorder, ownerObservation: ActivationOwnerObservation, - postRestartConfiguredModel: String?) -> GatewayTestWebSocketSession + postRestartConfiguredModel: String?, + replacementGate: AISetupRequestGate? = nil) -> GatewayTestWebSocketSession { let socketGeneration = AISetupSocketGeneration() return GatewayTestWebSocketSession(taskFactory: { @@ -693,6 +694,9 @@ private func makeRestartingAISetupSession( } switch request.method { case "openclaw.setup.detect": + if let replacementGate { + await replacementGate.wait() + } let response = postRestartConfiguredModel.map { persistedDetectedSetupResponse(id: request.id, configuredModel: $0) } ?? detectedSetupResponse( @@ -713,11 +717,17 @@ private func failedActivationResponse(id: String) -> Data { Data(#"{"type":"res","id":"\#(id)","ok":true,"payload":{"ok":false,"status":"auth","error":"rejected"}}"#.utf8) } -private func successfulActivationResponse(id: String, modelRef: String, latencyMs: Int) -> Data { - Data( +private func successfulActivationResponse( + id: String, + modelRef: String, + latencyMs: Int, + gatewayRestartRequired: Bool = false) -> Data +{ + let restartField = gatewayRestartRequired ? ",\"gatewayRestartRequired\":true" : "" + return Data( """ {"type":"res","id":"\(id)","ok":true,"payload":{ - "ok":true,"modelRef":"\(modelRef)","latencyMs":\(latencyMs),"lines":["Model ready"]}} + "ok":true,"modelRef":"\(modelRef)","latencyMs":\(latencyMs),"lines":["Model ready"]\(restartField)}} """.utf8) } @@ -1282,19 +1292,34 @@ struct OnboardingAISetupTests { let defaults = try #require(isolatedAISetupDefaults(suiteName: suiteName)) let recorder = AISetupRequestRecorder() let ownerObservation = ActivationOwnerObservation() + let replacementGate = AISetupRequestGate() let session = makeRestartingAISetupSession( suiteName: suiteName, recorder: recorder, ownerObservation: ownerObservation, - postRestartConfiguredModel: "openai/gpt-5.5") + postRestartConfiguredModel: "openai/gpt-5.5", + replacementGate: replacementGate) let url = try #require(URL(string: "ws://example.invalid")) let gateway = makeAISetupGateway(url: url, token: "route-token", session: session) - let model = makeAISetupModel(gateway: gateway, defaults: defaults) - var handoffCount = 0 - model.onConnected = { handoffCount += 1 } + let appState = AppState(preview: true) + appState.connectionMode = .local + var handoffs: [OnboardingDashboardHandoff] = [] + let view = OnboardingView( + state: appState, + aiSetupGateway: gateway, + systemAgentDefaults: defaults, + aiSetupRouteIdentityProvider: { "local" }, + dashboardHandoffOpener: { handoffs.append($0) }) - await model.detectAndAutoConnect() - await model.activate(kind: "codex-cli") + await view.aiSetup.detectAndAutoConnect() + let activation = Task { await view.aiSetup.activate(kind: "codex-cli") } + await replacementGate.waitUntilStarted() + guard case .activating = pendingState(defaults) else { + Issue.record("expected restart-required activation to remain pending") + return + } + await replacementGate.release() + await activation.value let activationOwner = try #require(ownerObservation.value()) #expect(session.snapshotMakeCount() >= 2) @@ -1304,11 +1329,12 @@ struct OnboardingAISetupTests { "openclaw.setup.detect", "openclaw.setup.verify", ]) - #expect(model.connected) - #expect(model.selectedKind == "codex-cli") - #expect(handoffCount == 1) + #expect(view.aiSetup.connected) + #expect(view.aiSetup.selectedKind == "codex-cli") #expect(storedActivationOwner(defaults) == activationOwner) #expect(pendingState(defaults) == .completed) + #expect(view.finish()) + #expect(handoffs == [.custodianOnboarding]) } @Test func `managed Gateway restart rejects mismatched persisted transition`() async throws { diff --git a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift index 0d7965f40083..88a07c95762e 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift @@ -12396,6 +12396,7 @@ public struct SystemAgentSetupActivateResult: Codable, Sendable { public let modelref: String? public let latencyms: Double? public let lines: [String]? + public let gatewayrestartrequired: Bool? public let status: AnyCodable? public let error: String? @@ -12404,6 +12405,7 @@ public struct SystemAgentSetupActivateResult: Codable, Sendable { modelref: String? = nil, latencyms: Double? = nil, lines: [String]? = nil, + gatewayrestartrequired: Bool? = nil, status: AnyCodable? = nil, error: String? = nil) { @@ -12411,6 +12413,7 @@ public struct SystemAgentSetupActivateResult: Codable, Sendable { self.modelref = modelref self.latencyms = latencyms self.lines = lines + self.gatewayrestartrequired = gatewayrestartrequired self.status = status self.error = error } @@ -12420,6 +12423,7 @@ public struct SystemAgentSetupActivateResult: Codable, Sendable { case modelref = "modelRef" case latencyms = "latencyMs" case lines + case gatewayrestartrequired = "gatewayRestartRequired" case status case error } diff --git a/packages/gateway-protocol/src/schema/openclaw.ts b/packages/gateway-protocol/src/schema/openclaw.ts index 969181f413c6..5dcab163157a 100644 --- a/packages/gateway-protocol/src/schema/openclaw.ts +++ b/packages/gateway-protocol/src/schema/openclaw.ts @@ -357,6 +357,8 @@ export const SystemAgentSetupActivateResultSchema = closedObject({ latencyMs: Type.Optional(Type.Number()), /** Human-readable setup summary lines (workspace, model, gateway). */ lines: Type.Optional(Type.Array(Type.String())), + /** The committed plugin source requires clients to reconnect before continuing. */ + gatewayRestartRequired: Type.Optional(Type.Literal(true)), /** Present on failure: coarse bucket for client copy + docs links. */ status: Type.Optional(SetupInferenceStatus), error: Type.Optional(Type.String()), diff --git a/src/commands/onboard-remote-gateway.test.ts b/src/commands/onboard-remote-gateway.test.ts index e1513bc0bf9f..1b8487099939 100644 --- a/src/commands/onboard-remote-gateway.test.ts +++ b/src/commands/onboard-remote-gateway.test.ts @@ -238,6 +238,131 @@ describe("runRemoteGatewayInferenceOnboarding", () => { }, ); + it.each([ + { + label: "request rejection", + firstVerification: () => + Promise.reject( + Object.assign(new Error("gateway restarting"), { + name: "GatewayClientRequestError", + gatewayCode: "UNAVAILABLE", + retryable: true, + retryAfterMs: 0, + }), + ), + }, + { + label: "typed unavailable result", + firstVerification: () => + Promise.resolve({ ok: false, status: "unavailable", error: "gateway restarting" }), + }, + ])("waits for a declared Gateway restart after $label", async ({ firstVerification }) => { + const methods: string[] = []; + let verifyAttempts = 0; + const callGatewayMock = vi.fn(async (options: CallGatewayCliOptions): Promise => { + methods.push(options.method); + if (options.method === "openclaw.setup.detect") { + return detectResult(); + } + if (options.method === "openclaw.setup.activate") { + return { + ok: true, + modelRef: "openai/gpt-5.5", + latencyMs: 250, + lines: ["Default model: openai/gpt-5.5"], + gatewayRestartRequired: true, + }; + } + if (options.method === "openclaw.setup.verify" && verifyAttempts++ === 0) { + return await firstVerification(); + } + if (options.method === "openclaw.setup.verify") { + return { ok: true, modelRef: "openai/gpt-5.5", latencyMs: 100 }; + } + throw new Error(`unexpected Gateway method ${options.method}`); + }); + const runGuidedOnboarding: RunGuidedOnboarding = async (_opts, runtime, deps) => { + const detection = await deps?.detect?.(); + const candidate = detection?.candidates.find((entry) => entry.kind === "codex-cli"); + if (!candidate) { + throw new Error("Codex candidate missing"); + } + const activation = await deps?.activate?.({ + kind: "codex-cli", + modelRef: candidate.modelRef, + surface: "cli", + runtime, + }); + expect(activation).toMatchObject({ ok: true, gatewayRestartRequired: true }); + }; + + await runRemoteGatewayInferenceOnboarding( + makeTarget(makeLocalConfig(), { token: "selected-token" }), + makeRuntime(), + { + callGateway: asGatewayCall(callGatewayMock), + runGuidedOnboarding, + }, + ); + + expect(methods).toEqual([ + "openclaw.setup.detect", + "openclaw.setup.activate", + "openclaw.setup.verify", + "openclaw.setup.verify", + ]); + }); + + it("bounds a late restart verification call by the remaining deadline", async () => { + const now = vi.spyOn(Date, "now").mockReturnValue(45_500).mockReturnValueOnce(1_000); + const callGatewayMock = vi.fn(async (options: CallGatewayCliOptions): Promise => { + if (options.method === "openclaw.setup.detect") { + return detectResult(); + } + if (options.method === "openclaw.setup.activate") { + return { + ok: true, + modelRef: "openai/gpt-5.5", + latencyMs: 250, + lines: ["Default model: openai/gpt-5.5"], + gatewayRestartRequired: true, + }; + } + if (options.method === "openclaw.setup.verify") { + return { ok: true, modelRef: "openai/gpt-5.5", latencyMs: 100 }; + } + throw new Error(`unexpected Gateway method ${options.method}`); + }); + const runGuidedOnboarding: RunGuidedOnboarding = async (_opts, runtime, deps) => { + await deps?.detect?.(); + await deps?.activate?.({ + kind: "codex-cli", + modelRef: "openai/gpt-5.5", + surface: "cli", + runtime, + }); + }; + + try { + await runRemoteGatewayInferenceOnboarding( + makeTarget(makeLocalConfig(), { token: "selected-token" }), + makeRuntime(), + { + callGateway: asGatewayCall(callGatewayMock), + runGuidedOnboarding, + }, + ); + } finally { + now.mockRestore(); + } + + expect( + callGatewayMock.mock.calls.find( + ([options]) => options.method === "openclaw.setup.verify", + )?.[0].timeoutMs, + ).toBe(500); + }); + it("hands an auth-free Gateway to the TUI as the exact bound route", async () => { const callGatewayMock = vi.fn(async (options: CallGatewayCliOptions): Promise => { if (options.method === "openclaw.setup.detect") { diff --git a/src/commands/onboard-remote-gateway.ts b/src/commands/onboard-remote-gateway.ts index 6b71eae0cd34..b600b13bd91c 100644 --- a/src/commands/onboard-remote-gateway.ts +++ b/src/commands/onboard-remote-gateway.ts @@ -1,5 +1,6 @@ // Remote-Gateway onboarding adapters keep inference detection and activation on the Gateway host. import { randomUUID } from "node:crypto"; +import { setTimeout as delay } from "node:timers/promises"; import type { SystemAgentChatResult, SystemAgentSetupActivateResult, @@ -7,7 +8,11 @@ import type { SystemAgentSetupVerifyResult, } from "../../packages/gateway-protocol/src/index.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import type { CallGatewayCliOptions } from "../gateway/call.js"; +import { + isGatewayClientRequestError, + isGatewayTransportError, + type CallGatewayCliOptions, +} from "../gateway/call.js"; import { defaultRuntime, type RuntimeEnv } from "../runtime.js"; import type { ActivateSetupInferenceParams, @@ -24,6 +29,7 @@ const GATEWAY_SETUP_ACTIVATE_TIMEOUT_MS = 150_000; const GATEWAY_CODEX_SETUP_ACTIVATE_TIMEOUT_MS = 480_000; const GATEWAY_SETUP_VERIFY_TIMEOUT_MS = 30_000; const GATEWAY_SYSTEM_AGENT_CHAT_TIMEOUT_MS = 190_000; +const GATEWAY_RESTART_WAIT_TIMEOUT_MS = 45_000; type CallGateway = (options: CallGatewayCliOptions) => Promise; @@ -137,6 +143,7 @@ function toSetupInferenceActivationResult( modelRef: result.modelRef, latencyMs: result.latencyMs, lines: result.lines, + ...(result.gatewayRestartRequired ? { gatewayRestartRequired: true } : {}), }; } if (!isSetupInferenceFailureStatus(result.status) || !result.error?.trim()) { @@ -255,11 +262,52 @@ export async function runRemoteGatewayInferenceOnboarding( if (!activation.ok) { return activation; } - const verification = await request({ - method: "openclaw.setup.verify", - payload: {}, - timeoutMs: GATEWAY_SETUP_VERIFY_TIMEOUT_MS, - }); + const restartDeadline = Date.now() + GATEWAY_RESTART_WAIT_TIMEOUT_MS; + let retryDelayMs = 250; + let verification: SystemAgentSetupVerifyResult | undefined; + for (;;) { + const remainingBeforeAttemptMs = restartDeadline - Date.now(); + if (activation.gatewayRestartRequired === true && remainingBeforeAttemptMs <= 0) { + break; + } + try { + verification = await request({ + method: "openclaw.setup.verify", + payload: {}, + timeoutMs: + activation.gatewayRestartRequired === true + ? Math.min(GATEWAY_SETUP_VERIFY_TIMEOUT_MS, remainingBeforeAttemptMs) + : GATEWAY_SETUP_VERIFY_TIMEOUT_MS, + }); + const retryableResult = + activation.gatewayRestartRequired === true && + !verification.ok && + verification.status === "unavailable"; + const remainingMs = restartDeadline - Date.now(); + if (!retryableResult || remainingMs <= 0) { + break; + } + await delay(Math.min(retryDelayMs, remainingMs)); + retryDelayMs = Math.min(retryDelayMs * 2, 2_000); + } catch (error) { + const retryable = + activation.gatewayRestartRequired === true && + (isGatewayTransportError(error) || + (isGatewayClientRequestError(error) && error.retryable)); + const remainingMs = restartDeadline - Date.now(); + if (!retryable || remainingMs <= 0) { + throw error; + } + const requestedDelay = isGatewayClientRequestError(error) + ? (error.retryAfterMs ?? retryDelayMs) + : retryDelayMs; + await delay(Math.min(requestedDelay, remainingMs)); + retryDelayMs = Math.min(retryDelayMs * 2, 2_000); + } + } + if (!verification) { + throw new Error("Gateway did not finish restarting before inference verification."); + } assertVerifiedActivation({ activation, verification, diff --git a/src/system-agent/setup-inference-activate-persist.ts b/src/system-agent/setup-inference-activate-persist.ts index 5f3aadbc3ee8..e1e8441df0ba 100644 --- a/src/system-agent/setup-inference-activate-persist.ts +++ b/src/system-agent/setup-inference-activate-persist.ts @@ -48,6 +48,7 @@ export type SetupInferenceActivationPersistenceState = { committedConfig: OpenClawConfig | undefined; autoLocalModelLeanApplied: boolean; codexInstallOwnership: "unknown" | "owned" | "unowned"; + gatewayRestartRequired: boolean; }; export async function persistActivatedSetupInference(input: { @@ -228,9 +229,8 @@ export async function persistActivatedSetupInference(input: { base: "source", // The transform stays side-effect free so a config conflict can retry // without replaying credential writes in another agent directory. - // Setup changes only hot-reloadable model, agent, and plugin-entry surfaces. - // Publish the verified route now so the next turn cannot reuse the old harness. - afterWrite: { mode: "auto" }, + // The install-record owner adds a restart follow-up when this commit adopts + // a new plugin source. Preserve that intent for structured setup clients. transform: async (current, context) => { const latestRuntime = context.snapshot.runtimeConfig ?? context.snapshot.config; // Validate that the candidate is still admissible before reporting @@ -316,6 +316,7 @@ export async function persistActivatedSetupInference(input: { }, }); committedConfig = committed.nextConfig; + state.gatewayRestartRequired = committed.followUp.requiresRestart; if (pendingCodexInstall) { codexInstallOwnership = "owned"; } @@ -368,6 +369,7 @@ export async function persistActivatedSetupInference(input: { throw error; } committedConfig = reconciledSnapshot?.sourceConfig ?? reconciledRuntime; + state.gatewayRestartRequired = pendingCodexInstall !== undefined; setupInferenceLog.warn( "Inference activation committed successfully despite a post-write cleanup error.", ); diff --git a/src/system-agent/setup-inference-activate.ts b/src/system-agent/setup-inference-activate.ts index 8b014280841e..63b429d94cd2 100644 --- a/src/system-agent/setup-inference-activate.ts +++ b/src/system-agent/setup-inference-activate.ts @@ -527,6 +527,7 @@ async function activateSetupInferenceUnredacted( } let committedConfig: OpenClawConfig | undefined; let autoLocalModelLeanApplied = false; + let gatewayRestartRequired = false; if (!needsPersistence) { const latestSnapshot = await readSnapshot(); const latestRuntime = @@ -571,6 +572,7 @@ async function activateSetupInferenceUnredacted( committedConfig, autoLocalModelLeanApplied, codexInstallOwnership, + gatewayRestartRequired, }; const persistenceFailure = await persistActivatedSetupInference({ params, @@ -597,7 +599,12 @@ async function activateSetupInferenceUnredacted( if (persistenceFailure) { return persistenceFailure; } - ({ committedConfig, autoLocalModelLeanApplied, codexInstallOwnership } = persistenceState); + ({ + committedConfig, + autoLocalModelLeanApplied, + codexInstallOwnership, + gatewayRestartRequired, + } = persistenceState); } if (codexRegistryNeedsReload && committedConfig) { const reloadedRuntimeConfig = await reloadCodexRegistryAfterActivation({ @@ -663,6 +670,9 @@ async function activateSetupInferenceUnredacted( modelRef: plan.modelRef, latencyMs: test.latencyMs, lines, + ...(params.surface === "gateway" && gatewayRestartRequired + ? { gatewayRestartRequired: true as const } + : {}), }; } finally { let codexCleanupError: SetupInferenceActivationIndeterminateError | undefined; diff --git a/src/system-agent/setup-inference-core.ts b/src/system-agent/setup-inference-core.ts index a6476b0bd5db..5044eb2b2337 100644 --- a/src/system-agent/setup-inference-core.ts +++ b/src/system-agent/setup-inference-core.ts @@ -121,7 +121,13 @@ export type SetupInferenceStatus = export type SetupInferenceFailureStatus = Exclude; export type ActivateSetupInferenceResult = - | { ok: true; modelRef: string; latencyMs: number; lines: string[] } + | { + ok: true; + modelRef: string; + latencyMs: number; + lines: string[]; + gatewayRestartRequired?: true; + } | { ok: false; status: SetupInferenceFailureStatus; error: string }; /** diff --git a/src/system-agent/setup-inference.test.ts b/src/system-agent/setup-inference.test.ts index a1086dcfda06..63a0a40f084e 100644 --- a/src/system-agent/setup-inference.test.ts +++ b/src/system-agent/setup-inference.test.ts @@ -586,7 +586,10 @@ function createConfigTransformHarness( }); state.sourceConfig = withoutPluginInstallRecords(transformed.nextConfig); state.runtimeConfig = materializeRuntimeAgentListForTest(state.sourceConfig); - return { nextConfig: state.sourceConfig }; + return { + nextConfig: state.sourceConfig, + followUp: { mode: "auto", requiresRestart: false }, + }; }); const readSnapshot = vi.fn(async () => ({ exists: true as const, @@ -1345,7 +1348,10 @@ async function runCodexSetupWithFinalConfig(params: { }); persistedConfig = withoutPluginInstallRecords(transformed.nextConfig); committed = true; - return { nextConfig: persistedConfig }; + return { + nextConfig: persistedConfig, + followUp: { mode: "auto", requiresRestart: false }, + }; }); const readConfigFileSnapshot = vi.fn(async () => { const runtimeConfig = committed ? persistedConfig : initialConfig; @@ -4185,7 +4191,10 @@ describe("activateSetupInference", () => { pendingCodexInstalls.push(transformed.plugins?.installs?.codex); persistedConfig = withoutPluginInstallRecords(transformed); activationCommitted = true; - return { nextConfig: persistedConfig }; + return { + nextConfig: persistedConfig, + followUp: { mode: "restart", reason: "plugin source changed", requiresRestart: true }, + }; }, ); const refreshPluginRegistry = vi.fn(async () => { @@ -4232,7 +4241,7 @@ describe("activateSetupInference", () => { ensurePluginRegistryLoaded: ensureRegistryLoaded, }, }); - expect(result.ok).toBe(true); + expect(result).toMatchObject({ ok: true, gatewayRestartRequired: true }); expect(runtimeLog).not.toHaveBeenCalled(); expect(ensureCodex).toHaveBeenCalledOnce(); expect(ensureCodex).toHaveBeenCalledWith( @@ -4301,9 +4310,7 @@ describe("activateSetupInference", () => { "reload-active-registry", ]); expect(transformConfig).toHaveBeenCalledWith( - expect.objectContaining({ - afterWrite: { mode: "auto" }, - }), + expect.not.objectContaining({ afterWrite: expect.anything() }), ); expect(refreshPluginRegistry).toHaveBeenCalledWith({ config: persistedConfig, @@ -4668,7 +4675,10 @@ describe("activateSetupInference", () => { pendingInstallRecords.push(pending); installIndex = { ...installIndex, ...pending }; persistedConfig = withoutPluginInstallRecords(transformed); - return { nextConfig: persistedConfig }; + return { + nextConfig: persistedConfig, + followUp: { mode: "auto", requiresRestart: false }, + }; }, ); @@ -5040,7 +5050,10 @@ describe("activateSetupInference", () => { committedInstallRecords.push(record); } currentConfig = withoutPluginInstallRecords(transformed.nextConfig); - return { nextConfig: currentConfig }; + return { + nextConfig: currentConfig, + followUp: { mode: "auto", requiresRestart: false }, + }; }, ); const readConfigFileSnapshot = vi.fn(async () => {