From 765bb37364f1372d6d458d27da8b5fc935ddaec8 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 17 Jul 2026 00:55:04 -0700 Subject: [PATCH] feat(setup): stream provider prepare progress to every surface (#109764) * feat: stream provider preparation progress * fix(android): refresh gateway method constants --- .../openclaw/app/gateway/GatewayProtocol.kt | 1 + extensions/ollama/index.ts | 1 + extensions/ollama/src/setup.test.ts | 230 +++++++++++++++++- extensions/ollama/src/setup.ts | 142 ++++++++++- .../auth-choice.apply.plugin-provider.test.ts | 9 +- src/gateway/methods/core-descriptors.ts | 1 + src/gateway/server-methods.ts | 1 + .../server-methods/system-agent.test.ts | 93 +++++++ src/gateway/server-methods/system-agent.ts | 76 ++++++ src/plugins/provider-auth-choice.ts | 27 +- src/wizard/session.test.ts | 71 ++++++ src/wizard/session.ts | 70 +++++- src/wizard/setup.shared.test.ts | 29 ++- src/wizard/setup.shared.ts | 8 +- 14 files changed, 741 insertions(+), 18 deletions(-) diff --git a/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt b/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt index 6d8df4279ede..d8671bd3bfda 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt @@ -140,6 +140,7 @@ enum class GatewayMethod( OpenclawSetupDetect("openclaw.setup.detect"), OpenclawSetupActivate("openclaw.setup.activate"), OpenclawSetupAuthStart("openclaw.setup.auth.start"), + OpenclawSetupPrepareStart("openclaw.setup.prepare.start"), WizardStart("wizard.start"), WizardNext("wizard.next"), WizardCancel("wizard.cancel"), diff --git a/extensions/ollama/index.ts b/extensions/ollama/index.ts index 6aee2289129a..10e8c681b337 100644 --- a/extensions/ollama/index.ts +++ b/extensions/ollama/index.ts @@ -730,6 +730,7 @@ export default definePluginEntry({ env: ctx.env, opts: ctx.opts as Record | undefined, prompter: ctx.prompter, + ...(ctx.signal ? { signal: ctx.signal } : {}), secretInputMode: ctx.secretInputMode, allowSecretRefPrompt: ctx.allowSecretRefPrompt, }); diff --git a/extensions/ollama/src/setup.test.ts b/extensions/ollama/src/setup.test.ts index c16a5f70c249..8259530146e2 100644 --- a/extensions/ollama/src/setup.test.ts +++ b/extensions/ollama/src/setup.test.ts @@ -42,6 +42,7 @@ vi.mock("openclaw/plugin-sdk/ssrf-runtime", async (importOriginal) => { function createOllamaFetchMock(params: { tags?: string[]; show?: Record; + capabilities?: Record; pullResponse?: Response; tagsError?: Error; meResponse?: Response; @@ -57,9 +58,15 @@ function createOllamaFetchMock(params: { if (url.endsWith("/api/show")) { const body = JSON.parse(requestBodyText(init?.body)) as { name?: string }; const contextWindow = body.name ? params.show?.[body.name] : undefined; - return contextWindow - ? jsonResponse({ model_info: { "llama.context_length": contextWindow } }) - : jsonResponse({}); + const capabilities = body.name + ? params.capabilities === undefined + ? ["tools"] + : params.capabilities[body.name] + : undefined; + return jsonResponse({ + ...(contextWindow ? { model_info: { "llama.context_length": contextWindow } } : {}), + ...(capabilities ? { capabilities } : {}), + }); } if (url.endsWith("/api/me")) { return params.meResponse ?? jsonResponse({}); @@ -79,6 +86,12 @@ function mockCallArg(mock: { mock: { calls: unknown[][] } }, index = 0, argIndex return mockCall(mock, index)?.at(argIndex); } +function abortReasonAsError(signal: AbortSignal): Error { + return signal.reason instanceof Error + ? signal.reason + : new Error("Request aborted", { cause: signal.reason }); +} + function createLocalPrompter(): WizardPrompter { return { select: vi.fn().mockResolvedValueOnce("local-only"), @@ -481,6 +494,217 @@ describe("ollama setup", () => { expect(model?.contextWindow).toBe(65536); }); + it("offers and streams a recommended pull when no installed model supports tools", async () => { + const progress = { update: vi.fn(), stop: vi.fn() }; + const prompter = { + select: vi.fn().mockResolvedValueOnce("local-only"), + text: vi.fn().mockResolvedValueOnce("http://127.0.0.1:11434"), + confirm: vi.fn().mockResolvedValueOnce(true), + progress: vi.fn(() => progress), + note: vi.fn(async () => undefined), + } as unknown as WizardPrompter; + const fetchMock = createOllamaFetchMock({ + tags: ["llama3:8b"], + show: { "gemma4:e4b": 131072 }, + capabilities: { + "llama3:8b": ["generate"], + "gemma4:e4b": ["tools"], + }, + pullResponse: new Response( + [ + '{"status":"pulling sha256:12345678","total":100,"completed":50}', + '{"status":"pulling sha256:12345678","total":100,"completed":100}', + '{"status":"success"}', + "", + ].join("\n"), + { status: 200 }, + ), + }); + vi.stubGlobal("fetch", fetchMock); + + const result = await promptAndConfigureOllama({ cfg: {}, prompter }); + + expect(prompter.confirm).toHaveBeenCalledWith({ + message: "No tools-capable Ollama model is installed. Pull gemma4:e4b (about 9.6 GB)?", + initialValue: false, + }); + const pullCall = fetchMock.mock.calls.find((call) => requestUrl(call[0]).endsWith("/api/pull")); + expect(pullCall).toBeDefined(); + expect(JSON.parse(requestBodyText(pullCall?.[1]?.body))).toEqual({ name: "gemma4:e4b" }); + expect(progress.update).toHaveBeenCalledWith("Downloading gemma4:e4b - pulling part - 50%"); + expect(progress.stop).toHaveBeenCalledWith("Downloaded gemma4:e4b"); + expect(result.config.models?.providers?.ollama?.models?.map((model) => model.id)).toContain( + "gemma4:e4b", + ); + expect( + result.config.models?.providers?.ollama?.models?.find((model) => model.id === "gemma4:e4b"), + ).toMatchObject({ + contextWindow: 131072, + compat: { supportsTools: true }, + }); + }); + + it("does not offer a pull when an installed Ollama model supports tools", async () => { + const prompter = { + ...createLocalPrompter(), + confirm: vi.fn(), + } as unknown as WizardPrompter; + const fetchMock = createOllamaFetchMock({ + tags: ["llama3:8b"], + capabilities: { "llama3:8b": ["tools"] }, + }); + vi.stubGlobal("fetch", fetchMock); + + await promptAndConfigureOllama({ cfg: {}, prompter }); + + expect(prompter.confirm).not.toHaveBeenCalled(); + expect(fetchMock.mock.calls.map((call) => requestUrl(call[0]))).not.toContain( + "http://127.0.0.1:11434/api/pull", + ); + }); + + it("does not pull the recommended Ollama model when declined", async () => { + const prompter = { + ...createLocalPrompter(), + confirm: vi.fn().mockResolvedValueOnce(false), + } as unknown as WizardPrompter; + const fetchMock = createOllamaFetchMock({ + tags: ["llama3:8b"], + capabilities: { "llama3:8b": ["generate"] }, + }); + vi.stubGlobal("fetch", fetchMock); + + await promptAndConfigureOllama({ cfg: {}, prompter }); + + expect(prompter.confirm).toHaveBeenCalledOnce(); + expect(fetchMock.mock.calls.map((call) => requestUrl(call[0]))).not.toContain( + "http://127.0.0.1:11434/api/pull", + ); + }); + + it("does not offer a pull when installed-model capability inspection fails", async () => { + const prompter = { + ...createLocalPrompter(), + confirm: vi.fn(), + } as unknown as WizardPrompter; + const baseFetch = createOllamaFetchMock({ tags: ["llama3:8b"] }); + const fetchMock = vi.fn(async (input: string | URL | Request, init?: RequestInit) => { + if (requestUrl(input).endsWith("/api/show")) { + return new Response("unavailable", { status: 503 }); + } + return await baseFetch(input, init); + }); + vi.stubGlobal("fetch", fetchMock); + + await expect(promptAndConfigureOllama({ cfg: {}, prompter })).rejects.toThrow( + "Failed to inspect Ollama model llama3:8b", + ); + expect(prompter.confirm).not.toHaveBeenCalled(); + }); + + it("checks all installed Ollama models before offering a recommended pull", async () => { + const prompter = { + ...createLocalPrompter(), + confirm: vi.fn(), + } as unknown as WizardPrompter; + const tags = Array.from({ length: 201 }, (_, index) => `model-${index}`); + const capabilities = Object.fromEntries( + tags.map((name, index) => [name, index === 200 ? ["tools"] : ["generate"]]), + ); + const fetchMock = createOllamaFetchMock({ tags, capabilities }); + vi.stubGlobal("fetch", fetchMock); + + const result = await promptAndConfigureOllama({ cfg: {}, prompter }); + + expect(prompter.confirm).not.toHaveBeenCalled(); + expect( + fetchMock.mock.calls.filter((call) => requestUrl(call[0]).endsWith("/api/show")), + ).toHaveLength(201); + expect( + result.config.models?.providers?.ollama?.models?.find((model) => model.id === "model-200"), + ).toMatchObject({ compat: { supportsTools: true } }); + }); + + it("aborts the exhaustive tools-capability scan with the setup session", async () => { + const controller = new AbortController(); + const prompter = { + ...createLocalPrompter(), + confirm: vi.fn(), + } as unknown as WizardPrompter; + const tags = Array.from({ length: 201 }, (_, index) => `model-${index}`); + const capabilities = Object.fromEntries(tags.map((name) => [name, ["generate"]])); + const baseFetch = createOllamaFetchMock({ tags, capabilities }); + let markScanStarted!: () => void; + const scanStarted = new Promise((resolve) => { + markScanStarted = resolve; + }); + const fetchMock = vi.fn(async (input: string | URL | Request, init?: RequestInit) => { + const body = init?.body ? (JSON.parse(requestBodyText(init.body)) as { name?: string }) : {}; + if (!requestUrl(input).endsWith("/api/show") || body.name !== "model-200") { + return await baseFetch(input, init); + } + markScanStarted(); + return await new Promise((_resolve, reject) => { + const signal = init?.signal; + if (!signal) { + reject(new Error("expected tools scan abort signal")); + return; + } + signal.addEventListener("abort", () => reject(abortReasonAsError(signal)), { once: true }); + }); + }); + vi.stubGlobal("fetch", fetchMock); + + const setup = promptAndConfigureOllama({ cfg: {}, prompter, signal: controller.signal }); + await scanStarted; + controller.abort(); + + await expect(setup).rejects.toMatchObject({ name: "AbortError" }); + expect(prompter.confirm).not.toHaveBeenCalled(); + }); + + it("aborts a recommended Ollama pull when the setup session is cancelled", async () => { + const controller = new AbortController(); + const progress = { update: vi.fn(), stop: vi.fn() }; + const prompter = { + select: vi.fn().mockResolvedValueOnce("local-only"), + text: vi.fn().mockResolvedValueOnce("http://127.0.0.1:11434"), + confirm: vi.fn().mockResolvedValueOnce(true), + progress: vi.fn(() => progress), + note: vi.fn(async () => undefined), + } as unknown as WizardPrompter; + const baseFetch = createOllamaFetchMock({ + tags: ["llama3:8b"], + capabilities: { "llama3:8b": ["generate"] }, + }); + let markPullStarted!: () => void; + const pullStarted = new Promise((resolve) => { + markPullStarted = resolve; + }); + const fetchMock = vi.fn(async (input: string | URL | Request, init?: RequestInit) => { + if (!requestUrl(input).endsWith("/api/pull")) { + return await baseFetch(input, init); + } + markPullStarted(); + return await new Promise((_resolve, reject) => { + const signal = init?.signal; + if (!signal) { + reject(new Error("expected pull abort signal")); + return; + } + signal.addEventListener("abort", () => reject(abortReasonAsError(signal)), { once: true }); + }); + }); + vi.stubGlobal("fetch", fetchMock); + + const setup = promptAndConfigureOllama({ cfg: {}, prompter, signal: controller.signal }); + await pullStarted; + controller.abort(); + + await expect(setup).rejects.toThrow("Failed to download recommended Ollama model"); + expect(progress.stop).toHaveBeenCalledWith(expect.stringContaining("Failed to download")); + }); + describe("ensureOllamaModelPulled", () => { it("pulls model when not available locally", async () => { vi.useFakeTimers(); diff --git a/extensions/ollama/src/setup.ts b/extensions/ollama/src/setup.ts index 4d20e4dd5369..ee5524bf6984 100644 --- a/extensions/ollama/src/setup.ts +++ b/extensions/ollama/src/setup.ts @@ -49,6 +49,9 @@ const OLLAMA_CONTEXT_ENRICH_LIMIT = 200; const OLLAMA_CLOUD_MAX_DISCOVERED_MODELS = 500; const OLLAMA_PULL_RESPONSE_TIMEOUT_MS = 30_000; const OLLAMA_PULL_STREAM_IDLE_TIMEOUT_MS = 300_000; +const OLLAMA_RECOMMENDED_TOOLS_MODEL = "gemma4:e4b"; +const OLLAMA_RECOMMENDED_TOOLS_MODEL_SIZE = "about 9.6 GB"; +const OLLAMA_TOOLS_SCAN_CONCURRENCY = 8; type OllamaSetupOptions = { customBaseUrl?: string; @@ -223,6 +226,7 @@ async function pullOllamaModelCore(params: { baseUrl: string; modelName: string; onStatus?: (status: string, percent: number | null) => void; + signal?: AbortSignal; }): Promise { const baseUrl = resolveOllamaApiBase(params.baseUrl); const modelName = normalizeOllamaModelName(params.modelName) ?? params.modelName.trim(); @@ -232,6 +236,7 @@ async function pullOllamaModelCore(params: { OLLAMA_PULL_RESPONSE_TIMEOUT_MS, ); try { + params.signal?.throwIfAborted(); const { response, release } = await fetchWithSsrFGuard({ url: `${baseUrl}/api/pull`, init: { @@ -239,7 +244,9 @@ async function pullOllamaModelCore(params: { headers: { "Content-Type": "application/json" }, body: JSON.stringify({ name: modelName }), }, - signal: responseController.signal, + signal: params.signal + ? AbortSignal.any([responseController.signal, params.signal]) + : responseController.signal, policy: buildOllamaBaseUrlSsrFPolicy(baseUrl), auditContext: "ollama-setup.pull", }); @@ -331,11 +338,13 @@ async function pullOllamaModel( baseUrl: string, modelName: string, prompter: WizardPrompter, + signal?: AbortSignal, ): Promise { const spinner = prompter.progress(`Downloading ${modelName}...`); const result = await pullOllamaModelCore({ baseUrl, modelName, + ...(signal ? { signal } : {}), onStatus: (status, percent) => { const displayStatus = formatOllamaPullStatus(status); if (displayStatus.hidePercent) { @@ -548,11 +557,91 @@ async function resolveHostBackedSuggestedModelNames(params: { return OLLAMA_SUGGESTED_MODELS_LOCAL; } +function parseOllamaSetupShowInfo(data: { + model_info?: Record; + capabilities?: unknown; + parameters?: unknown; +}): Pick { + let contextWindow: number | undefined; + for (const [key, value] of Object.entries(data.model_info ?? {})) { + if (key.endsWith(".context_length") && typeof value === "number" && value > 0) { + contextWindow = Math.floor(value); + break; + } + } + if (typeof data.parameters === "string") { + for (const line of data.parameters.split(/\r?\n/)) { + const match = line.trim().match(/^num_ctx\s+(-?\d+)\b/); + const parsed = match?.[1] ? Number.parseInt(match[1], 10) : undefined; + if (parsed && parsed > 0 && (contextWindow === undefined || parsed > contextWindow)) { + contextWindow = parsed; + } + } + } + const capabilities = Array.isArray(data.capabilities) + ? data.capabilities.filter((value): value is string => typeof value === "string") + : undefined; + return { contextWindow, capabilities }; +} + +async function inspectOllamaModelsForSetup( + baseUrl: string, + models: OllamaModelWithContext[], + signal?: AbortSignal, +): Promise { + const apiBase = resolveOllamaApiBase(baseUrl); + const inspected: OllamaModelWithContext[] = []; + for (let index = 0; index < models.length; index += OLLAMA_TOOLS_SCAN_CONCURRENCY) { + signal?.throwIfAborted(); + const batch = models.slice(index, index + OLLAMA_TOOLS_SCAN_CONCURRENCY); + const results = await Promise.all( + batch.map(async (model) => { + try { + const requestSignal = signal + ? AbortSignal.any([AbortSignal.timeout(3000), signal]) + : AbortSignal.timeout(3000); + const { response, release } = await fetchWithSsrFGuard({ + url: `${apiBase}/api/show`, + init: { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ name: model.name }), + signal: requestSignal, + }, + signal: requestSignal, + policy: buildOllamaBaseUrlSsrFPolicy(apiBase), + auditContext: "ollama-setup.tools-scan", + }); + try { + if (!response.ok) { + throw new Error(`Ollama model inspection failed with HTTP ${response.status}`); + } + const data = await readProviderJsonResponse<{ + model_info?: Record; + capabilities?: unknown; + parameters?: unknown; + }>(response, "ollama-setup.tools-scan"); + return Object.assign({}, model, parseOllamaSetupShowInfo(data)); + } finally { + await release(); + } + } catch (error) { + signal?.throwIfAborted(); + throw new Error(`Failed to inspect Ollama model ${model.name}`, { cause: error }); + } + }), + ); + inspected.push(...results); + } + return inspected; +} + async function promptAndConfigureHostBackedOllama(params: { cfg: OpenClawConfig; mode: HostBackedOllamaInteractiveMode; prompter: WizardPrompter; env?: NodeJS.ProcessEnv; + signal?: AbortSignal; }): Promise { const baseUrl = await promptForOllamaBaseUrl(params.prompter, params.env); const { reachable, models } = await fetchOllamaModels(baseUrl); @@ -562,12 +651,55 @@ async function promptAndConfigureHostBackedOllama(params: { throw new WizardCancelledError("Ollama not reachable"); } - const enrichedModels = await enrichOllamaModelsWithContext( + let inspectedModels = await inspectOllamaModelsForSetup( baseUrl, models.slice(0, OLLAMA_CONTEXT_ENRICH_LIMIT), + params.signal, ); - const discoveredModelsByName = new Map(enrichedModels.map((model) => [model.name, model])); - const discoveredModelNames = models.map((model) => model.name); + const supportsTools = (model: OllamaModelWithContext) => + model.capabilities?.includes("tools") === true; + let hasToolsCapableModel = inspectedModels.some(supportsTools); + if (!hasToolsCapableModel && models.length > OLLAMA_CONTEXT_ENRICH_LIMIT) { + const remainingModels = await inspectOllamaModelsForSetup( + baseUrl, + models.slice(OLLAMA_CONTEXT_ENRICH_LIMIT), + params.signal, + ); + inspectedModels = [...inspectedModels, ...remainingModels]; + hasToolsCapableModel = remainingModels.some(supportsTools); + } + const discoveredModelsByName = new Map(inspectedModels.map((model) => [model.name, model])); + let discoveredModelNames = models.map((model) => model.name); + if (!hasToolsCapableModel) { + const shouldPullRecommended = await params.prompter.confirm({ + message: `No tools-capable Ollama model is installed. Pull ${OLLAMA_RECOMMENDED_TOOLS_MODEL} (${OLLAMA_RECOMMENDED_TOOLS_MODEL_SIZE})?`, + initialValue: false, + }); + if (shouldPullRecommended) { + if ( + !(await pullOllamaModel( + baseUrl, + OLLAMA_RECOMMENDED_TOOLS_MODEL, + params.prompter, + params.signal, + )) + ) { + throw new WizardCancelledError("Failed to download recommended Ollama model"); + } + params.signal?.throwIfAborted(); + const [recommendedModel] = await inspectOllamaModelsForSetup( + baseUrl, + [{ name: OLLAMA_RECOMMENDED_TOOLS_MODEL }], + params.signal, + ); + if (recommendedModel) { + discoveredModelsByName.set(recommendedModel.name, recommendedModel); + } + discoveredModelNames = mergeUniqueModelNames(discoveredModelNames, [ + OLLAMA_RECOMMENDED_TOOLS_MODEL, + ]); + } + } const suggestedModelNames = await resolveHostBackedSuggestedModelNames({ mode: params.mode, baseUrl, @@ -592,6 +724,7 @@ export async function promptAndConfigureOllama(params: { prompter: WizardPrompter; secretInputMode?: SecretInputMode; allowSecretRefPrompt?: boolean; + signal?: AbortSignal; }): Promise { const mode = (await params.prompter.select({ message: "Ollama mode", @@ -640,6 +773,7 @@ export async function promptAndConfigureOllama(params: { mode, prompter: params.prompter, env: params.env, + ...(params.signal ? { signal: params.signal } : {}), }); } diff --git a/src/commands/auth-choice.apply.plugin-provider.test.ts b/src/commands/auth-choice.apply.plugin-provider.test.ts index a1d890409133..b51c6697edc9 100644 --- a/src/commands/auth-choice.apply.plugin-provider.test.ts +++ b/src/commands/auth-choice.apply.plugin-provider.test.ts @@ -531,7 +531,10 @@ describe("applyAuthChoiceLoadedPluginProvider", () => { }, })) as never); - const note = vi.fn(async () => {}); + const events: string[] = []; + const note = vi.fn(async () => { + events.push("note"); + }); const method: ProviderAuthMethod = { id: "local", label: "Local", @@ -576,6 +579,9 @@ describe("applyAuthChoiceLoadedPluginProvider", () => { note, } as unknown as ApplyAuthChoiceParams["prompter"], method, + beforePersistentEffect: () => { + events.push("lock"); + }, }); expect(result.defaultModel).toBe(LOCAL_DEFAULT_MODEL); @@ -592,6 +598,7 @@ describe("applyAuthChoiceLoadedPluginProvider", () => { "Detected local provider runtime.\nPulled model metadata.", "Provider notes", ); + expect(events).toEqual(["note", "lock"]); }); it("normalizes retired Google Gemini default models returned by auth methods", async () => { diff --git a/src/gateway/methods/core-descriptors.ts b/src/gateway/methods/core-descriptors.ts index 2e2aa5b0364b..bd6007695768 100644 --- a/src/gateway/methods/core-descriptors.ts +++ b/src/gateway/methods/core-descriptors.ts @@ -73,6 +73,7 @@ const CORE_GATEWAY_METHOD_SPECS: readonly CoreGatewayMethodSpec[] = [ // without the shared three-write budget so the automatic ladder can finish. { name: "openclaw.setup.activate", scope: "operator.admin" }, { name: "openclaw.setup.auth.start", scope: "operator.admin" }, + { name: "openclaw.setup.prepare.start", scope: "operator.admin" }, { name: "wizard.start", scope: "operator.admin" }, { name: "wizard.next", scope: "operator.admin" }, { name: "wizard.cancel", scope: "operator.admin" }, diff --git a/src/gateway/server-methods.ts b/src/gateway/server-methods.ts index 909b1c625d1c..67b50b420ac6 100644 --- a/src/gateway/server-methods.ts +++ b/src/gateway/server-methods.ts @@ -536,6 +536,7 @@ export const coreGatewayHandlers: GatewayRequestHandlers = { "openclaw.setup.verify", "openclaw.setup.activate", "openclaw.setup.auth.start", + "openclaw.setup.prepare.start", ], loadHandlers: loadSystemAgentHandlers, }), diff --git a/src/gateway/server-methods/system-agent.test.ts b/src/gateway/server-methods/system-agent.test.ts index 19e2b9645954..8e3ae6263f2d 100644 --- a/src/gateway/server-methods/system-agent.test.ts +++ b/src/gateway/server-methods/system-agent.test.ts @@ -29,12 +29,26 @@ const setupInferenceMocks = vi.hoisted(() => ({ detectSetupInference: vi.fn(), verifySetupInference: vi.fn(), })); +const providerAuthChoiceMocks = vi.hoisted(() => ({ + applyAuthChoiceLoadedPluginProvider: vi.fn(), +})); +const setupSharedMocks = vi.hoisted(() => ({ + readSetupConfigFileSnapshot: vi.fn(), + writeWizardConfigFile: vi.fn(), +})); vi.mock("../../system-agent/setup-inference.js", () => ({ activateSetupInference: setupInferenceMocks.activateSetupInference, detectSetupInference: setupInferenceMocks.detectSetupInference, verifySetupInference: setupInferenceMocks.verifySetupInference, })); +vi.mock("../../plugins/provider-auth-choice.js", () => ({ + applyAuthChoiceLoadedPluginProvider: providerAuthChoiceMocks.applyAuthChoiceLoadedPluginProvider, +})); +vi.mock("../../wizard/setup.shared.js", () => ({ + readSetupConfigFileSnapshot: setupSharedMocks.readSetupConfigFileSnapshot, + writeWizardConfigFile: setupSharedMocks.writeWizardConfigFile, +})); type RespondCall = { ok: boolean; @@ -138,6 +152,16 @@ beforeEach(async () => { latencyMs: 10, binding: verifiedInference, }); + setupSharedMocks.readSetupConfigFileSnapshot.mockResolvedValue({ + exists: true, + valid: true, + path: "/tmp/openclaw.json", + hash: "prepare-base-hash", + sourceConfig: verifiedConfig, + config: verifiedConfig, + issues: [], + }); + setupSharedMocks.writeWizardConfigFile.mockImplementation(async (config) => config); }); afterEach(() => { @@ -145,6 +169,9 @@ afterEach(() => { setupInferenceMocks.activateSetupInference.mockReset(); setupInferenceMocks.detectSetupInference.mockReset(); setupInferenceMocks.verifySetupInference.mockReset(); + providerAuthChoiceMocks.applyAuthChoiceLoadedPluginProvider.mockReset(); + setupSharedMocks.readSetupConfigFileSnapshot.mockReset(); + setupSharedMocks.writeWizardConfigFile.mockReset(); verifiedInference = undefined; verifiedInferenceDeps = undefined; resetCommandQueueStateForTest(); @@ -297,6 +324,72 @@ describe("openclaw.setup.auth.start", () => { }); }); +describe("openclaw.setup.prepare.start", () => { + it("runs the selected provider method in a shared wizard session and commits its config", async () => { + const preparedConfig: OpenClawConfig = { + ...verifiedConfig, + models: { providers: { ollama: { baseUrl: "http://127.0.0.1:11434", models: [] } } }, + }; + providerAuthChoiceMocks.applyAuthChoiceLoadedPluginProvider.mockImplementationOnce( + async (params) => { + await params.prompter.note("Model ready", "Ollama"); + await params.beforePersistentEffect(); + return { config: preparedConfig }; + }, + ); + const wizardSessions = new Map(); + const context = { + wizardSessions, + findRunningWizard: () => undefined, + purgeWizardSession: (id: string) => wizardSessions.delete(id), + } as unknown as GatewayRequestContext; + const { calls, respond } = makeRespond(); + + await expectDefined( + systemAgentHandlers["openclaw.setup.prepare.start"], + 'systemAgentHandlers["openclaw.setup.prepare.start"] test invariant', + )({ + params: { + sessionId: "prepare-session-1", + authChoice: "ollama", + workspace: "/tmp/models-workspace", + }, + respond, + context, + } as never); + + expect(calls[0]).toMatchObject({ + ok: true, + payload: { sessionId: "prepare-session-1", done: false, status: "running" }, + }); + const session = wizardSessions.get("prepare-session-1"); + const note = await session.next(); + expect(note).toMatchObject({ + done: false, + step: { type: "note", title: "Ollama", message: "Model ready" }, + }); + expect(providerAuthChoiceMocks.applyAuthChoiceLoadedPluginProvider).toHaveBeenCalledWith( + expect.objectContaining({ + authChoice: "ollama", + config: verifiedConfig, + workspaceDir: "/tmp/models-workspace", + setDefaultModel: false, + preserveExistingDefaultModel: true, + signal: session.signal, + isRemote: true, + }), + ); + await session.answer(note.step.id, null); + await expect(session.next()).resolves.toMatchObject({ done: true, status: "done" }); + expect(setupSharedMocks.writeWizardConfigFile).toHaveBeenCalledWith(preparedConfig, { + allowConfigSizeDrop: false, + baseSnapshot: expect.objectContaining({ hash: "prepare-base-hash" }), + baseHash: "prepare-base-hash", + migrationBaseConfig: verifiedConfig, + }); + }); +}); + describe("openclaw.chat", () => { it("refuses to create a session before inference is available", async () => { setupInferenceMocks.verifySetupInference.mockResolvedValueOnce({ diff --git a/src/gateway/server-methods/system-agent.ts b/src/gateway/server-methods/system-agent.ts index b3d63d005f2a..702e8a47915b 100644 --- a/src/gateway/server-methods/system-agent.ts +++ b/src/gateway/server-methods/system-agent.ts @@ -24,6 +24,7 @@ import { isSystemAgentInferenceUnavailableError } from "../../system-agent/infer import { buildOnboardingWelcome } from "../../system-agent/onboarding-welcome.js"; import { describeSystemAgentPersistentOperation } from "../../system-agent/operations.js"; import { formatSystemAgentStartupMessage } from "../../system-agent/overview.js"; +import { resolveUserPath } from "../../utils.js"; import { WizardSession } from "../../wizard/session.js"; import { buildRequestedApprovalEvent, @@ -48,6 +49,7 @@ export type SystemAgentChatSession = const MAX_SYSTEM_AGENT_SESSIONS = 8; const PROVIDER_AUTH_SESSION_TIMEOUT_MS = 25 * 60 * 1000; +const PROVIDER_PREPARE_SESSION_TIMEOUT_MS = 2 * 60 * 60 * 1000; const SYSTEM_AGENT_GATEWAY_EXECUTION_KEY = "gateway"; const systemAgentGatewayExecutionQueue = new KeyedAsyncQueue(); const systemAgentSessionQueues = new WeakMap< @@ -286,6 +288,80 @@ export const systemAgentHandlers: GatewayRequestHandlers = { // Return ownership immediately so the client can cancel while provider auth waits. respond(true, { sessionId, done: false, status: "running" }, undefined); }, + /** Run one provider-owned prepare flow over the shared wizard transport. */ + "openclaw.setup.prepare.start": async ({ params, respond, context }) => { + if ( + !assertValidParams( + params, + validateSystemAgentSetupAuthStartParams, + "openclaw.setup.prepare.start", + respond, + ) + ) { + return; + } + if (context.findRunningWizard()) { + respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, "wizard already running")); + return; + } + const sessionId = params.sessionId; + const session = new WizardSession( + async (prompter, signal) => { + await runExclusiveSystemAgentSetupActivation(async () => + runSystemAgentGatewayTask(async () => { + const [{ applyAuthChoiceLoadedPluginProvider }, setupShared] = await Promise.all([ + import("../../plugins/provider-auth-choice.js"), + import("../../wizard/setup.shared.js"), + ]); + const snapshot = await setupShared.readSetupConfigFileSnapshot(); + if (!snapshot.valid) { + throw new Error("Config is invalid. Run `openclaw doctor` before preparing a model."); + } + // Match the classic wizard: mutate the authored shape, not runtimeConfig, + // so setup never writes resolved runtime defaults into openclaw.json. + const baseConfig = snapshot.exists ? snapshot.sourceConfig : {}; + const workspaceDir = params.workspace?.trim() + ? resolveUserPath(params.workspace.trim()) + : undefined; + const applied = await applyAuthChoiceLoadedPluginProvider({ + authChoice: params.authChoice, + config: baseConfig, + prompter, + runtime: { + ...defaultRuntime, + exit: (code: number | undefined): never => { + throw new Error(`setup step exited with code ${String(code)}`); + }, + }, + setDefaultModel: false, + preserveExistingDefaultModel: true, + ...(workspaceDir ? { workspaceDir } : {}), + signal, + isRemote: true, + beforePersistentEffect: () => { + signal.throwIfAborted(); + session.lockCancellation(); + }, + }); + if (!applied || applied.retrySelection) { + throw new Error(`Provider prepare method is unavailable: ${params.authChoice}`); + } + signal.throwIfAborted(); + session.lockCancellation(); + await setupShared.writeWizardConfigFile(applied.config, { + allowConfigSizeDrop: false, + baseSnapshot: snapshot, + ...(snapshot.hash ? { baseHash: snapshot.hash } : {}), + migrationBaseConfig: baseConfig, + }); + }), + ); + }, + { timeoutMs: PROVIDER_PREPARE_SESSION_TIMEOUT_MS }, + ); + context.wizardSessions.set(sessionId, session); + respond(true, { sessionId, done: false, status: "running" }, undefined); + }, /** * Structured onboarding: live-test one candidate and persist it on success. * Single-flight per gateway process because testing and persistence span diff --git a/src/plugins/provider-auth-choice.ts b/src/plugins/provider-auth-choice.ts index 4e97f5199c0d..22539280a267 100644 --- a/src/plugins/provider-auth-choice.ts +++ b/src/plugins/provider-auth-choice.ts @@ -43,6 +43,10 @@ type ApplyProviderAuthChoiceParams = { setDefaultModel: boolean; preserveExistingDefaultModel?: boolean; agentId?: string; + workspaceDir?: string; + signal?: AbortSignal; + isRemote?: boolean; + beforePersistentEffect?: () => void | Promise; opts?: Partial; }; @@ -305,6 +309,9 @@ export async function runProviderPluginAuthMethod(params: { agentDir?: string; agentId?: string; workspaceDir?: string; + signal?: AbortSignal; + isRemote?: boolean; + beforePersistentEffect?: () => void | Promise; emitNotes?: boolean; secretInputMode?: ProviderAuthOptionBag["secretInputMode"]; allowSecretRefPrompt?: boolean; @@ -324,11 +331,18 @@ export async function runProviderPluginAuthMethod(params: { method: params.method, agentDir, workspaceDir, + ...(params.signal ? { signal: params.signal } : {}), + ...(params.isRemote !== undefined ? { isRemote: params.isRemote } : {}), secretInputMode: params.secretInputMode, allowSecretRefPrompt: params.allowSecretRefPrompt, opts: params.opts, }); + if (params.emitNotes !== false && result.notes && result.notes.length > 0) { + await params.prompter.note(result.notes.join("\n"), "Provider notes"); + } + + await params.beforePersistentEffect?.(); for (const profile of result.profiles) { await upsertAuthProfileWithLockOrThrow({ profileId: profile.profileId, @@ -342,10 +356,6 @@ export async function runProviderPluginAuthMethod(params: { result, }); - if (params.emitNotes !== false && result.notes && result.notes.length > 0) { - await params.prompter.note(result.notes.join("\n"), "Provider notes"); - } - const defaultModel = result.defaultModel ? normalizeAgentModelRefForConfig(result.defaultModel) : undefined; @@ -361,7 +371,9 @@ export async function applyAuthChoiceLoadedPluginProvider( ): Promise { const agentId = params.agentId ?? resolveDefaultAgentId(params.config); const workspaceDir = - resolveAgentWorkspaceDir(params.config, agentId) ?? resolveDefaultAgentWorkspaceDir(); + params.workspaceDir ?? + resolveAgentWorkspaceDir(params.config, agentId) ?? + resolveDefaultAgentWorkspaceDir(); let nextConfig = params.config; let enabledConfig = params.config; const { @@ -480,6 +492,11 @@ export async function applyAuthChoiceLoadedPluginProvider( agentDir: params.agentDir, agentId: params.agentId, workspaceDir, + ...(params.signal ? { signal: params.signal } : {}), + ...(params.isRemote !== undefined ? { isRemote: params.isRemote } : {}), + ...(params.beforePersistentEffect + ? { beforePersistentEffect: params.beforePersistentEffect } + : {}), secretInputMode: params.opts?.secretInputMode, allowSecretRefPrompt: false, opts: params.opts, diff --git a/src/wizard/session.test.ts b/src/wizard/session.test.ts index d9752ff7f51d..9d3af9bbca5b 100644 --- a/src/wizard/session.test.ts +++ b/src/wizard/session.test.ts @@ -272,4 +272,75 @@ describe("WizardSession", () => { } await session.answer(plainStep.id, "alice"); }); + + test("bridges confirm, progress updates, and notes in order", async () => { + let markInitialUpdateQueued!: () => void; + const initialUpdateQueued = new Promise((resolve) => { + markInitialUpdateQueued = resolve; + }); + let releaseHalfway!: () => void; + const halfway = new Promise((resolve) => { + releaseHalfway = resolve; + }); + let releaseDone!: () => void; + const done = new Promise((resolve) => { + releaseDone = resolve; + }); + const session = new WizardSession(async (prompter) => { + await prompter.confirm({ message: "Download model?", initialValue: false }); + const progress = prompter.progress("Starting download"); + progress.update("Downloading model... 10%"); + markInitialUpdateQueued(); + await halfway; + progress.update("Downloading model... 50%"); + await done; + progress.stop("Model downloaded"); + await prompter.note("Ready to use", "Prepared"); + }); + + const confirm = await session.next(); + expect(confirm.step).toMatchObject({ + type: "confirm", + message: "Download model?", + initialValue: false, + }); + if (!confirm.step) { + throw new Error("expected confirm step"); + } + await session.answer(confirm.step.id, true); + await initialUpdateQueued; + + expect(await session.next()).toMatchObject({ + step: { + type: "progress", + message: "Starting download", + executor: "gateway", + }, + }); + + expect(await session.next()).toMatchObject({ + step: { type: "progress", message: "Downloading model... 10%" }, + }); + + const halfwayStep = session.next(); + releaseHalfway(); + expect(await halfwayStep).toMatchObject({ + step: { type: "progress", message: "Downloading model... 50%" }, + }); + + const doneStep = session.next(); + releaseDone(); + const completedProgress = await doneStep; + expect(completedProgress).toMatchObject({ + step: { type: "progress", message: "Model downloaded" }, + }); + if (!completedProgress.step) { + throw new Error("expected completed progress step"); + } + await expect(session.answer(completedProgress.step.id, undefined)).resolves.toBeUndefined(); + + expect(await session.next()).toMatchObject({ + step: { type: "note", title: "Prepared", message: "Ready to use" }, + }); + }); }); diff --git a/src/wizard/session.ts b/src/wizard/session.ts index a0e0be57a497..26fcd4581d2e 100644 --- a/src/wizard/session.ts +++ b/src/wizard/session.ts @@ -186,10 +186,24 @@ class WizardSessionPrompter implements WizardPrompter { return Boolean(res); } - progress(_label: string): WizardProgress { + progress(label: string): WizardProgress { + let stopped = false; + this.session.pushProgress(label); return { - update: (_message) => {}, - stop: (_message) => {}, + update: (message) => { + if (!stopped) { + this.session.pushProgress(message); + } + }, + stop: (message) => { + if (stopped) { + return; + } + stopped = true; + if (message) { + this.session.pushProgress(message); + } + }, }; } @@ -218,6 +232,8 @@ export class WizardSession { private readonly abortController = new AbortController(); private readonly expiryTimer: ReturnType | undefined; private currentStep: WizardStep | null = null; + private progressSteps: WizardStep[] = []; + private deliveredProgressStepIds = new Set(); private stepDeferred: Deferred | null = null; private pendingTerminalResolution = false; private cancellationLocked = false; @@ -251,6 +267,11 @@ export class WizardSession { } async next(): Promise { + const progressStep = this.progressSteps.shift(); + if (progressStep) { + this.rememberDeliveredProgressStep(progressStep.id); + return { done: false, step: progressStep, status: this.status }; + } if (this.currentStep) { return { done: false, step: this.currentStep, status: this.status }; } @@ -292,6 +313,12 @@ export class WizardSession { async answer(stepId: string, value: unknown): Promise { const pending = this.answerDeferred.get(stepId); if (!pending) { + // Gateway-owned progress steps never block the provider run. Older + // clients still acknowledge every rendered step, so accept that stale + // acknowledgement while newer clients poll without an answer. + if (this.deliveredProgressStepIds.delete(stepId)) { + return undefined; + } throw new Error("wizard: no pending step"); } const normalizedValue = pending.text ? normalizeTextAnswer(value) : value; @@ -322,6 +349,8 @@ export class WizardSession { pending.deferred.reject(new WizardCancelledError()); } this.answerDeferred.clear(); + this.progressSteps = []; + this.deliveredProgressStepIds.clear(); this.resolveStep(null); return true; } @@ -340,6 +369,41 @@ export class WizardSession { this.resolveStep(step); } + pushProgress(message: string) { + if (this.status !== "running") { + return; + } + const step: WizardStep = { + id: randomUUID(), + type: "progress", + message, + executor: "gateway", + }; + if (this.stepDeferred) { + this.rememberDeliveredProgressStep(step.id); + this.resolveStep(step); + return; + } + // Keep the oldest unread event and the newest snapshot. This preserves the + // initial label while bounding bursty pull updates between client polls. + if (this.progressSteps.length >= 2) { + this.progressSteps[this.progressSteps.length - 1] = step; + return; + } + this.progressSteps.push(step); + } + + private rememberDeliveredProgressStep(stepId: string) { + this.deliveredProgressStepIds.add(stepId); + if (this.deliveredProgressStepIds.size <= 64) { + return; + } + const oldest = this.deliveredProgressStepIds.values().next().value; + if (oldest) { + this.deliveredProgressStepIds.delete(oldest); + } + } + queueExternalUrl(url: string) { this.pendingExternalUrl = url; } diff --git a/src/wizard/setup.shared.test.ts b/src/wizard/setup.shared.test.ts index 6c725fff2953..6c0f12a43e2a 100644 --- a/src/wizard/setup.shared.test.ts +++ b/src/wizard/setup.shared.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import type { OpenClawConfig } from "../config/types.openclaw.js"; +import type { ConfigFileSnapshot, OpenClawConfig } from "../config/types.openclaw.js"; import { withoutPluginInstallRecords } from "../plugins/installed-plugin-index-records.js"; const mocks = vi.hoisted(() => ({ @@ -97,4 +97,31 @@ describe("writeWizardConfigFile pending install ownership", () => { }), ); }); + + it("preserves an absent config snapshot through the final write", async () => { + const config: OpenClawConfig = { gateway: { port: 18789 } }; + const baseSnapshot: ConfigFileSnapshot = { + path: "/tmp/openclaw.json", + exists: false, + raw: null, + parsed: undefined, + sourceConfig: {}, + resolved: {}, + valid: true, + runtimeConfig: {}, + config: {}, + issues: [], + warnings: [], + legacyIssues: [], + }; + + await writeWizardConfigFile(config, { baseSnapshot }); + + const commit = mocks.commitConfigWriteWithPendingPluginInstalls.mock.calls[0]?.[0]?.commit; + expect(commit).toBeTypeOf("function"); + await commit(config); + expect(mocks.replaceConfigFile).toHaveBeenCalledWith( + expect.objectContaining({ nextConfig: config, snapshot: baseSnapshot }), + ); + }); }); diff --git a/src/wizard/setup.shared.ts b/src/wizard/setup.shared.ts index 93fb009e16f6..86ebdb83d594 100644 --- a/src/wizard/setup.shared.ts +++ b/src/wizard/setup.shared.ts @@ -2,7 +2,7 @@ import { isDeepStrictEqual } from "node:util"; import type { GatewayAuthChoice, OnboardOptions } from "../commands/onboard-types.js"; import { createConfigIO, replaceConfigFile, resolveGatewayPort } from "../config/config.js"; -import type { OpenClawConfig } from "../config/types.openclaw.js"; +import type { ConfigFileSnapshot, OpenClawConfig } from "../config/types.openclaw.js"; import { commitConfigWriteWithPendingPluginInstalls, hasPendingPluginInstallRecords, @@ -58,12 +58,15 @@ export async function writeWizardConfigFile( allowConfigSizeDrop?: boolean; /** Reject the write if config changed after the caller's verified snapshot. */ baseHash?: string; + /** Preserve an absent-file precondition that cannot be represented by baseHash. */ + baseSnapshot?: ConfigFileSnapshot; migrationBaseConfig?: OpenClawConfig; onPendingPluginInstallMigration?: () => void; } = {}, ): Promise { let config = configInput; let baseHash = opts.baseHash; + let baseSnapshot = opts.baseSnapshot; const allowConfigSizeDrop = opts.allowConfigSizeDrop === true; if (!allowConfigSizeDrop && hasPendingPluginInstallRecords(config)) { // Explicit undefined means this writer already migrated its baseline; an omitted @@ -82,6 +85,7 @@ export async function writeWizardConfigFile( commit: async (nextConfig, writeOptions) => { return await replaceConfigFile({ nextConfig, + ...(baseSnapshot ? { snapshot: baseSnapshot } : {}), ...(baseHash !== undefined ? { baseHash } : {}), ...(writeOptions ? { writeOptions } : {}), afterWrite: { mode: "auto" }, @@ -89,6 +93,7 @@ export async function writeWizardConfigFile( }, }); baseHash = migration.persistedHash ?? undefined; + baseSnapshot = undefined; config = stripPendingPluginInstallRecords( config, unchangedPendingPluginInstallRecordIds(config, migrationBaseConfig), @@ -102,6 +107,7 @@ export async function writeWizardConfigFile( commit: async (nextConfig, writeOptions) => { return await replaceConfigFile({ nextConfig, + ...(baseSnapshot ? { snapshot: baseSnapshot } : {}), ...(baseHash !== undefined ? { baseHash } : {}), ...(writeOptions ? { writeOptions } : {}), afterWrite: { mode: "auto" },