diff --git a/extensions/openai/openai-provider.live.test.ts b/extensions/openai/openai-provider.live.test.ts index 836ae21ce0ed..5f85ca3566b6 100644 --- a/extensions/openai/openai-provider.live.test.ts +++ b/extensions/openai/openai-provider.live.test.ts @@ -5,7 +5,13 @@ import { describe, expect, it } from "vitest"; import { buildOpenAIProvider } from "./openai-provider.js"; const OPENAI_API_KEY = process.env.OPENAI_API_KEY ?? ""; -const DEFAULT_LIVE_MODEL_IDS = ["chat-latest", "gpt-5.5", "gpt-5.4-mini", "gpt-5.4-nano"] as const; +const DEFAULT_LIVE_MODEL_IDS = [ + "gpt-5.6", + "chat-latest", + "gpt-5.5", + "gpt-5.4-mini", + "gpt-5.4-nano", +] as const; const liveEnabled = OPENAI_API_KEY.trim().length > 0 && process.env.OPENCLAW_LIVE_TEST === "1"; const describeLive = liveEnabled ? describe : describe.skip; @@ -22,6 +28,25 @@ type LiveModelCase = { function resolveLiveModelCase(modelId: string): LiveModelCase { switch (modelId) { + case "gpt-5.6": + case "gpt-5.6-sol": + case "gpt-5.6-terra": + case "gpt-5.6-luna": + return { + modelId, + templateId: "gpt-5.5", + templateName: "GPT-5.5", + cost: + modelId === "gpt-5.6-terra" + ? { input: 2.5, output: 15, cacheRead: 0.25, cacheWrite: 3.125 } + : modelId === "gpt-5.6-luna" + ? { input: 1, output: 6, cacheRead: 0.1, cacheWrite: 1.25 } + : { input: 5, output: 30, cacheRead: 0.5, cacheWrite: 6.25 }, + contextWindow: 1_050_000, + maxTokens: 128_000, + reasoning: true, + textVerbosity: "low", + }; case "chat-latest": return { modelId, @@ -39,7 +64,7 @@ function resolveLiveModelCase(modelId: string): LiveModelCase { templateId: "gpt-5.5", templateName: "GPT-5.5", cost: { input: 5, output: 30, cacheRead: 0.5, cacheWrite: 0 }, - contextWindow: 1_000_000, + contextWindow: 1_050_000, maxTokens: 128_000, reasoning: true, textVerbosity: "low", @@ -50,7 +75,7 @@ function resolveLiveModelCase(modelId: string): LiveModelCase { templateId: "gpt-5.4-pro", templateName: "GPT-5.4 Pro", cost: { input: 30, output: 180, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 1_000_000, + contextWindow: 1_050_000, maxTokens: 128_000, reasoning: true, textVerbosity: "low", @@ -60,8 +85,8 @@ function resolveLiveModelCase(modelId: string): LiveModelCase { modelId, templateId: "gpt-5.2", templateName: "GPT-5.2", - cost: { input: 1.75, output: 14, cacheRead: 0.175, cacheWrite: 0 }, - contextWindow: 400_000, + cost: { input: 2.5, output: 15, cacheRead: 0.25, cacheWrite: 0 }, + contextWindow: 1_050_000, maxTokens: 128_000, reasoning: true, textVerbosity: "low", @@ -71,8 +96,8 @@ function resolveLiveModelCase(modelId: string): LiveModelCase { modelId, templateId: "gpt-5.2-pro", templateName: "GPT-5.2 Pro", - cost: { input: 21, output: 168, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 400_000, + cost: { input: 30, output: 180, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 1_050_000, maxTokens: 128_000, reasoning: true, textVerbosity: "low", @@ -82,7 +107,7 @@ function resolveLiveModelCase(modelId: string): LiveModelCase { modelId, templateId: "gpt-5-mini", templateName: "GPT-5 mini", - cost: { input: 0.25, output: 2, cacheRead: 0.025, cacheWrite: 0 }, + cost: { input: 0.75, output: 4.5, cacheRead: 0.075, cacheWrite: 0 }, contextWindow: 400_000, maxTokens: 128_000, reasoning: true, @@ -93,7 +118,7 @@ function resolveLiveModelCase(modelId: string): LiveModelCase { modelId, templateId: "gpt-5-nano", templateName: "GPT-5 nano", - cost: { input: 0.05, output: 0.4, cacheRead: 0.005, cacheWrite: 0 }, + cost: { input: 0.2, output: 1.25, cacheRead: 0.02, cacheWrite: 0 }, contextWindow: 400_000, maxTokens: 128_000, reasoning: true, diff --git a/extensions/openai/openai-provider.test.ts b/extensions/openai/openai-provider.test.ts index d8016509c780..f456680af8b9 100644 --- a/extensions/openai/openai-provider.test.ts +++ b/extensions/openai/openai-provider.test.ts @@ -493,6 +493,12 @@ describe("buildOpenAIProvider", () => { data: [ { id: "gpt-5.6", object: "model" }, { id: "gpt-5.5", object: "model" }, + { id: "chat-latest", object: "model" }, + { id: "gpt-5.4", object: "model" }, + { id: "gpt-5.4-pro", object: "model" }, + { id: "gpt-5.4-mini", object: "model" }, + { id: "gpt-5.4-nano", object: "model" }, + { id: "gpt-5.3-codex-spark", object: "model" }, { id: "not-in-manifest", object: "model" }, ], }), @@ -508,6 +514,47 @@ describe("buildOpenAIProvider", () => { expect(provider.apiKey).toBe("sk-openai"); expect(provider.models.map((model) => model.id)).toContain("gpt-5.6"); expect(provider.models.map((model) => model.id)).toContain("gpt-5.5"); + expect(provider.models.map((model) => model.id)).toEqual( + expect.arrayContaining([ + "chat-latest", + "gpt-5.4", + "gpt-5.4-pro", + "gpt-5.4-mini", + "gpt-5.4-nano", + ]), + ); + expect(provider.models.find((model) => model.id === "chat-latest")).toMatchObject({ + api: "openai-responses", + baseUrl: "https://api.openai.com/v1", + contextWindow: 400_000, + maxTokens: 128_000, + reasoning: false, + cost: { input: 5, output: 30, cacheRead: 0.5, cacheWrite: 0 }, + }); + expect(provider.models.find((model) => model.id === "gpt-5.4-pro")).toMatchObject({ + api: "openai-responses", + baseUrl: "https://api.openai.com/v1", + contextWindow: 1_050_000, + maxTokens: 128_000, + reasoning: true, + input: ["text"], + cost: { input: 30, output: 180, cacheRead: 0, cacheWrite: 0 }, + }); + expect(provider.models.find((model) => model.id === "gpt-5.4-mini")).toMatchObject({ + api: "openai-responses", + baseUrl: "https://api.openai.com/v1", + contextWindow: 400_000, + maxTokens: 128_000, + cost: { input: 0.75, output: 4.5, cacheRead: 0.075, cacheWrite: 0 }, + }); + expect(provider.models.find((model) => model.id === "gpt-5.4-nano")).toMatchObject({ + api: "openai-responses", + baseUrl: "https://api.openai.com/v1", + contextWindow: 400_000, + maxTokens: 128_000, + cost: { input: 0.2, output: 1.25, cacheRead: 0.02, cacheWrite: 0 }, + }); + expect(provider.models.map((model) => model.id)).not.toContain("gpt-5.3-codex-spark"); expect(provider.models.map((model) => model.id)).not.toContain("not-in-manifest"); const fetchParams = vi.mocked(fetchGuard).mock.calls[0]?.[0]; expect(fetchParams?.url).toBe("https://api.openai.com/v1/models"); @@ -521,6 +568,38 @@ describe("buildOpenAIProvider", () => { expect(release).toHaveBeenCalledOnce(); }); + it("does not surface platform models omitted by the account's live catalog", async () => { + const fetchGuard: LiveModelCatalogFetchGuard = vi.fn(async () => ({ + response: Response.json({ data: [{ id: "gpt-5.5", object: "model" }] }), + finalUrl: "https://api.openai.com/v1/models", + release: async () => undefined, + })); + + const provider = await buildOpenAILiveProviderConfig({ + apiKey: "sk-openai", + fetchGuard, + }); + + expect(provider.models.map((model) => model.id)).toEqual(["gpt-5.5"]); + }); + + it("keeps only manifest fallback models when OpenAI discovery is unavailable", async () => { + const fetchGuard: LiveModelCatalogFetchGuard = vi.fn(async () => ({ + response: new Response("temporarily unavailable", { status: 503 }), + finalUrl: "https://api.openai.com/v1/models", + release: async () => undefined, + })); + + const provider = await buildOpenAILiveProviderConfig({ + apiKey: "sk-openai", + fetchGuard, + }); + + expect(provider.models.map((model) => model.id)).toEqual( + manifest.modelCatalog.providers.openai.models.map((model) => model.id), + ); + }); + it("skips OpenAI live discovery for custom OpenAI-compatible base URLs", async () => { const customBaseUrl = "https://example-proxy.invalid/v1"; const fetchGuard: LiveModelCatalogFetchGuard = vi.fn(async () => { diff --git a/extensions/openai/openai-provider.ts b/extensions/openai/openai-provider.ts index b47336e1be8c..2bddb3dd624c 100644 --- a/extensions/openai/openai-provider.ts +++ b/extensions/openai/openai-provider.ts @@ -185,6 +185,60 @@ function buildOpenAIManifestModelsForBaseUrl(baseUrl: string): ModelDefinitionCo ); } +function buildOpenAIDiscoverablePlatformModels(baseUrl: string): ModelDefinitionConfig[] { + const models = [ + { + id: OPENAI_CHAT_LATEST_MODEL_ID, + name: "Chat Latest", + reasoning: false, + cost: OPENAI_CHAT_LATEST_COST, + contextWindow: 400_000, + }, + { + id: OPENAI_GPT_54_MODEL_ID, + name: "GPT-5.4", + reasoning: true, + cost: OPENAI_GPT_54_COST, + contextWindow: OPENAI_GPT_54_CONTEXT_TOKENS, + }, + { + id: OPENAI_GPT_54_PRO_MODEL_ID, + name: "GPT-5.4 Pro", + reasoning: true, + cost: OPENAI_GPT_54_PRO_COST, + contextWindow: OPENAI_GPT_54_PRO_CONTEXT_TOKENS, + }, + { + id: OPENAI_GPT_54_MINI_MODEL_ID, + name: "GPT-5.4 Mini", + reasoning: true, + cost: OPENAI_GPT_54_MINI_COST, + contextWindow: OPENAI_GPT_54_MINI_CONTEXT_TOKENS, + }, + { + id: OPENAI_GPT_54_NANO_MODEL_ID, + name: "GPT-5.4 Nano", + reasoning: true, + cost: OPENAI_GPT_54_NANO_COST, + contextWindow: OPENAI_GPT_54_NANO_CONTEXT_TOKENS, + }, + ] as const; + + // First-party discovery must retain provider-owned costs and capabilities; + // generic projection would otherwise surface valid models as zero-cost. + return models.map(({ id, name, reasoning, cost, contextWindow }) => ({ + id, + name, + reasoning, + cost, + contextWindow, + api: "openai-responses", + baseUrl, + input: id === OPENAI_GPT_54_PRO_MODEL_ID ? ["text"] : ["text", "image"], + maxTokens: OPENAI_GPT_54_MAX_TOKENS, + })); +} + async function buildOpenAILiveProviderConfig( params: BuildOpenAILiveProviderConfigParams, ): Promise { @@ -207,6 +261,33 @@ async function buildOpenAILiveProviderConfig( api: "openai-responses", }, models, + projectRows: (rows, fallback) => { + const discoveredIds = new Set( + rows.flatMap((row) => { + if (!row || typeof row !== "object" || Array.isArray(row)) { + return []; + } + const candidate = row as { id?: unknown; object?: unknown }; + if (candidate.object !== undefined && candidate.object !== "model") { + return []; + } + const modelId = typeof candidate.id === "string" ? candidate.id.trim() : ""; + return modelId ? [modelId] : []; + }), + ); + const selectedIds = new Set(); + // Discovery alone confirms account access; leave the manifest as the + // advisory fallback when OpenAI cannot return an authenticated catalog. + return [...fallback.models, ...buildOpenAIDiscoverablePlatformModels(baseUrl)].filter( + (model) => { + if (!discoveredIds.has(model.id) || selectedIds.has(model.id)) { + return false; + } + selectedIds.add(model.id); + return true; + }, + ); + }, apiKey: params.apiKey, discoveryApiKey: params.discoveryApiKey, fetchGuard: params.fetchGuard, diff --git a/test/openai-onboarding.live.test.ts b/test/openai-onboarding.live.test.ts new file mode 100644 index 000000000000..021ffe073e9d --- /dev/null +++ b/test/openai-onboarding.live.test.ts @@ -0,0 +1,177 @@ +// Real-key onboarding must persist an env reference and complete the default first turn. +import { execFile } from "node:child_process"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; +import { describe, expect, it } from "vitest"; +import { extractAgentReplyTexts } from "../scripts/e2e/lib/agent-turn-output.mjs"; +import { readPersistedAuthProfileStoreRaw } from "../src/agents/auth-profiles/sqlite.js"; +import { isLiveTestEnabled } from "../src/agents/live-test-helpers.js"; +import { createOpenClawTestState } from "../src/test-utils/openclaw-test-state.js"; + +const execFileAsync = promisify(execFile); +const openAiApiKey = process.env.OPENAI_API_KEY?.trim() ?? ""; +const describeLive = isLiveTestEnabled() && openAiApiKey.length > 0 ? describe : describe.skip; +const replyMarker = "OPENCLAW_OPENAI_ONBOARDING_OK"; + +async function runOpenClaw(args: string[], env: NodeJS.ProcessEnv): Promise { + try { + const result = await execFileAsync(process.execPath, ["scripts/run-node.mjs", ...args], { + cwd: path.resolve(import.meta.dirname, ".."), + env, + maxBuffer: 2 * 1024 * 1024, + timeout: 180_000, + }); + expect(result.stdout.includes(openAiApiKey)).toBe(false); + expect(result.stderr.includes(openAiApiKey)).toBe(false); + return result.stdout; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(message.replaceAll(openAiApiKey, "[REDACTED]")); + } +} + +function assertOpenAiEnvProfile(agentDir: string): void { + const store = readPersistedAuthProfileStoreRaw(agentDir) as { + profiles?: Record>; + } | null; + expect(store?.profiles).toBeDefined(); + // Assert on booleans before inspecting the profile so a broken inline-key + // migration can never echo a real live credential in Vitest diagnostics. + expect(JSON.stringify(store).includes(openAiApiKey)).toBe(false); + const profile = Object.values(store?.profiles ?? {}).find( + (candidate) => candidate.type === "api_key" && candidate.provider === "openai", + ); + const keyRef = profile?.keyRef as + | { source?: unknown; provider?: unknown; id?: unknown } + | undefined; + expect(profile !== undefined).toBe(true); + expect(profile?.type === "api_key").toBe(true); + expect(profile?.provider === "openai").toBe(true); + expect(keyRef?.source === "env").toBe(true); + expect(keyRef?.provider === "default").toBe(true); + expect(keyRef?.id === "OPENAI_API_KEY").toBe(true); + expect(Object.hasOwn(profile ?? {}, "key")).toBe(false); +} + +function summarizeAgentOutput(stdout: string): string { + const trimmed = stdout.trim(); + const jsonStart = trimmed.lastIndexOf("\n{"); + const rawJson = jsonStart >= 0 ? trimmed.slice(jsonStart + 1) : trimmed; + try { + const payload = JSON.parse(rawJson) as { + status?: string; + error?: unknown; + payloads?: Array<{ isError?: boolean; text?: string }>; + meta?: { provider?: string; model?: string; stopReason?: string; error?: unknown }; + result?: { + status?: string; + payloads?: Array<{ isError?: boolean; text?: string }>; + meta?: { provider?: string; model?: string; stopReason?: string; error?: unknown }; + }; + }; + const meta = payload.result?.meta ?? payload.meta; + const payloads = payload.result?.payloads ?? payload.payloads ?? []; + return JSON.stringify({ + status: payload.result?.status ?? payload.status, + provider: meta?.provider, + model: meta?.model, + stopReason: meta?.stopReason, + hasError: payload.error !== undefined || meta?.error !== undefined, + payloadCount: payloads.length, + errorPayloadCount: payloads.filter((entry) => entry.isError === true).length, + outputBytes: Buffer.byteLength(stdout), + }); + } catch { + return JSON.stringify({ outputBytes: Buffer.byteLength(stdout), validJson: false }); + } +} + +describeLive("fresh OpenAI onboarding live", () => { + it("keeps repeated onboarding secret-safe and runs the actual default model", async () => { + const state = await createOpenClawTestState({ + label: "openai-onboarding-live", + layout: "state-only", + scenario: "empty", + applyEnv: false, + // CLI children must take the production path, not inherit Vitest-only + // provider discovery and runtime shortcuts from the live-test worker. + env: { + NODE_ENV: undefined, + VITEST: undefined, + VITEST_POOL_ID: undefined, + VITEST_WORKER_ID: undefined, + OPENCLAW_TEST_FAST: undefined, + OPENCLAW_TEST_HOME: undefined, + OPENCLAW_TEST_MINIMAL_GATEWAY: undefined, + OPENCLAW_TEST_TRUST_BUNDLED_PLUGINS_DIR: undefined, + OPENCLAW_BUNDLED_PLUGINS_DIR: undefined, + OPENCLAW_DISABLE_BUNDLED_PLUGINS: undefined, + OPENCLAW_PLUGIN_CATALOG_PATHS: undefined, + OPENCLAW_PLUGINS_PATHS: undefined, + }, + }); + + try { + await expect(fs.access(state.configPath)).rejects.toThrow(); + const onboardArgs = [ + "onboard", + "--non-interactive", + "--accept-risk", + "--mode", + "local", + "--auth-choice", + "openai-api-key", + "--secret-input-mode", + "ref", + "--gateway-bind", + "loopback", + "--skip-daemon", + "--skip-ui", + "--skip-skills", + "--skip-health", + "--suppress-gateway-token-output", + "--json", + ]; + + for (let attempt = 0; attempt < 2; attempt += 1) { + await runOpenClaw(onboardArgs, state.env); + const rawConfig = await fs.readFile(state.configPath, "utf8"); + expect(rawConfig.includes(openAiApiKey)).toBe(false); + const config = JSON.parse(rawConfig) as { + agents?: { defaults?: { model?: { primary?: string } } }; + gateway?: { mode?: string }; + }; + expect(config.agents?.defaults?.model?.primary).toBe("openai/gpt-5.6"); + expect(config.gateway?.mode).toBe("local"); + assertOpenAiEnvProfile(state.agentDir()); + } + + await expect(fs.access(path.join(state.agentDir(), "auth-profiles.json"))).rejects.toThrow(); + + const stdout = await runOpenClaw( + [ + "agent", + "--local", + "--agent", + "main", + "--session-id", + "openai-onboarding-live-default", + "--message", + `Return exactly ${replyMarker} and no other text.`, + "--thinking", + "off", + "--json", + ], + state.env, + ); + expect( + extractAgentReplyTexts(stdout).some((reply) => reply.includes(replyMarker)), + `default OpenAI agent turn returned ${summarizeAgentOutput(stdout)}`, + ).toBe(true); + assertOpenAiEnvProfile(state.agentDir()); + } finally { + await state.cleanup(); + } + }, 300_000); +}); diff --git a/test/scripts/test-live-shard.test.ts b/test/scripts/test-live-shard.test.ts index 7fa2aa30453d..4cdf668d9c08 100644 --- a/test/scripts/test-live-shard.test.ts +++ b/test/scripts/test-live-shard.test.ts @@ -116,6 +116,7 @@ describe("scripts/test-live-shard", () => { expect(selectLiveShardFiles("native-live-test", allFiles)).toEqual([ "test/image-generation.infer-cli.live.test.ts", "test/image-generation.runtime.live.test.ts", + "test/openai-onboarding.live.test.ts", ]); expect(selectLiveShardFiles("native-live-extensions-media", allFiles)).toEqual([ "extensions/minimax/minimax.live.test.ts",