fix(openai): harden live and packaged onboarding (#114288)

This commit is contained in:
Peter Steinberger
2026-07-27 00:50:49 -04:00
committed by GitHub
parent 71eb9a3ec9
commit 60aac74672
12 changed files with 477 additions and 138 deletions
+62 -16
View File
@@ -537,7 +537,7 @@ describe("buildOpenAIProvider", () => {
contextWindow: 1_050_000,
maxTokens: 128_000,
reasoning: true,
input: ["text"],
input: ["text", "image"],
cost: { input: 30, output: 180, cacheRead: 0, cacheWrite: 0 },
});
expect(provider.models.find((model) => model.id === "gpt-5.4-mini")).toMatchObject({
@@ -583,6 +583,31 @@ describe("buildOpenAIProvider", () => {
expect(provider.models.map((model) => model.id)).toEqual(["gpt-5.5"]);
});
it.each([
["returns an empty model list", () => Response.json({ data: [] })],
[
"returns only unsupported models",
() => Response.json({ data: [{ id: "not-in-manifest", object: "model" }] }),
],
["rejects the API key", () => new Response("unauthorized", { status: 401 })],
["denies account access", () => new Response("forbidden", { status: 403 })],
])("does not invent available OpenAI models when discovery %s", async (_label, response) => {
const release = vi.fn(async () => undefined);
const fetchGuard: LiveModelCatalogFetchGuard = vi.fn(async () => ({
response: response(),
finalUrl: "https://api.openai.com/v1/models",
release,
}));
const provider = await buildOpenAILiveProviderConfig({
apiKey: "sk-openai-unavailable",
fetchGuard,
});
expect(provider.models).toEqual([]);
expect(release).toHaveBeenCalledOnce();
});
it("keeps only manifest fallback models when OpenAI discovery is unavailable", async () => {
const fetchGuard: LiveModelCatalogFetchGuard = vi.fn(async () => ({
response: new Response("temporarily unavailable", { status: 503 }),
@@ -1002,23 +1027,10 @@ describe("buildOpenAIProvider", () => {
).not.toContainEqual({ id: "ultra" });
});
it.each([
["fails", () => new Response("temporarily unavailable", { status: 503 })],
["returns no models", () => Response.json({ models: [] })],
[
"returns hidden-only models",
() =>
Response.json({
models: [
{ slug: "gpt-5.6-sol", display_name: "GPT-5.6 Sol", visibility: "hide" },
{ slug: "gpt-5.5", display_name: "GPT-5.5", show_in_picker: false },
],
}),
],
])("keeps static OpenAI OAuth rows when Codex catalog discovery %s", async (_label, response) => {
it("keeps static OpenAI OAuth rows when Codex catalog discovery fails", async () => {
const release = vi.fn(async () => undefined);
const fetchGuard: LiveModelCatalogFetchGuard = vi.fn(async () => ({
response: response(),
response: new Response("temporarily unavailable", { status: 503 }),
finalUrl: "https://chatgpt.com/backend-api/codex/models?client_version=1.0.0",
release,
}));
@@ -1048,6 +1060,40 @@ describe("buildOpenAIProvider", () => {
expect(release).toHaveBeenCalledOnce();
});
it.each([
["returns no models", () => Response.json({ models: [] })],
[
"returns only hidden models",
() =>
Response.json({
models: [
{ slug: "gpt-5.6-sol", display_name: "GPT-5.6 Sol", visibility: "hide" },
{ slug: "gpt-5.5", display_name: "GPT-5.5", show_in_picker: false },
],
}),
],
["rejects the subscription token", () => new Response("unauthorized", { status: 401 })],
["denies account access", () => new Response("forbidden", { status: 403 })],
])("does not invent OAuth models when the account catalog %s", async (_label, response) => {
const release = vi.fn(async () => undefined);
const fetchGuard: LiveModelCatalogFetchGuard = vi.fn(async () => ({
response: response(),
finalUrl: "https://chatgpt.com/backend-api/codex/models?client_version=1.0.0",
release,
}));
const provider = await buildOpenAICodexLiveProviderConfig({
discoveryApiKey: "oauth-token-no-visible-models",
accountId: "acct-openai-workspace",
fetchGuard,
});
expect(provider.api).toBe("openai-chatgpt-responses");
expect(provider.auth).toBe("oauth");
expect(provider.models).toEqual([]);
expect(release).toHaveBeenCalledOnce();
});
it("keeps the deprecated Codex provider builder on the public API barrel", async () => {
const { buildOpenAICodexProviderPlugin } = await import("./api.js");
const provider = buildOpenAICodexProviderPlugin();
+70 -62
View File
@@ -5,8 +5,8 @@ import type {
} from "openclaw/plugin-sdk/plugin-entry";
import { createProviderApiKeyAuthMethod } from "openclaw/plugin-sdk/provider-auth-api-key";
import {
buildLiveModelProviderConfig,
getCachedLiveProviderModelRows,
LiveModelCatalogHttpError,
type LiveModelCatalogFetchGuard,
} from "openclaw/plugin-sdk/provider-catalog-live-runtime";
import { buildManifestModelProviderConfig } from "openclaw/plugin-sdk/provider-catalog-shared";
@@ -234,7 +234,7 @@ function buildOpenAIDiscoverablePlatformModels(baseUrl: string): ModelDefinition
contextWindow,
api: "openai-responses",
baseUrl,
input: id === OPENAI_GPT_54_PRO_MODEL_ID ? ["text"] : ["text", "image"],
input: ["text", "image"],
maxTokens: OPENAI_GPT_54_MAX_TOKENS,
}));
}
@@ -245,56 +245,61 @@ async function buildOpenAILiveProviderConfig(
const baseUrl =
normalizeOptionalString(params.baseUrl) ?? resolveOpenAIDefaultBaseUrl(params.env);
const models = buildOpenAIManifestModelsForBaseUrl(baseUrl);
if (!shouldFetchOpenAILiveModels(baseUrl)) {
return {
baseUrl,
api: "openai-responses",
apiKey: params.apiKey,
models,
};
}
return await buildLiveModelProviderConfig({
providerId: PROVIDER_ID,
endpoint: OPENAI_MODELS_ENDPOINT,
providerConfig: {
baseUrl,
api: "openai-responses",
},
const fallback: ModelProviderConfig = {
baseUrl,
api: "openai-responses",
...(params.apiKey ? { apiKey: params.apiKey } : {}),
models,
projectRows: (rows, fallback) => {
const discoveredIds = new Set(
rows.flatMap((row) => {
if (!row || typeof row !== "object" || Array.isArray(row)) {
return [];
}
const candidate = row as { id?: unknown; object?: unknown };
if (candidate.object !== undefined && candidate.object !== "model") {
return [];
}
const modelId = typeof candidate.id === "string" ? candidate.id.trim() : "";
return modelId ? [modelId] : [];
}),
);
const selectedIds = new Set<string>();
// Discovery alone confirms account access; leave the manifest as the
// advisory fallback when OpenAI cannot return an authenticated catalog.
return [...fallback.models, ...buildOpenAIDiscoverablePlatformModels(baseUrl)].filter(
(model) => {
if (!discoveredIds.has(model.id) || selectedIds.has(model.id)) {
return false;
}
selectedIds.add(model.id);
return true;
},
);
},
apiKey: params.apiKey,
discoveryApiKey: params.discoveryApiKey,
fetchGuard: params.fetchGuard,
signal: params.signal,
ttlMs: OPENAI_MODELS_CACHE_TTL_MS,
auditContext: "openai-model-discovery",
});
};
if (!shouldFetchOpenAILiveModels(baseUrl)) {
return fallback;
}
try {
const rows = await getCachedLiveProviderModelRows({
providerId: PROVIDER_ID,
endpoint: OPENAI_MODELS_ENDPOINT,
apiKey: params.apiKey,
discoveryApiKey: params.discoveryApiKey,
fetchGuard: params.fetchGuard,
signal: params.signal,
ttlMs: OPENAI_MODELS_CACHE_TTL_MS,
auditContext: "openai-model-discovery",
});
const discoveredIds = new Set(
rows.flatMap((row) => {
if (!row || typeof row !== "object" || Array.isArray(row)) {
return [];
}
const candidate = row as { id?: unknown; object?: unknown };
if (candidate.object !== undefined && candidate.object !== "model") {
return [];
}
const modelId = typeof candidate.id === "string" ? candidate.id.trim() : "";
return modelId ? [modelId] : [];
}),
);
const selectedIds = new Set<string>();
// A successful account catalog is authoritative even when it has no
// visible supported models; static rows cannot grant model access.
return {
...fallback,
models: [...models, ...buildOpenAIDiscoverablePlatformModels(baseUrl)].filter((model) => {
if (!discoveredIds.has(model.id) || selectedIds.has(model.id)) {
return false;
}
selectedIds.add(model.id);
return true;
}),
};
} catch (error) {
if (
error instanceof LiveModelCatalogHttpError &&
(error.status === 401 || error.status === 403)
) {
return { ...fallback, models: [] };
}
return fallback;
}
}
function readCodexModelString(row: unknown, key: string): string | undefined {
@@ -578,18 +583,21 @@ async function buildOpenAICodexLiveProviderConfig(params: {
const models = rows
.map(buildOpenAICodexModelFromLiveRow)
.filter((model): model is ModelDefinitionConfig => Boolean(model));
if (models.length > 0) {
// Successful Codex OAuth discovery is account-scoped and authoritative
// for the picker/list catalog. Do not merge static OpenAI fallback rows
// into a successful live catalog.
return {
baseUrl: OPENAI_CODEX_RESPONSES_BASE_URL,
api: "openai-chatgpt-responses",
auth: "oauth",
models,
};
// A successful account-scoped response is authoritative even when all
// rows are hidden; static hints must not invent subscription access.
return {
baseUrl: OPENAI_CODEX_RESPONSES_BASE_URL,
api: "openai-chatgpt-responses",
auth: "oauth",
models,
};
} catch (error) {
if (
error instanceof LiveModelCatalogHttpError &&
(error.status === 401 || error.status === 403)
) {
return { ...buildOpenAICodexStaticProviderConfig(), models: [] };
}
} catch {
// Codex/ChatGPT discovery is advisory. Static OpenAI rows stay available
// when OAuth refresh or the remote model list is unavailable.
}
+27 -16
View File
@@ -61,7 +61,10 @@ export function applyMockOpenAiModelConfig(cfg, params) {
...(params.includeImageDefaults
? {
imageModel: { primary: modelRef, timeoutMs: 30_000 },
imageGenerationModel: { primary: "openai/gpt-image-1", timeoutMs: 30_000 },
mediaModels: {
...cfg.agents?.defaults?.mediaModels,
image: { primary: "openai/gpt-image-1", timeoutMs: 30_000 },
},
}
: {}),
models: {
@@ -72,24 +75,32 @@ export function applyMockOpenAiModelConfig(cfg, params) {
},
},
},
...(Array.isArray(cfg.agents?.list)
...(cfg.agents?.entries
? {
list: cfg.agents.list.map((agent) => ({
...agent,
model: { ...agent.model, primary: modelRef },
models: {
...agent.models,
[modelRef]: {
...agent.models?.[modelRef],
agentRuntime: { id: "openclaw" },
params: {
...agent.models?.[modelRef]?.params,
transport: "sse",
openaiWsWarmup: false,
entries: Object.fromEntries(
Object.entries(cfg.agents.entries).map(([agentId, agent]) => [
agentId,
{
...agent,
model: {
...(typeof agent.model === "object" && agent.model !== null ? agent.model : {}),
primary: modelRef,
},
models: {
...agent.models,
[modelRef]: {
...agent.models?.[modelRef],
agentRuntime: { id: "openclaw" },
params: {
...agent.models?.[modelRef]?.params,
transport: "sse",
openaiWsWarmup: false,
},
},
},
},
},
})),
]),
),
}
: {}),
};
@@ -136,9 +136,7 @@ function assertMockModelConfig() {
const provider = cfg.models?.providers?.openai;
const defaultModel = cfg.agents?.defaults?.model?.primary;
const defaultRuntime = cfg.agents?.defaults?.models?.[expectedModelRef]?.agentRuntime?.id;
const agent = Array.isArray(cfg.agents?.list)
? (cfg.agents.list.find((entry) => entry?.id === "main") ?? cfg.agents.list[0])
: undefined;
const agent = cfg.agents?.entries?.main;
const agentModel = agent?.model?.primary;
const agentRuntime = agent?.models?.[expectedModelRef]?.agentRuntime?.id;
if (provider?.baseUrl !== expectedBaseUrl) {
@@ -160,12 +158,12 @@ function assertMockModelConfig() {
if (defaultRuntime !== "openclaw") {
throw new Error(`mock default runtime was not preserved; got ${defaultRuntime}`);
}
if (agent && agentModel !== expectedModelRef) {
if (agentModel !== expectedModelRef) {
throw new Error(
`mock agent model was not preserved; expected ${expectedModelRef}, got ${agentModel}`,
);
}
if (agent && agentRuntime !== "openclaw") {
if (agentRuntime !== "openclaw") {
throw new Error(`mock agent runtime was not preserved; got ${agentRuntime}`);
}
}
+1 -1
View File
@@ -131,7 +131,7 @@ stop_gateway() {
}
cleanup_wizard_case() {
exec 3>&- 2>/dev/null || true
{ exec 3>&-; } 2>/dev/null || true
openclaw_e2e_stop_process "${wizard_pid:-}"
stop_gateway "${gw_pid:-}"
rm -rf "${input_fifo_dir:-}"
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
set -Eeuo pipefail
trap "" PIPE
export TERM=xterm-256color
export NO_COLOR=1
@@ -33,7 +33,7 @@ mock_pid=""
wizard_pid=""
input_fifo_dir=""
cleanup() {
exec 3>&- 2>/dev/null || true
{ exec 3>&-; } 2>/dev/null || true
openclaw_e2e_stop_process "${wizard_pid:-}"
openclaw_e2e_stop_process "${mock_pid:-}"
if [ -n "${input_fifo_dir:-}" ]; then
@@ -51,9 +51,7 @@ dump_debug_logs() {
"$ONBOARD_LOG" \
"$OPENAI_LOG" \
"$MOCK_REQUEST_LOG" \
"$AGENT_LOG" \
"$OPENCLAW_CONFIG_PATH" \
"$HOME/.openclaw/agents/main/agent/auth-profiles.json"
"$AGENT_LOG"
}
trap 'status=$?; dump_debug_logs "$status"; exit "$status"' ERR
@@ -88,6 +86,7 @@ wait_for_log() {
}
openclaw_e2e_install_package "$INSTALL_LOG"
echo "Installed the OpenClaw package."
command -v openclaw >/dev/null
package_root="$(openclaw_e2e_package_root)"
entry="$(openclaw_e2e_package_entrypoint "$package_root")"
@@ -95,11 +94,12 @@ openclaw_e2e_enable_openclaw_cli_timeout
mock_pid="$(openclaw_e2e_start_mock_openai "$MOCK_PORT" "$OPENAI_LOG")"
openclaw_e2e_wait_mock_openai "$MOCK_PORT"
echo "Mock OpenAI provider is ready."
input_fifo_dir="$(mktemp -d "$scenario_tmp/input.XXXXXX")"
input_fifo="$input_fifo_dir/stdin.fifo"
mkfifo "$input_fifo"
openclaw_e2e_run_script_with_pty "node \"$entry\" onboard --flow quickstart --mode local --auth-choice skip --gateway-port \"$PORT\" --gateway-bind loopback --skip-daemon --skip-ui --skip-channels --skip-skills --skip-health" "$ONBOARD_LOG" <"$input_fifo" >/dev/null 2>&1 &
openclaw_e2e_run_script_with_pty "node \"$entry\" onboard --flow quickstart --mode local --auth-choice skip --gateway-port \"$PORT\" --gateway-bind loopback --skip-daemon --skip-ui --skip-channels --skip-skills --skip-health --suppress-gateway-token-output" "$ONBOARD_LOG" <"$input_fifo" >/dev/null 2>&1 &
wizard_pid="$!"
exec 3>"$input_fifo"
@@ -113,6 +113,7 @@ wizard_pid=""
exec 3>&-
rm -rf "$input_fifo_dir"
input_fifo_dir=""
echo "Interactive typed onboarding completed."
node scripts/e2e/lib/release-scenarios/assertions.mjs assert-session-memory-hook-enabled
@@ -129,17 +130,22 @@ openclaw onboard \
--skip-ui \
--skip-channels \
--skip-skills \
--skip-health >>"$ONBOARD_LOG" 2>&1
--skip-health \
--suppress-gateway-token-output >>"$ONBOARD_LOG" 2>&1
node scripts/e2e/lib/release-scenarios/assertions.mjs assert-openai-env-ref "$OPENAI_API_KEY"
echo "OpenAI environment-reference onboarding completed."
node scripts/e2e/lib/release-scenarios/assertions.mjs configure-mock-openai "$MOCK_PORT"
openclaw agent --local \
if ! openclaw agent --local \
--agent main \
--session-id release-typed-onboarding-agent \
--message "Return marker $SUCCESS_MARKER" \
--thinking off \
--json >"$AGENT_LOG" 2>&1
--json >"$AGENT_LOG" 2>&1; then
dump_debug_logs 1
exit 1
fi
node scripts/e2e/lib/release-scenarios/assertions.mjs assert-agent-turn "$SUCCESS_MARKER" "$AGENT_LOG" "$MOCK_REQUEST_LOG"
echo "Release typed onboarding scenario passed."
+15 -10
View File
@@ -121,6 +121,19 @@ function serviceLane(name, command, options = {}) {
});
}
function releaseTypedOnboardingLane() {
return npmLane(
"release-typed-onboarding",
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:release-typed-onboarding",
{
resources: ["npm", "service"],
stateScenario: "empty",
timeoutMs: 20 * 60 * 1000,
weight: 3,
},
);
}
function createPackageUpdateMaintenanceLanes() {
return [
npmLane("doctor-switch", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:doctor-switch", {
@@ -410,16 +423,7 @@ export const mainLanes = [
weight: 4,
},
),
npmLane(
"release-typed-onboarding",
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:release-typed-onboarding",
{
resources: ["npm", "service"],
stateScenario: "empty",
timeoutMs: 20 * 60 * 1000,
weight: 3,
},
),
releaseTypedOnboardingLane(),
npmLane(
"release-media-memory",
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:release-media-memory",
@@ -789,6 +793,7 @@ const releasePathPackageInstallOpenAiLanes = [
timeoutMs: 30 * 60 * 1000,
weight: 3,
}),
releaseTypedOnboardingLane(),
];
const releasePathPackageInstallAnthropicLanes = [
+123 -7
View File
@@ -1,13 +1,17 @@
// Real-key onboarding must persist an env reference and complete the default first turn.
import { execFile } from "node:child_process";
import { execFile, spawn, type ChildProcess } from "node:child_process";
import { once } from "node:events";
import fs from "node:fs/promises";
import path from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import { promisify } from "node:util";
import { describe, expect, it } from "vitest";
import { extractAgentReplyTexts } from "../scripts/e2e/lib/agent-turn-output.mjs";
import { terminateManagedChild } from "../scripts/lib/managed-child-process.mjs";
import { readPersistedAuthProfileStoreRaw } from "../src/agents/auth-profiles/sqlite.js";
import { isLiveTestEnabled } from "../src/agents/live-test-helpers.js";
import { createOpenClawTestState } from "../src/test-utils/openclaw-test-state.js";
import { getDeterministicFreePortBlock } from "../src/test-utils/ports.js";
const execFileAsync = promisify(execFile);
const openAiApiKey = process.env.OPENAI_API_KEY?.trim() ?? "";
@@ -39,9 +43,7 @@ function assertOpenAiEnvProfile(agentDir: string): void {
// Assert on booleans before inspecting the profile so a broken inline-key
// migration can never echo a real live credential in Vitest diagnostics.
expect(JSON.stringify(store).includes(openAiApiKey)).toBe(false);
const profile = Object.values(store?.profiles ?? {}).find(
(candidate) => candidate.type === "api_key" && candidate.provider === "openai",
);
const profile = store?.profiles?.["openai:api-key"];
const keyRef = profile?.keyRef as
| { source?: unknown; provider?: unknown; id?: unknown }
| undefined;
@@ -87,11 +89,59 @@ function summarizeAgentOutput(stdout: string): string {
}
}
async function waitForIsolatedGatewayReady(gateway: ChildProcess, port: number): Promise<void> {
const deadline = Date.now() + 60_000;
let startupError = false;
gateway.once("error", () => {
startupError = true;
});
while (Date.now() < deadline) {
if (startupError || gateway.exitCode !== null || gateway.signalCode !== null) {
throw new Error("isolated onboarding gateway exited before becoming ready");
}
try {
const response = await fetch(`http://127.0.0.1:${port}/readyz`, {
signal: AbortSignal.timeout(1_000),
});
if (response.ok) {
const readiness = (await response.json()) as {
ready?: boolean;
failing?: unknown[];
};
if (readiness.ready === true && (readiness.failing?.length ?? 0) === 0) {
return;
}
}
} catch {
// Startup is complete only when the owned gateway reports real readiness.
}
await delay(250);
}
throw new Error("isolated onboarding gateway did not report ready");
}
async function stopIsolatedGateway(gateway: ChildProcess | undefined): Promise<void> {
if (!gateway || gateway.exitCode !== null || gateway.signalCode !== null) {
return;
}
const exited = once(gateway, "exit").then(() => true);
terminateManagedChild(gateway);
if (!(await Promise.race([exited, delay(5_000, false)]))) {
terminateManagedChild(gateway, "SIGKILL");
await Promise.race([exited, delay(5_000)]);
}
}
describeLive("fresh OpenAI onboarding live", () => {
it("keeps repeated onboarding secret-safe and runs the actual default model", async () => {
const state = await createOpenClawTestState({
label: "openai-onboarding-live",
layout: "state-only",
layout: "home",
scenario: "empty",
applyEnv: false,
// CLI children must take the production path, not inherit Vitest-only
@@ -109,10 +159,19 @@ describeLive("fresh OpenAI onboarding live", () => {
OPENCLAW_DISABLE_BUNDLED_PLUGINS: undefined,
OPENCLAW_PLUGIN_CATALOG_PATHS: undefined,
OPENCLAW_PLUGINS_PATHS: undefined,
OPENCLAW_WORKSPACE_DIR: undefined,
OPENCLAW_PROFILE: undefined,
OPENCLAW_GATEWAY_TOKEN: undefined,
OPENCLAW_GATEWAY_PASSWORD: undefined,
OPENCLAW_GATEWAY_URL: undefined,
OPENCLAW_GATEWAY_PORT: undefined,
},
});
let gateway: ChildProcess | undefined;
try {
const gatewayPort = await getDeterministicFreePortBlock({ offsets: [0, 1, 2, 4] });
expect(state.env.HOME).toBe(state.home);
await expect(fs.access(state.configPath)).rejects.toThrow();
const onboardArgs = [
"onboard",
@@ -126,6 +185,8 @@ describeLive("fresh OpenAI onboarding live", () => {
"ref",
"--gateway-bind",
"loopback",
"--gateway-port",
String(gatewayPort),
"--skip-daemon",
"--skip-ui",
"--skip-skills",
@@ -134,16 +195,28 @@ describeLive("fresh OpenAI onboarding live", () => {
"--json",
];
let firstGatewayToken: string | undefined;
for (let attempt = 0; attempt < 2; attempt += 1) {
await runOpenClaw(onboardArgs, state.env);
const rawConfig = await fs.readFile(state.configPath, "utf8");
expect(rawConfig.includes(openAiApiKey)).toBe(false);
const config = JSON.parse(rawConfig) as {
agents?: { defaults?: { model?: { primary?: string } } };
gateway?: { mode?: string };
agents?: { defaults?: { model?: { primary?: string }; workspace?: string } };
gateway?: { mode?: string; auth?: { mode?: string; token?: string } };
};
expect(config.agents?.defaults?.model?.primary).toBe("openai/gpt-5.6");
expect(config.agents?.defaults?.workspace).toBe(
path.join(state.home, ".openclaw", "workspace"),
);
expect(config.gateway?.mode).toBe("local");
expect(config.gateway?.auth?.mode).toBe("token");
const gatewayToken = config.gateway?.auth?.token;
expect(typeof gatewayToken === "string" && gatewayToken.length > 0).toBe(true);
if (firstGatewayToken === undefined) {
firstGatewayToken = gatewayToken;
} else {
expect(gatewayToken === firstGatewayToken).toBe(true);
}
assertOpenAiEnvProfile(state.agentDir());
}
@@ -170,7 +243,50 @@ describeLive("fresh OpenAI onboarding live", () => {
`default OpenAI agent turn returned ${summarizeAgentOutput(stdout)}`,
).toBe(true);
assertOpenAiEnvProfile(state.agentDir());
gateway = spawn(
process.execPath,
[
"scripts/run-node.mjs",
"gateway",
"run",
"--bind",
"loopback",
"--port",
String(gatewayPort),
],
{
cwd: path.resolve(import.meta.dirname, ".."),
detached: process.platform !== "win32",
env: state.env,
stdio: "ignore",
},
);
await waitForIsolatedGatewayReady(gateway, gatewayPort);
await runOpenClaw(["health", "--json"], state.env);
const gatewayStdout = await runOpenClaw(
[
"agent",
"--agent",
"main",
"--session-id",
"openai-onboarding-live-gateway",
"--message",
`Return exactly ${replyMarker} and no other text.`,
"--thinking",
"off",
"--json",
],
state.env,
);
expect(
extractAgentReplyTexts(gatewayStdout).some((reply) => reply.includes(replyMarker)),
`gateway-backed OpenAI agent turn returned ${summarizeAgentOutput(gatewayStdout)}`,
).toBe(true);
assertOpenAiEnvProfile(state.agentDir());
} finally {
await stopIsolatedGateway(gateway);
await state.cleanup();
}
}, 300_000);
+10
View File
@@ -2331,6 +2331,16 @@ grep -qx -- "OPENCLAW_E2E_COMMAND_TIMEOUT=23s" "$TMPDIR/package-args"
);
});
it("preserves actionable, secret-safe typed onboarding failure diagnostics", () => {
const script = readFileSync(RELEASE_TYPED_ONBOARDING_SCENARIO_PATH, "utf8");
expect(script).toContain("set -Eeuo pipefail");
expect(script).toContain("{ exec 3>&-; } 2>/dev/null || true");
expect(script).toContain("--suppress-gateway-token-output");
expect(script).not.toContain("exec 3>&- 2>/dev/null || true");
expect(script).not.toContain('"$HOME/.openclaw/agents/main/agent/auth-profiles.json"');
});
it("keeps append-only mock E2E state under per-run scratch roots", () => {
const scripts = [
{
+20
View File
@@ -277,6 +277,7 @@ describe("scripts/lib/docker-e2e-plan", () => {
expect(laneNames).toContain("install-e2e-openai");
expect(laneNames).toContain("openai-chat-tools");
expect(laneNames).toContain("live-codex-npm-plugin");
expect(laneNames).toContain("release-typed-onboarding");
expect(laneNames).toContain("install-e2e-anthropic");
expect(laneNames).toContain("update-channel-switch");
expect(laneNames).not.toContain("plugins");
@@ -310,6 +311,7 @@ describe("scripts/lib/docker-e2e-plan", () => {
expect(laneNames).not.toContain("live-codex-npm-plugin");
expect(laneNames).not.toContain("install-e2e-anthropic");
expect(laneNames).toContain("codex-on-demand");
expect(laneNames).toContain("release-typed-onboarding");
expect(laneNames).toContain("update-channel-switch");
});
@@ -396,6 +398,23 @@ describe("scripts/lib/docker-e2e-plan", () => {
"openai-chat-tools",
"live-codex-npm-plugin",
"codex-on-demand",
"release-typed-onboarding",
]);
expect(
packageInstallOpenAi.lanes
.filter((lane) => lane.name === "release-typed-onboarding")
.map(summarizeLane),
).toEqual([
{
command: "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:release-typed-onboarding",
imageKind: "bare",
live: false,
name: "release-typed-onboarding",
resources: ["docker", "npm", "service"],
stateScenario: "empty",
timeoutMs: 1_200_000,
weight: 3,
},
]);
expect(packageInstallAnthropic.lanes.map((lane) => lane.name)).toEqual([
"install-e2e-anthropic",
@@ -656,6 +675,7 @@ describe("scripts/lib/docker-e2e-plan", () => {
"openai-chat-tools",
"live-codex-npm-plugin",
"codex-on-demand",
"release-typed-onboarding",
"install-e2e-anthropic",
"npm-onboard-channel-agent",
"npm-onboard-discord-channel-agent",
+51 -12
View File
@@ -8,16 +8,19 @@ import {
type MockConfig = {
agents: {
defaults: {
imageGenerationModel?: unknown;
imageModel?: unknown;
mediaModels?: { image?: unknown; video?: unknown };
model?: unknown;
models: Record<string, unknown>;
};
list: Array<{
id: string;
model: { primary: string };
models: Record<string, { agentRuntime?: unknown; params: Record<string, unknown> }>;
}>;
entries: Record<
string,
{
model?: string | { primary: string };
models?: Record<string, { agentRuntime?: unknown; params: Record<string, unknown> }>;
workspace?: string;
}
>;
};
models: {
providers: {
@@ -49,21 +52,22 @@ describe("scripts/e2e/lib/fixtures/mock-openai-config.mjs", () => {
const cfg: MockConfig = {
agents: {
defaults: {
mediaModels: { video: { primary: "example/video" } },
models: {
"openai/gpt-5.4": { params: { preserved: true } },
},
},
list: [
{
id: "release-agent",
entries: {
"release-agent": {
model: { primary: "openai/gpt-5.4" },
workspace: "/tmp/release-agent",
models: {
"openai/gpt-5.5": {
params: { existing: true },
},
},
},
],
},
},
models: {
providers: {
@@ -95,8 +99,11 @@ describe("scripts/e2e/lib/fixtures/mock-openai-config.mjs", () => {
}),
]);
expect(cfg.agents.defaults).toMatchObject({
imageGenerationModel: { primary: "openai/gpt-image-1", timeoutMs: 30_000 },
imageModel: { primary: "openai/gpt-5.5", timeoutMs: 30_000 },
mediaModels: {
image: { primary: "openai/gpt-image-1", timeoutMs: 30_000 },
video: { primary: "example/video" },
},
model: { primary: "openai/gpt-5.5" },
models: {
"openai/gpt-5.4": { params: { preserved: true } },
@@ -106,14 +113,46 @@ describe("scripts/e2e/lib/fixtures/mock-openai-config.mjs", () => {
},
},
});
expect(cfg.agents.list[0]).toMatchObject({
expect(cfg.agents.defaults).not.toHaveProperty("imageGenerationModel");
expect(cfg.agents.entries["release-agent"]).toMatchObject({
model: { primary: "openai/gpt-5.5" },
workspace: "/tmp/release-agent",
models: {
"openai/gpt-5.5": {
agentRuntime: { id: "openclaw" },
params: { existing: true, transport: "sse", openaiWsWarmup: false },
},
},
});
expect(cfg.plugins).toEqual({ enabled: true });
});
it.each([
["string model", "openai/gpt-5.4"],
["missing model", undefined],
])("rewrites a canonical agent with a %s", (_label, model) => {
const cfg = {
agents: {
defaults: { models: {} },
entries: {
main: {
...(model === undefined ? {} : { model }),
},
},
},
models: { providers: {} },
};
applyMockOpenAiModelConfig(cfg, { mockPort: 18181 });
expect(cfg.agents.entries.main).toEqual({
model: { primary: "openai/gpt-5.6-luna" },
models: {
"openai/gpt-5.6-luna": {
agentRuntime: { id: "openclaw" },
params: { transport: "sse", openaiWsWarmup: false },
},
},
});
});
});
@@ -137,6 +137,86 @@ describe("npm onboard channel agent assertions", () => {
}
});
it("configures and validates the canonical main agent's mock model", () => {
const home = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-onboard-mock-agent-"));
const configPath = path.join(home, ".openclaw", "openclaw.json");
try {
fs.mkdirSync(path.dirname(configPath), { recursive: true });
fs.writeFileSync(
configPath,
JSON.stringify({
agents: {
defaults: { models: {} },
entries: { main: { default: true, model: "openai/gpt-5.6" } },
},
models: { providers: {} },
}),
);
expect(runMockModelAssert(home, "configure-mock-model", "18181").status).toBe(0);
expect(runMockModelAssert(home, "assert-mock-model-config", "18181").status).toBe(0);
const cfg = JSON.parse(fs.readFileSync(configPath, "utf8")) as {
agents: {
entries: Record<
string,
{
default?: boolean;
model?: { primary?: string };
models?: Record<string, { agentRuntime?: { id?: string } }>;
}
>;
};
};
expect(cfg.agents.entries.main).toMatchObject({
default: true,
model: { primary: "openai/gpt-5.6-luna" },
models: {
"openai/gpt-5.6-luna": { agentRuntime: { id: "openclaw" } },
},
});
} finally {
fs.rmSync(home, { force: true, recursive: true });
}
});
it("rejects a canonical main agent that does not use the configured mock model", () => {
const home = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-onboard-mock-agent-"));
const configPath = path.join(home, ".openclaw", "openclaw.json");
try {
fs.mkdirSync(path.dirname(configPath), { recursive: true });
fs.writeFileSync(
configPath,
JSON.stringify({
agents: {
defaults: { models: {} },
entries: { main: { default: true, model: "openai/gpt-5.6" } },
},
models: { providers: {} },
}),
);
expect(runMockModelAssert(home, "configure-mock-model", "18181").status).toBe(0);
const cfg = JSON.parse(fs.readFileSync(configPath, "utf8")) as {
agents: { entries: Record<string, { model?: { primary?: string } }> };
};
const mainAgent = cfg.agents.entries.main;
if (!mainAgent) {
throw new Error("mock configuration did not contain the main agent");
}
mainAgent.model = { primary: "openai/gpt-5.6" };
fs.writeFileSync(configPath, JSON.stringify(cfg));
const result = runMockModelAssert(home, "assert-mock-model-config", "18181");
expect(result.status).not.toBe(0);
expect(result.stderr).toContain("mock agent model was not preserved");
} finally {
fs.rmSync(home, { force: true, recursive: true });
}
});
it("validates OpenAI env refs from the SQLite auth profile store", () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-onboard-assertions-"));
const agentDir = path.join(tempDir, ".openclaw", "agents", "main", "agent");