From 6930538500edbae0fb6ef19164a67edbfa62c900 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 26 May 2026 18:51:23 +0100 Subject: [PATCH] ci: require codex profiles for live probes --- src/agents/live-test-helpers.test.ts | 21 ++++++++++++++++- src/agents/live-test-helpers.ts | 16 +++++++++++++ src/agents/models.profiles.live.test.ts | 18 +++++++++++---- .../openai-reasoning-compat.live.test.ts | 23 +++++++++++++++---- src/agents/tool-replay-repair.live.test.ts | 23 +++++++++++++++---- 5 files changed, 86 insertions(+), 15 deletions(-) diff --git a/src/agents/live-test-helpers.test.ts b/src/agents/live-test-helpers.test.ts index 8b209bbc8ffe..85d58872bb5a 100644 --- a/src/agents/live-test-helpers.test.ts +++ b/src/agents/live-test-helpers.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from "vitest"; -import { isLiveProfileKeyModeEnabled, isLiveTestEnabled } from "./live-test-helpers.js"; +import { + isLiveProfileKeyModeEnabled, + isLiveTestEnabled, + requiresLiveProfileCredential, + resolveLiveCredentialPrecedence, +} from "./live-test-helpers.js"; describe("isLiveTestEnabled", () => { it("treats LIVE and OPENCLAW_LIVE_TEST as shared live gates", () => { @@ -21,3 +26,17 @@ describe("isLiveProfileKeyModeEnabled", () => { expect(isLiveProfileKeyModeEnabled({ LIVE: "1" })).toBe(false); }); }); + +describe("live credential precedence", () => { + it("uses profile-first auth for Codex even when the global live mode is env-first", () => { + expect(resolveLiveCredentialPrecedence("openai-codex", false)).toBe("profile-first"); + expect(requiresLiveProfileCredential("openai-codex", false)).toBe(true); + }); + + it("keeps env-first auth for normal providers unless profile keys are required", () => { + expect(resolveLiveCredentialPrecedence("openai", false)).toBe("env-first"); + expect(resolveLiveCredentialPrecedence("openai", true)).toBe("profile-first"); + expect(requiresLiveProfileCredential("openai", false)).toBe(false); + expect(requiresLiveProfileCredential("openai", true)).toBe(true); + }); +}); diff --git a/src/agents/live-test-helpers.ts b/src/agents/live-test-helpers.ts index ef47bc93d3ed..260819b23dad 100644 --- a/src/agents/live-test-helpers.ts +++ b/src/agents/live-test-helpers.ts @@ -16,6 +16,22 @@ export function isLiveProfileKeyModeEnabled(env: NodeJS.ProcessEnv = process.env return isTruthyEnvValue(env.OPENCLAW_LIVE_REQUIRE_PROFILE_KEYS); } +export function requiresLiveProfileCredential( + provider: string, + requireProfileKeys: boolean, +): boolean { + return requireProfileKeys || provider === "openai-codex"; +} + +export function resolveLiveCredentialPrecedence( + provider: string, + requireProfileKeys: boolean, +): "profile-first" | "env-first" { + return requiresLiveProfileCredential(provider, requireProfileKeys) + ? "profile-first" + : "env-first"; +} + export function createSingleUserPromptMessage(content = LIVE_OK_PROMPT) { return [ { diff --git a/src/agents/models.profiles.live.test.ts b/src/agents/models.profiles.live.test.ts index fbcfc820f7df..dc508cbecfca 100644 --- a/src/agents/models.profiles.live.test.ts +++ b/src/agents/models.profiles.live.test.ts @@ -41,7 +41,12 @@ import { shouldSkipLiveModelImageProbe, } from "./live-model-turn-probes.js"; import { createLiveTargetMatcher } from "./live-target-matcher.js"; -import { isLiveProfileKeyModeEnabled, isLiveTestEnabled } from "./live-test-helpers.js"; +import { + isLiveProfileKeyModeEnabled, + isLiveTestEnabled, + requiresLiveProfileCredential, + resolveLiveCredentialPrecedence, +} from "./live-test-helpers.js"; import { isLiveBillingDrift, isLiveRateLimitDrift, @@ -60,7 +65,6 @@ import { const LIVE = isLiveTestEnabled(); const DIRECT_ENABLED = Boolean(process.env.OPENCLAW_LIVE_MODELS?.trim()); const REQUIRE_PROFILE_KEYS = isLiveProfileKeyModeEnabled(); -const LIVE_CREDENTIAL_PRECEDENCE = REQUIRE_PROFILE_KEYS ? "profile-first" : "env-first"; const LIVE_HEARTBEAT_MS = Math.max(1_000, toInt(process.env.OPENCLAW_LIVE_HEARTBEAT_MS, 30_000)); const LIVE_SETUP_TIMEOUT_MS = Math.max( 1_000, @@ -862,9 +866,15 @@ describeLive("live models (profile keys)", () => { const apiKeyInfo = await getApiKeyForModel({ model, cfg, - credentialPrecedence: LIVE_CREDENTIAL_PRECEDENCE, + credentialPrecedence: resolveLiveCredentialPrecedence( + model.provider, + REQUIRE_PROFILE_KEYS, + ), }); - if (REQUIRE_PROFILE_KEYS && !apiKeyInfo.source.startsWith("profile:")) { + if ( + requiresLiveProfileCredential(model.provider, REQUIRE_PROFILE_KEYS) && + !apiKeyInfo.source.startsWith("profile:") + ) { skipped.push({ model: id, reason: `non-profile credential source: ${apiKeyInfo.source}`, diff --git a/src/agents/openai-reasoning-compat.live.test.ts b/src/agents/openai-reasoning-compat.live.test.ts index f68d4eec3f5a..5df15e9bdd95 100644 --- a/src/agents/openai-reasoning-compat.live.test.ts +++ b/src/agents/openai-reasoning-compat.live.test.ts @@ -10,6 +10,8 @@ import { isLiveProfileKeyModeEnabled, isLiveTestEnabled, logLiveProgress, + requiresLiveProfileCredential, + resolveLiveCredentialPrecedence, } from "./live-test-helpers.js"; import { getApiKeyForModel, requireApiKey } from "./model-auth.js"; import { ensureOpenClawModelsJson } from "./models-config.js"; @@ -18,7 +20,6 @@ import { discoverAuthStorage, discoverModels } from "./pi-model-discovery.js"; const LIVE = isLiveTestEnabled(); const REQUIRE_PROFILE_KEYS = isLiveProfileKeyModeEnabled(); -const LIVE_CREDENTIAL_PRECEDENCE = REQUIRE_PROFILE_KEYS ? "profile-first" : "env-first"; const DEFAULT_TARGET_MODEL_REF = "openai-codex/gpt-5.1-codex-mini"; const TARGET_MODEL_REF = process.env.OPENCLAW_LIVE_OPENAI_REASONING_COMPAT_MODEL?.trim() || DEFAULT_TARGET_MODEL_REF; @@ -116,14 +117,20 @@ describeLive("openai reasoning compat live", () => { apiKeyInfo = await getApiKeyForModel({ model, cfg, - credentialPrecedence: LIVE_CREDENTIAL_PRECEDENCE, + credentialPrecedence: resolveLiveCredentialPrecedence( + model.provider, + REQUIRE_PROFILE_KEYS, + ), }); } catch (error) { logProgress(`[openai-reasoning-compat] skip (${String(error)})`); return; } - if (REQUIRE_PROFILE_KEYS && !apiKeyInfo.source.startsWith("profile:")) { + if ( + requiresLiveProfileCredential(model.provider, REQUIRE_PROFILE_KEYS) && + !apiKeyInfo.source.startsWith("profile:") + ) { logProgress( `[openai-reasoning-compat] skip (non-profile credential source: ${apiKeyInfo.source})`, ); @@ -170,14 +177,20 @@ describeLive("openai reasoning compat live", () => { apiKeyInfo = await getApiKeyForModel({ model, cfg, - credentialPrecedence: LIVE_CREDENTIAL_PRECEDENCE, + credentialPrecedence: resolveLiveCredentialPrecedence( + model.provider, + REQUIRE_PROFILE_KEYS, + ), }); } catch (error) { logProgress(`[openai-reasoning-compat] skip (${String(error)})`); return; } - if (REQUIRE_PROFILE_KEYS && !apiKeyInfo.source.startsWith("profile:")) { + if ( + requiresLiveProfileCredential(model.provider, REQUIRE_PROFILE_KEYS) && + !apiKeyInfo.source.startsWith("profile:") + ) { logProgress( `[openai-reasoning-compat] skip (non-profile credential source: ${apiKeyInfo.source})`, ); diff --git a/src/agents/tool-replay-repair.live.test.ts b/src/agents/tool-replay-repair.live.test.ts index 1efe8cc62305..d843cb1c175b 100644 --- a/src/agents/tool-replay-repair.live.test.ts +++ b/src/agents/tool-replay-repair.live.test.ts @@ -11,6 +11,8 @@ import { isLiveProfileKeyModeEnabled, isLiveTestEnabled, logLiveProgress, + requiresLiveProfileCredential, + resolveLiveCredentialPrecedence, } from "./live-test-helpers.js"; import { getApiKeyForModel, requireApiKey } from "./model-auth.js"; import { ensureOpenClawModelsJson } from "./models-config.js"; @@ -20,7 +22,6 @@ import { transformTransportMessages } from "./transport-message-transform.js"; const LIVE = isLiveTestEnabled(); const REQUIRE_PROFILE_KEYS = isLiveProfileKeyModeEnabled(); -const LIVE_CREDENTIAL_PRECEDENCE = REQUIRE_PROFILE_KEYS ? "profile-first" : "env-first"; const DEFAULT_TARGET_MODEL_REFS = "openai-codex/gpt-5.5,google/gemini-3-flash-preview"; const TARGET_MODEL_REFS = parseTargetModelRefs( process.env.OPENCLAW_LIVE_TOOL_REPLAY_REPAIR_MODELS ?? DEFAULT_TARGET_MODEL_REFS, @@ -214,14 +215,20 @@ describeLive("tool replay repair live", () => { apiKeyInfo = await getApiKeyForModel({ model, cfg, - credentialPrecedence: LIVE_CREDENTIAL_PRECEDENCE, + credentialPrecedence: resolveLiveCredentialPrecedence( + model.provider, + REQUIRE_PROFILE_KEYS, + ), }); } catch (error) { logProgress(`[tool-replay-repair] skip ${target.ref} (${String(error)})`); return; } - if (REQUIRE_PROFILE_KEYS && !apiKeyInfo.source.startsWith("profile:")) { + if ( + requiresLiveProfileCredential(model.provider, REQUIRE_PROFILE_KEYS) && + !apiKeyInfo.source.startsWith("profile:") + ) { logProgress( `[tool-replay-repair] skip ${target.ref} (non-profile credential source: ${apiKeyInfo.source})`, ); @@ -319,14 +326,20 @@ describeLive("tool replay repair live", () => { apiKeyInfo = await getApiKeyForModel({ model, cfg, - credentialPrecedence: LIVE_CREDENTIAL_PRECEDENCE, + credentialPrecedence: resolveLiveCredentialPrecedence( + model.provider, + REQUIRE_PROFILE_KEYS, + ), }); } catch (error) { logProgress(`[tool-replay-repair] skip ${target.ref} (${String(error)})`); return; } - if (REQUIRE_PROFILE_KEYS && !apiKeyInfo.source.startsWith("profile:")) { + if ( + requiresLiveProfileCredential(model.provider, REQUIRE_PROFILE_KEYS) && + !apiKeyInfo.source.startsWith("profile:") + ) { logProgress( `[tool-replay-repair] skip ${target.ref} (non-profile credential source: ${apiKeyInfo.source})`, );