diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index 1b5d99537a9e..1e42555caa9e 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -18,6 +18,7 @@ const { buildQaDockerHarnessImage, runQaDockerUp, defaultQaRuntimeModelForMode, + resolveQaRuntimeModelPair, readQaScenarioPack, } = vi.hoisted(() => ({ runQaManualLane: vi.fn(), @@ -32,6 +33,7 @@ const { runQaDockerUp: vi.fn(), defaultQaRuntimeModelForMode: vi.fn<(mode: string, options?: { alternate?: boolean }) => string>(), + resolveQaRuntimeModelPair: vi.fn(), readQaScenarioPack: vi.fn<() => QaScenarioPack>(), })); @@ -76,6 +78,7 @@ vi.mock("./docker-up.runtime.js", () => ({ vi.mock("./model-selection.runtime.js", () => ({ defaultQaRuntimeModelForMode, + resolveQaRuntimeModelPair, })); vi.mock("./scenario-catalog.js", async (importOriginal) => { @@ -106,12 +109,29 @@ import { QaSuiteInfraError } from "./errors.js"; import { QA_EVIDENCE_FILENAME } from "./evidence-summary.js"; import { runQaTelegramCommand } from "./live-transports/telegram/cli.runtime.js"; import { defaultQaModelForMode as defaultQaProviderModelForMode } from "./model-selection.js"; +import { resolveQaLiveFrontierAlternateModel } from "./providers/live-frontier/model-selection.runtime.js"; import type { QaTransportAdapterFactory } from "./qa-transport-registry.js"; import type { QaProviderModeInput } from "./run-config.js"; import { expandQaScenarioExecutionCells } from "./scenario-lane.js"; import type { QaSuiteRunParams } from "./suite.js"; const DEFAULT_LIVE_FRONTIER_MODEL = defaultQaProviderModelForMode("live-frontier"); +function resolveMockQaRuntimeModelPair(params: { + providerMode: string; + primaryModel?: string; + alternateModel?: string; +}) { + const primaryModel = + params.primaryModel?.trim() || defaultQaRuntimeModelForMode(params.providerMode); + const alternateModel = + params.alternateModel?.trim() || + (params.providerMode === "live-frontier" + ? (resolveQaLiveFrontierAlternateModel(primaryModel) ?? + defaultQaRuntimeModelForMode(params.providerMode, { alternate: true })) + : defaultQaRuntimeModelForMode(params.providerMode, { alternate: true })); + return { primaryModel, alternateModel }; +} + const QA_PASSING_SUITE_SCENARIO = { name: "channel chat baseline", status: "pass" as const, @@ -340,6 +360,7 @@ describe("qa cli runtime", () => { (mode: string, options?: { alternate?: boolean }) => defaultQaProviderModelForMode(mode as QaProviderModeInput, options), ); + resolveQaRuntimeModelPair.mockImplementation(resolveMockQaRuntimeModelPair); readQaScenarioPack.mockClear(); runQaSuite.mockImplementation(async (params) => { const observedCells = executionCellsForSuiteParams(params); @@ -3262,39 +3283,43 @@ describe("qa cli runtime", () => { transportId: "qa-channel", providerMode: "live-frontier", primaryModel: DEFAULT_LIVE_FRONTIER_MODEL, - alternateModel: DEFAULT_LIVE_FRONTIER_MODEL, + alternateModel: "openai/gpt-5.6-luna", fastMode: undefined, message: "read qa kickoff and reply short", timeoutMs: undefined, }); }); - it("keeps an explicit manual primary model as the alternate default", async () => { - await runQaManualLaneCommand({ - repoRoot: "/tmp/openclaw-repo", - providerMode: "live-frontier", - primaryModel: "anthropic/claude-sonnet-4-6", - message: "read qa kickoff and reply short", - }); + it.each(["anthropic/claude-sonnet-4-6", "openai/gpt-5.6-sol"])( + "keeps explicit manual primary %s single-model when the alternate is omitted", + async (primaryModel) => { + await runQaManualLaneCommand({ + repoRoot: "/tmp/openclaw-repo", + providerMode: "live-frontier", + primaryModel, + message: "read qa kickoff and reply short", + }); - expect(runQaManualLane).toHaveBeenCalledWith({ - repoRoot: path.resolve("/tmp/openclaw-repo"), - transportId: "qa-channel", - providerMode: "live-frontier", - primaryModel: "anthropic/claude-sonnet-4-6", - alternateModel: "anthropic/claude-sonnet-4-6", - fastMode: undefined, - message: "read qa kickoff and reply short", - timeoutMs: undefined, - }); - }); + expect(runQaManualLane).toHaveBeenCalledWith({ + repoRoot: path.resolve("/tmp/openclaw-repo"), + transportId: "qa-channel", + providerMode: "live-frontier", + primaryModel, + alternateModel: primaryModel, + fastMode: undefined, + message: "read qa kickoff and reply short", + timeoutMs: undefined, + }); + }, + ); it("defaults manual frontier runs onto Codex OAuth when the runtime resolver prefers it", async () => { - defaultQaRuntimeModelForMode.mockImplementation((mode, options) => - mode === "live-frontier" - ? "openai/gpt-5.6-luna" - : defaultQaProviderModelForMode(mode as QaProviderModeInput, options), - ); + defaultQaRuntimeModelForMode.mockImplementation((mode, options) => { + if (mode === "live-frontier" && !options?.alternate) { + return "openai/gpt-5.6-luna"; + } + return defaultQaProviderModelForMode(mode as QaProviderModeInput, options); + }); await runQaManualLaneCommand({ repoRoot: "/tmp/openclaw-repo", @@ -3306,7 +3331,7 @@ describe("qa cli runtime", () => { transportId: "qa-channel", providerMode: "live-frontier", primaryModel: "openai/gpt-5.6-luna", - alternateModel: "openai/gpt-5.6-luna", + alternateModel: "openai/gpt-5.6-sol", fastMode: undefined, message: "read qa kickoff and reply short", timeoutMs: undefined, diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts index bf886fdef9bf..993ffd724457 100644 --- a/extensions/qa-lab/src/cli.runtime.ts +++ b/extensions/qa-lab/src/cli.runtime.ts @@ -46,6 +46,7 @@ import { import { startQaLabServer } from "./lab-server.js"; import { listLiveTransportQaAdapterFactories } from "./live-transports/cli.js"; import { runQaManualLane } from "./manual-lane.runtime.js"; +import { resolveQaRuntimeModelPair } from "./model-selection.runtime.js"; import { runQaMultipass } from "./multipass.runtime.js"; import { qaProfileEvidencePlan, type QaProfileEvidencePlan } from "./profile-evidence-plan.js"; import { @@ -193,17 +194,14 @@ function resolveQaManualLaneModels(opts: { primaryModel?: string; alternateModel?: string; }) { - const primaryModel = opts.primaryModel?.trim() || defaultQaModelForMode(opts.providerMode); - const alternateModel = opts.alternateModel?.trim(); - return { - primaryModel, - alternateModel: - alternateModel && alternateModel.length > 0 - ? alternateModel - : opts.primaryModel?.trim() - ? primaryModel - : defaultQaModelForMode(opts.providerMode, true), - }; + // `qa manual --model` is a one-model probe unless the operator also supplies + // `--alt-model`; materialize that contract before shared pair resolution. + const explicitPrimaryModel = opts.primaryModel?.trim(); + return resolveQaRuntimeModelPair({ + ...opts, + primaryModel: explicitPrimaryModel, + alternateModel: opts.alternateModel?.trim() || explicitPrimaryModel, + }); } function parseQaThinkingLevel( diff --git a/extensions/qa-lab/src/gateway-child.test.ts b/extensions/qa-lab/src/gateway-child.test.ts index 3d26c45cf892..368313633f52 100644 --- a/extensions/qa-lab/src/gateway-child.test.ts +++ b/extensions/qa-lab/src/gateway-child.test.ts @@ -1084,15 +1084,19 @@ describe("buildQaRuntimeEnv", () => { expect(configProfile.provider).toBe("openai"); expect(configProfile.mode).toBe("api_key"); expect(configProfile.displayName).toBe("QA live openai env credential"); + expect(Object.values(cfg.auth?.profiles ?? {})).not.toContainEqual( + expect.objectContaining({ provider: "anthropic" }), + ); for (const agentId of ["main", "qa"]) { - const storeProfile = requireAuthProfile( - readAuthProfileStore(stateDir, agentId).profiles, - "qa-live-openai-env", - ); + const profiles = readAuthProfileStore(stateDir, agentId).profiles; + const storeProfile = requireAuthProfile(profiles, "qa-live-openai-env"); expect(storeProfile.type).toBe("api_key"); expect(storeProfile.provider).toBe("openai"); expect(storeProfile.key).toBe("qa-live-not-a-real-key"); + expect(Object.values(profiles)).not.toContainEqual( + expect.objectContaining({ provider: "anthropic" }), + ); } }); diff --git a/extensions/qa-lab/src/model-selection.runtime.test.ts b/extensions/qa-lab/src/model-selection.runtime.test.ts index 3a2ad9300bdd..ed3b7acfb117 100644 --- a/extensions/qa-lab/src/model-selection.runtime.test.ts +++ b/extensions/qa-lab/src/model-selection.runtime.test.ts @@ -18,7 +18,10 @@ vi.mock("openclaw/plugin-sdk/agent-runtime", () => ({ listProfilesForProvider, })); -import { defaultQaRuntimeModelForMode } from "./model-selection.runtime.js"; +import { + defaultQaRuntimeModelForMode, + resolveQaRuntimeModelPair, +} from "./model-selection.runtime.js"; describe("qa model selection runtime", () => { beforeEach(() => { @@ -34,26 +37,36 @@ describe("qa model selection runtime", () => { resolveEnvApiKey.mockReturnValue({ apiKey: "sk-test" }); expect(defaultQaRuntimeModelForMode("live-frontier")).toBe("openai/gpt-5.6"); + expect(resolveQaRuntimeModelPair({ providerMode: "live-frontier" })).toEqual({ + primaryModel: "openai/gpt-5.6", + alternateModel: "openai/gpt-5.6-luna", + }); expect(loadAuthProfileStoreForRuntime).not.toHaveBeenCalled(); }); - it("prefers the Codex OAuth live default when only Codex auth profiles are available", () => { - loadAuthProfileStoreForRuntime.mockReturnValue({ - profiles: { - "openai:user@example.com": { - provider: "openai", - type: "oauth", + it.each(["oauth", "token"] as const)( + "prefers the Codex live default for a stored %s profile", + (type) => { + loadAuthProfileStoreForRuntime.mockReturnValue({ + profiles: { + "openai:user@example.com": { + provider: "openai", + type, + }, }, - }, - }); + }); - expect(defaultQaRuntimeModelForMode("live-frontier")).toBe("openai/gpt-5.6-luna"); - expect(loadAuthProfileStoreForRuntime).toHaveBeenCalledWith(undefined, { - readOnly: true, - allowKeychainPrompt: false, - externalCliProviderIds: ["openai"], - }); - }); + expect(resolveQaRuntimeModelPair({ providerMode: "live-frontier" })).toEqual({ + primaryModel: "openai/gpt-5.6-luna", + alternateModel: "openai/gpt-5.6-sol", + }); + expect(loadAuthProfileStoreForRuntime).toHaveBeenCalledWith(undefined, { + readOnly: true, + allowKeychainPrompt: false, + externalCliProviderIds: ["openai"], + }); + }, + ); it("keeps the OpenAI live default when stored OpenAI profiles are available", () => { loadAuthProfileStoreForRuntime.mockReturnValue({ @@ -68,6 +81,66 @@ describe("qa model selection runtime", () => { expect(defaultQaRuntimeModelForMode("live-frontier")).toBe("openai/gpt-5.6"); }); + it.each(["openai/gpt-5.6", "openai/gpt-5.6-sol"])( + "derives Luna after explicit Sol primary %s", + (primaryModel) => { + expect(resolveQaRuntimeModelPair({ providerMode: "live-frontier", primaryModel })).toEqual({ + primaryModel, + alternateModel: "openai/gpt-5.6-luna", + }); + }, + ); + + it("derives Sol after an explicit Luna primary", () => { + expect( + resolveQaRuntimeModelPair({ + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.6-luna", + }), + ).toEqual({ + primaryModel: "openai/gpt-5.6-luna", + alternateModel: "openai/gpt-5.6-sol", + }); + }); + + it("falls back through the provider default for an unmapped primary", () => { + expect( + resolveQaRuntimeModelPair({ + providerMode: "live-frontier", + primaryModel: "anthropic/claude-sonnet-4-6", + }), + ).toEqual({ + primaryModel: "anthropic/claude-sonnet-4-6", + alternateModel: "openai/gpt-5.6", + }); + }); + + it("preserves an explicit alternate model", () => { + expect( + resolveQaRuntimeModelPair({ + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.6", + alternateModel: "openai/gpt-5.6-terra", + }), + ).toEqual({ + primaryModel: "openai/gpt-5.6", + alternateModel: "openai/gpt-5.6-terra", + }); + }); + + it.each([ + ["openai/gpt-5.4", "openai/gpt-5.4"], + ["openai/gpt-5.6", "openai/gpt-5.6-sol"], + ])("preserves the explicit model pair %s / %s", (primaryModel, alternateModel) => { + expect( + resolveQaRuntimeModelPair({ + providerMode: "live-frontier", + primaryModel, + alternateModel, + }), + ).toEqual({ primaryModel, alternateModel }); + }); + it("leaves mock defaults unchanged", () => { expect(defaultQaRuntimeModelForMode("mock-openai")).toBe("mock-openai/gpt-5.6-luna"); expect(defaultQaRuntimeModelForMode("mock-openai", { alternate: true })).toBe( diff --git a/extensions/qa-lab/src/model-selection.runtime.ts b/extensions/qa-lab/src/model-selection.runtime.ts index 1dd2bf3a2d5b..c91708b10a90 100644 --- a/extensions/qa-lab/src/model-selection.runtime.ts +++ b/extensions/qa-lab/src/model-selection.runtime.ts @@ -2,14 +2,14 @@ import { defaultQaModelForMode, normalizeQaProviderMode, + type QaProviderMode, type QaProviderModeInput, } from "./model-selection.js"; import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "./providers/index.js"; -import { resolveQaLiveFrontierPreferredModel } from "./providers/live-frontier/model-selection.runtime.js"; - -function resolveQaPreferredLiveModel() { - return resolveQaLiveFrontierPreferredModel(); -} +import { + resolveQaLiveFrontierAlternateModel, + resolveQaLiveFrontierPreferredModel, +} from "./providers/live-frontier/model-selection.runtime.js"; export function defaultQaRuntimeModelForMode( mode: QaProviderModeInput, @@ -21,10 +21,32 @@ export function defaultQaRuntimeModelForMode( const preferredLiveModel = options?.preferredLiveModel ?? (normalizeQaProviderMode(mode) === DEFAULT_QA_LIVE_PROVIDER_MODE - ? resolveQaPreferredLiveModel() + ? resolveQaLiveFrontierPreferredModel() : undefined); return defaultQaModelForMode(mode, { ...options, preferredLiveModel, }); } + +export function resolveQaRuntimeModelPair(params: { + providerMode: QaProviderModeInput; + primaryModel?: string; + alternateModel?: string; + resolveDefaultModel?: (mode: QaProviderMode, alternate?: boolean) => string; +}) { + const providerMode = normalizeQaProviderMode(params.providerMode); + const normalizeModel = (model: string | undefined) => model?.trim() || undefined; + const resolveDefaultModel = + params.resolveDefaultModel ?? + ((mode: QaProviderModeInput, alternate = false) => + defaultQaRuntimeModelForMode(mode, alternate ? { alternate: true } : undefined)); + const primaryModel = normalizeModel(params.primaryModel) ?? resolveDefaultModel(providerMode); + const alternateModel = + normalizeModel(params.alternateModel) ?? + (providerMode === DEFAULT_QA_LIVE_PROVIDER_MODE + ? (resolveQaLiveFrontierAlternateModel(primaryModel) ?? + resolveDefaultModel(providerMode, true)) + : resolveDefaultModel(providerMode, true)); + return { primaryModel, alternateModel }; +} diff --git a/extensions/qa-lab/src/providers/live-frontier/model-selection.runtime.ts b/extensions/qa-lab/src/providers/live-frontier/model-selection.runtime.ts index 2b819fa04bb9..d8adb7a0620e 100644 --- a/extensions/qa-lab/src/providers/live-frontier/model-selection.runtime.ts +++ b/extensions/qa-lab/src/providers/live-frontier/model-selection.runtime.ts @@ -7,6 +7,16 @@ import { resolveEnvApiKey } from "openclaw/plugin-sdk/provider-auth"; const QA_CODEX_OAUTH_LIVE_MODEL = "openai/gpt-5.6-luna"; +export function resolveQaLiveFrontierAlternateModel(primaryModel: string) { + const normalized = primaryModel.toLowerCase(); + if (normalized === QA_CODEX_OAUTH_LIVE_MODEL) { + return "openai/gpt-5.6-sol"; + } + return normalized === "openai/gpt-5.6" || normalized === "openai/gpt-5.6-sol" + ? QA_CODEX_OAUTH_LIVE_MODEL + : undefined; +} + export function resolveQaLiveFrontierPreferredModel() { if (resolveEnvApiKey("openai")?.apiKey) { return undefined; diff --git a/extensions/qa-lab/src/qa-gateway-config.test.ts b/extensions/qa-lab/src/qa-gateway-config.test.ts index b309211e63d3..f7ca14d43704 100644 --- a/extensions/qa-lab/src/qa-gateway-config.test.ts +++ b/extensions/qa-lab/src/qa-gateway-config.test.ts @@ -270,23 +270,45 @@ describe("buildQaGatewayConfig", () => { providerMode: "live-frontier", fastMode: true, primaryModel: "openai/gpt-5.6-luna", - alternateModel: "openai/gpt-5.6-luna", + alternateModel: "openai/gpt-5.6-sol", ...createQaChannelTransportParams(), }); expect(getPrimaryModel(cfg.agents?.defaults?.model)).toBe("openai/gpt-5.6-luna"); expect(getPrimaryModel(cfg.agents?.entries?.qa?.model)).toBe("openai/gpt-5.6-luna"); - expect(getModelFallbacks(cfg.agents?.defaults?.model)).toBeUndefined(); - expect(getModelFallbacks(cfg.agents?.entries?.qa?.model)).toBeUndefined(); + expect(getModelFallbacks(cfg.agents?.defaults?.model)).toEqual(["openai/gpt-5.6-sol"]); + expect(getModelFallbacks(cfg.agents?.entries?.qa?.model)).toEqual(["openai/gpt-5.6-sol"]); expect(cfg.models).toBeUndefined(); expect(cfg.memory?.search?.remote).toBeUndefined(); expect(cfg.plugins?.allow).toEqual(["acpx", "memory-core", "qa-lab", "openai", "qa-channel"]); + expect(cfg.plugins?.allow).not.toContain("anthropic"); expect(cfg.plugins?.entries?.openai).toEqual({ enabled: true }); expect(cfg.agents?.defaults?.models?.["openai/gpt-5.6-luna"]).toEqual({ params: { transport: "sse", openaiWsWarmup: false, fastMode: true }, }); }); + it.each([ + ["openai/gpt-5.6", "openai/gpt-5.6-luna"], + ["openai/gpt-5.6-sol", "openai/gpt-5.6-luna"], + ["openai/gpt-5.6-luna", "openai/gpt-5.6-sol"], + ])("keeps an omitted live alternate on OpenAI for %s", (primary, alternate) => { + const cfg = buildQaGatewayConfig({ + bind: "loopback", + gatewayPort: 18789, + gatewayToken: "token", + workspaceDir: "/tmp/qa-workspace", + providerMode: "live-frontier", + primaryModel: primary, + ...createQaChannelTransportParams(), + }); + + expect(getPrimaryModel(cfg.agents?.defaults?.model)).toBe(primary); + expect(getModelFallbacks(cfg.agents?.defaults?.model)).toEqual([alternate]); + expect(cfg.plugins?.allow).toContain("openai"); + expect(cfg.plugins?.allow).not.toContain("anthropic"); + }); + it("keeps inferred live providers when scenarios require additional plugins", () => { const cfg = buildQaGatewayConfig({ bind: "loopback", @@ -350,7 +372,7 @@ describe("buildQaGatewayConfig", () => { providerMode, forcedRuntime: "codex", primaryModel: "openai/gpt-5.6-luna", - alternateModel: "openai/gpt-5.6-luna", + alternateModel: "openai/gpt-5.6-sol", }); expect(cfg.plugins?.allow).toContain("codex"); @@ -442,7 +464,7 @@ describe("buildQaGatewayConfig", () => { workspaceDir: "/tmp/qa-workspace", providerMode: "live-frontier", primaryModel: "codex-cli/test-model", - alternateModel: "codex-cli/test-model", + alternateModel: "codex-cli/test-model-alt", imageGenerationModel: null, enabledPluginIds: ["openai"], ...createQaChannelTransportParams(), @@ -462,7 +484,7 @@ describe("buildQaGatewayConfig", () => { workspaceDir: "/tmp/qa-workspace", providerMode: "live-frontier", primaryModel: "custom-openai/model-a", - alternateModel: "custom-openai/model-a", + alternateModel: "custom-openai/model-b", imageGenerationModel: null, enabledPluginIds: ["openai"], ...createQaChannelTransportParams(), @@ -500,7 +522,7 @@ describe("buildQaGatewayConfig", () => { workspaceDir: "/tmp/qa-workspace", providerMode: "live-frontier", primaryModel: "openai/gpt-5.6-luna", - alternateModel: "openai/gpt-5.6-luna", + alternateModel: "openai/gpt-5.6-sol", thinkingDefault: "xhigh", ...createQaChannelTransportParams(), }); @@ -509,6 +531,22 @@ describe("buildQaGatewayConfig", () => { expect(cfg.agents?.defaults?.models?.["openai/gpt-5.6-luna"]?.params?.thinking).toBe("xhigh"); }); + it("preserves an intentional explicit same-model pair without a fallback", () => { + const cfg = buildQaGatewayConfig({ + bind: "loopback", + gatewayPort: 18789, + gatewayToken: "token", + workspaceDir: "/tmp/qa-workspace", + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.4", + alternateModel: "openai/gpt-5.4", + ...createQaChannelTransportParams(), + }); + + expect(getPrimaryModel(cfg.agents?.defaults?.model)).toBe("openai/gpt-5.4"); + expect(getModelFallbacks(cfg.agents?.defaults?.model)).toBeUndefined(); + }); + it("can disable control ui for suite-only gateway children", () => { const cfg = buildQaGatewayConfig({ bind: "loopback", diff --git a/extensions/qa-lab/src/qa-gateway-config.ts b/extensions/qa-lab/src/qa-gateway-config.ts index bd509d87d53f..28f5db900a32 100644 --- a/extensions/qa-lab/src/qa-gateway-config.ts +++ b/extensions/qa-lab/src/qa-gateway-config.ts @@ -4,11 +4,11 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared"; import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; import { - defaultQaModelForMode, normalizeQaProviderMode, splitQaModelRef, type QaProviderMode, } from "./model-selection.js"; +import { resolveQaRuntimeModelPair } from "./model-selection.runtime.js"; import { getQaProvider } from "./providers/index.js"; import { DEFAULT_QA_PROVIDER_MODE } from "./providers/index.js"; import { QA_FRONTIER_PROVIDER_IDS } from "./providers/live-frontier/catalog.js"; @@ -37,11 +37,6 @@ export function mergeQaControlUiAllowedOrigins(extraOrigins?: string[]) { return uniqueStrings([...DEFAULT_QA_CONTROL_UI_ALLOWED_ORIGINS, ...normalizedExtra]); } -function normalizeQaGatewayModelRef(input: string | undefined, fallback: string) { - const model = input?.trim(); - return model && model.length > 0 ? model : fallback; -} - function remapQaMockModelRefForCodex(modelRef: string) { const split = splitQaModelRef(modelRef); return split?.provider === "mock-openai" ? `openai/${split.model}` : modelRef; @@ -78,14 +73,12 @@ export function buildQaGatewayConfig(params: { const providerMode = normalizeQaProviderMode(params.providerMode ?? DEFAULT_QA_PROVIDER_MODE); const provider = getQaProvider(providerMode); const usesCodexMockAppServer = params.forcedRuntime === "codex" && providerMode === "mock-openai"; - const normalizedPrimaryModel = normalizeQaGatewayModelRef( - params.primaryModel, - defaultQaModelForMode(providerMode), - ); - const normalizedAlternateModel = normalizeQaGatewayModelRef( - params.alternateModel, - defaultQaModelForMode(providerMode, { alternate: true }), - ); + const { primaryModel: normalizedPrimaryModel, alternateModel: normalizedAlternateModel } = + resolveQaRuntimeModelPair({ + providerMode, + primaryModel: params.primaryModel, + alternateModel: params.alternateModel, + }); const primaryModel = usesCodexMockAppServer ? remapQaMockModelRefForCodex(normalizedPrimaryModel) : normalizedPrimaryModel; diff --git a/extensions/qa-lab/src/run-config.test.ts b/extensions/qa-lab/src/run-config.test.ts index 74ff5433a2f7..dc05c91f2386 100644 --- a/extensions/qa-lab/src/run-config.test.ts +++ b/extensions/qa-lab/src/run-config.test.ts @@ -2,19 +2,22 @@ import path from "node:path"; import { beforeEach, describe, expect, it, vi } from "vitest"; -const { defaultQaRuntimeModelForMode } = vi.hoisted(() => ({ +const { defaultQaRuntimeModelForMode, resolveQaRuntimeModelPair } = vi.hoisted(() => ({ defaultQaRuntimeModelForMode: vi.fn<(mode: string, options?: { alternate?: boolean }) => string>(), + resolveQaRuntimeModelPair: vi.fn(), })); vi.mock("./model-selection.runtime.js", () => ({ defaultQaRuntimeModelForMode, + resolveQaRuntimeModelPair, })); import { defaultQaModelForMode as defaultQaProviderModelForMode } from "./model-selection.js"; import { resolveQaRunProfileExecutionSelection, resolveQaRunProfileMembership, } from "./profile-planning.js"; +import { resolveQaLiveFrontierAlternateModel } from "./providers/live-frontier/model-selection.runtime.js"; import { createIdleQaRunnerSnapshot, createQaRunOutputDir, @@ -28,7 +31,26 @@ import { type QaScorecardTaxonomyReport, } from "./scorecard-taxonomy.js"; -const DEFAULT_LIVE_FRONTIER_MODEL = defaultQaProviderModelForMode("live-frontier"); +function resolveMockQaRuntimeModelPair(params: { + providerMode: string; + primaryModel?: string; + alternateModel?: string; + resolveDefaultModel?: (mode: string, alternate?: boolean) => string; +}) { + const resolveDefaultModel = + params.resolveDefaultModel ?? + ((mode: string, alternate = false) => + defaultQaRuntimeModelForMode(mode, alternate ? { alternate: true } : undefined)); + const primaryModel = params.primaryModel?.trim() || resolveDefaultModel(params.providerMode); + const alternateModel = + params.alternateModel?.trim() || + (params.providerMode === "live-frontier" + ? (resolveQaLiveFrontierAlternateModel(primaryModel) ?? + resolveDefaultModel(params.providerMode, true)) + : resolveDefaultModel(params.providerMode, true)); + return { primaryModel, alternateModel }; +} + const profiles: QaScorecardTaxonomyReport["profiles"] = [ { id: "smoke-ci", @@ -92,6 +114,7 @@ describe("qa run config", () => { (mode: string, options?: { alternate?: boolean }) => defaultQaProviderModelForMode(mode as QaProviderModeInput, options), ); + resolveQaRuntimeModelPair.mockImplementation(resolveMockQaRuntimeModelPair); }); it("creates a canonical smoke-profile request without copying profile membership", () => { @@ -132,7 +155,7 @@ describe("qa run config", () => { evidenceMode: "full", providerMode: "live-frontier", primaryModel: "openai/gpt-5.6-luna", - alternateModel: DEFAULT_LIVE_FRONTIER_MODEL, + alternateModel: "openai/gpt-5.6-sol", fastMode: true, runtimePair: null, runtimePairLane: null, @@ -781,11 +804,12 @@ describe("qa run config", () => { }); it("prefers the Codex OAuth default when the runtime resolver says it is available", () => { - defaultQaRuntimeModelForMode.mockImplementation((mode, options) => - mode === "live-frontier" - ? "openai/gpt-5.6-luna" - : defaultQaProviderModelForMode(mode as QaProviderModeInput, options), - ); + defaultQaRuntimeModelForMode.mockImplementation((mode, options) => { + if (mode === "live-frontier" && !options?.alternate) { + return "openai/gpt-5.6-luna"; + } + return defaultQaProviderModelForMode(mode as QaProviderModeInput, options); + }); expect(normalizeQaRunSelection({ profile: "release" }, scenarios, profiles)).toEqual({ profile: "release", @@ -794,7 +818,7 @@ describe("qa run config", () => { evidenceMode: "full", providerMode: "live-frontier", primaryModel: "openai/gpt-5.6-luna", - alternateModel: "openai/gpt-5.6-luna", + alternateModel: "openai/gpt-5.6-sol", fastMode: true, runtimePair: null, runtimePairLane: null, diff --git a/extensions/qa-lab/src/run-config.ts b/extensions/qa-lab/src/run-config.ts index 70a30d767754..d07dd30e1f26 100644 --- a/extensions/qa-lab/src/run-config.ts +++ b/extensions/qa-lab/src/run-config.ts @@ -9,7 +9,10 @@ import type { QaLabRunSelection, } from "../runner-contract.js"; import { defaultQaModelForMode as defaultStaticQaModelForMode } from "./model-selection.js"; -import { defaultQaRuntimeModelForMode } from "./model-selection.runtime.js"; +import { + defaultQaRuntimeModelForMode, + resolveQaRuntimeModelPair, +} from "./model-selection.runtime.js"; import { resolveQaRunProfileExecutionSelection, resolveQaRunProfileMembership, @@ -73,8 +76,7 @@ function createDefaultQaRunSelection( channelDriver: profile.channelDriver, evidenceMode: profile.evidenceMode, providerMode, - primaryModel: resolveDefaultModel(providerMode), - alternateModel: resolveDefaultModel(providerMode, true), + ...resolveQaRuntimeModelPair({ providerMode, resolveDefaultModel }), fastMode: getQaProvider(providerMode).kind === "live", runtimePair: null, runtimePairLane: null, @@ -93,11 +95,6 @@ export function normalizeQaProviderMode(input: unknown): QaProviderMode { throw new Error(`unknown QA provider mode${details}`); } -function normalizeModel(input: unknown, fallback: string) { - const value = typeof input === "string" ? input.trim() : ""; - return value || fallback; -} - function normalizeScenarioIds(input: unknown, scenarios: QaSeedScenario[]): string[] | null { if (input === undefined || input === null) { return null; @@ -230,17 +227,18 @@ export function normalizeQaRunSelection( const providerMode = normalizeQaProviderMode( payload.providerMode ?? (profile === "smoke-ci" ? "mock-openai" : undefined), ); + const models = resolveQaRuntimeModelPair({ + providerMode, + primaryModel: typeof payload.primaryModel === "string" ? payload.primaryModel : undefined, + alternateModel: typeof payload.alternateModel === "string" ? payload.alternateModel : undefined, + }); return { profile, channel: normalizeQaChannel(payload.channel), channelDriver: normalizeQaChannelDriver(payload.channelDriver, profileDefaults.channelDriver), evidenceMode: normalizeQaEvidenceMode(payload.evidenceMode, profileDefaults.evidenceMode), providerMode, - primaryModel: normalizeModel(payload.primaryModel, defaultQaModelForMode(providerMode)), - alternateModel: normalizeModel( - payload.alternateModel, - defaultQaModelForMode(providerMode, true), - ), + ...models, fastMode: getQaProvider(providerMode).kind === "live" || payload.fastMode === true, runtimePair: normalizeQaRuntimePair(payload.runtimePair), runtimePairLane: normalizeQaRuntimePairLane(payload.runtimePairLane), diff --git a/extensions/qa-lab/src/suite-launch.runtime.ts b/extensions/qa-lab/src/suite-launch.runtime.ts index 59181ce79d83..f1be725cd3e0 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.ts @@ -13,6 +13,7 @@ import { type QaEvidenceSummaryJson, } from "./evidence-summary.js"; import { isQaFastModeEnabled } from "./model-selection.js"; +import { resolveQaRuntimeModelPair } from "./model-selection.runtime.js"; import { DEFAULT_QA_PROVIDER_MODE } from "./providers/index.js"; import { defaultQaSuiteConcurrencyForTransport, @@ -765,10 +766,11 @@ async function runUnifiedQaSuite(params: { }) : undefined; progress?.start(); - const primaryModel = - params.runParams?.primaryModel?.trim() || defaultQaModelForMode(providerMode); - const alternateModel = - params.runParams?.alternateModel?.trim() || defaultQaModelForMode(providerMode, true); + const { primaryModel, alternateModel } = resolveQaRuntimeModelPair({ + providerMode, + primaryModel: params.runParams?.primaryModel, + alternateModel: params.runParams?.alternateModel, + }); const fastMode = typeof params.runParams?.fastMode === "boolean" ? params.runParams.fastMode diff --git a/extensions/qa-lab/src/suite-model-selection.test.ts b/extensions/qa-lab/src/suite-model-selection.test.ts new file mode 100644 index 000000000000..10851266dbe6 --- /dev/null +++ b/extensions/qa-lab/src/suite-model-selection.test.ts @@ -0,0 +1,32 @@ +// QA Lab tests cover suite model-pair resolution. +import { describe, expect, it } from "vitest"; +import { resolveRequestedQaSuiteModels } from "./suite-model-selection.js"; + +describe("resolveRequestedQaSuiteModels", () => { + it("derives Luna after an explicit Sol primary", () => { + expect( + resolveRequestedQaSuiteModels({ + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.6-sol", + scenarios: [], + }), + ).toMatchObject({ + primaryModel: "openai/gpt-5.6-sol", + alternateModel: "openai/gpt-5.6-luna", + }); + }); + + it("preserves an explicit alternate", () => { + expect( + resolveRequestedQaSuiteModels({ + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.6-luna", + alternateModel: "openai/gpt-5.6-terra", + scenarios: [], + }), + ).toMatchObject({ + primaryModel: "openai/gpt-5.6-luna", + alternateModel: "openai/gpt-5.6-terra", + }); + }); +}); diff --git a/extensions/qa-lab/src/suite-model-selection.ts b/extensions/qa-lab/src/suite-model-selection.ts index 7d26ca6e1a4e..a2a0e1f9a2fd 100644 --- a/extensions/qa-lab/src/suite-model-selection.ts +++ b/extensions/qa-lab/src/suite-model-selection.ts @@ -4,18 +4,13 @@ import { normalizeQaProviderMode, type QaProviderMode, } from "./model-selection.js"; +import { resolveQaRuntimeModelPair } from "./model-selection.runtime.js"; import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "./providers/index.js"; -import { defaultQaModelForMode } from "./run-config.js"; import { resolveQaScenarioRequiredProviderMode, type QaSeedScenarioWithSource, } from "./scenario-catalog.js"; -function normalizeQaSuiteModelRef(input: string | undefined, fallback: string) { - const model = input?.trim(); - return model && model.length > 0 ? model : fallback; -} - export function resolveRequestedQaSuiteModels(params: { alternateModel?: string; fastMode?: boolean; @@ -36,14 +31,11 @@ export function resolveRequestedQaSuiteModels(params: { const providerMode = normalizeQaProviderMode( params.providerMode ?? selectedProviderMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE, ); - const primaryModel = normalizeQaSuiteModelRef( - params.primaryModel, - defaultQaModelForMode(providerMode), - ); - const alternateModel = normalizeQaSuiteModelRef( - params.alternateModel, - defaultQaModelForMode(providerMode, true), - ); + const { primaryModel, alternateModel } = resolveQaRuntimeModelPair({ + providerMode, + primaryModel: params.primaryModel, + alternateModel: params.alternateModel, + }); return { alternateModel, fastMode: params.fastMode ?? isQaFastModeEnabled({ primaryModel, alternateModel }), diff --git a/extensions/qa-lab/src/suite-provider-selection.test.ts b/extensions/qa-lab/src/suite-provider-selection.test.ts index 00abb75f459f..86f6a1535fa4 100644 --- a/extensions/qa-lab/src/suite-provider-selection.test.ts +++ b/extensions/qa-lab/src/suite-provider-selection.test.ts @@ -140,9 +140,10 @@ describe("qa suite provider selection", () => { }; expect(summary.run).toMatchObject({ providerMode: "live-frontier", - primaryModel: expect.stringMatching(/^openai\//), - alternateModel: expect.stringMatching(/^openai\//), + primaryModel: "openai/gpt-5.6", + alternateModel: "openai/gpt-5.6-luna", }); + expect(summary.run.primaryModel).not.toBe(summary.run.alternateModel); } finally { await rm(repoRoot, { recursive: true, force: true }); }