fix(qa): resolve live model pairs after auth selection (#121065)

Centralize QA model-pair construction after resolving the actual primary,
preserve explicit and manual single-model selections, and keep omitted
live-frontier alternates on the provider-owned OpenAI Sol/Luna policy.

Punchcard-Session: amber-workshop-workshop-36

Co-authored-by: Dallin Romney <6581799+RomneyDa@users.noreply.github.com>
This commit is contained in:
Vincent Koc
2026-08-10 02:59:12 +08:00
committed by GitHub
parent 26e74d8d18
commit 126b5a08e7
14 changed files with 337 additions and 125 deletions
+50 -25
View File
@@ -18,6 +18,7 @@ const {
buildQaDockerHarnessImage,
runQaDockerUp,
defaultQaRuntimeModelForMode,
resolveQaRuntimeModelPair,
readQaScenarioPack,
} = vi.hoisted(() => ({
runQaManualLane: vi.fn(),
@@ -32,6 +33,7 @@ const {
runQaDockerUp: vi.fn(),
defaultQaRuntimeModelForMode:
vi.fn<(mode: string, options?: { alternate?: boolean }) => string>(),
resolveQaRuntimeModelPair: vi.fn(),
readQaScenarioPack: vi.fn<() => QaScenarioPack>(),
}));
@@ -76,6 +78,7 @@ vi.mock("./docker-up.runtime.js", () => ({
vi.mock("./model-selection.runtime.js", () => ({
defaultQaRuntimeModelForMode,
resolveQaRuntimeModelPair,
}));
vi.mock("./scenario-catalog.js", async (importOriginal) => {
@@ -106,12 +109,29 @@ import { QaSuiteInfraError } from "./errors.js";
import { QA_EVIDENCE_FILENAME } from "./evidence-summary.js";
import { runQaTelegramCommand } from "./live-transports/telegram/cli.runtime.js";
import { defaultQaModelForMode as defaultQaProviderModelForMode } from "./model-selection.js";
import { resolveQaLiveFrontierAlternateModel } from "./providers/live-frontier/model-selection.runtime.js";
import type { QaTransportAdapterFactory } from "./qa-transport-registry.js";
import type { QaProviderModeInput } from "./run-config.js";
import { expandQaScenarioExecutionCells } from "./scenario-lane.js";
import type { QaSuiteRunParams } from "./suite.js";
const DEFAULT_LIVE_FRONTIER_MODEL = defaultQaProviderModelForMode("live-frontier");
function resolveMockQaRuntimeModelPair(params: {
providerMode: string;
primaryModel?: string;
alternateModel?: string;
}) {
const primaryModel =
params.primaryModel?.trim() || defaultQaRuntimeModelForMode(params.providerMode);
const alternateModel =
params.alternateModel?.trim() ||
(params.providerMode === "live-frontier"
? (resolveQaLiveFrontierAlternateModel(primaryModel) ??
defaultQaRuntimeModelForMode(params.providerMode, { alternate: true }))
: defaultQaRuntimeModelForMode(params.providerMode, { alternate: true }));
return { primaryModel, alternateModel };
}
const QA_PASSING_SUITE_SCENARIO = {
name: "channel chat baseline",
status: "pass" as const,
@@ -340,6 +360,7 @@ describe("qa cli runtime", () => {
(mode: string, options?: { alternate?: boolean }) =>
defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
);
resolveQaRuntimeModelPair.mockImplementation(resolveMockQaRuntimeModelPair);
readQaScenarioPack.mockClear();
runQaSuite.mockImplementation(async (params) => {
const observedCells = executionCellsForSuiteParams(params);
@@ -3262,39 +3283,43 @@ describe("qa cli runtime", () => {
transportId: "qa-channel",
providerMode: "live-frontier",
primaryModel: DEFAULT_LIVE_FRONTIER_MODEL,
alternateModel: DEFAULT_LIVE_FRONTIER_MODEL,
alternateModel: "openai/gpt-5.6-luna",
fastMode: undefined,
message: "read qa kickoff and reply short",
timeoutMs: undefined,
});
});
it("keeps an explicit manual primary model as the alternate default", async () => {
await runQaManualLaneCommand({
repoRoot: "/tmp/openclaw-repo",
providerMode: "live-frontier",
primaryModel: "anthropic/claude-sonnet-4-6",
message: "read qa kickoff and reply short",
});
it.each(["anthropic/claude-sonnet-4-6", "openai/gpt-5.6-sol"])(
"keeps explicit manual primary %s single-model when the alternate is omitted",
async (primaryModel) => {
await runQaManualLaneCommand({
repoRoot: "/tmp/openclaw-repo",
providerMode: "live-frontier",
primaryModel,
message: "read qa kickoff and reply short",
});
expect(runQaManualLane).toHaveBeenCalledWith({
repoRoot: path.resolve("/tmp/openclaw-repo"),
transportId: "qa-channel",
providerMode: "live-frontier",
primaryModel: "anthropic/claude-sonnet-4-6",
alternateModel: "anthropic/claude-sonnet-4-6",
fastMode: undefined,
message: "read qa kickoff and reply short",
timeoutMs: undefined,
});
});
expect(runQaManualLane).toHaveBeenCalledWith({
repoRoot: path.resolve("/tmp/openclaw-repo"),
transportId: "qa-channel",
providerMode: "live-frontier",
primaryModel,
alternateModel: primaryModel,
fastMode: undefined,
message: "read qa kickoff and reply short",
timeoutMs: undefined,
});
},
);
it("defaults manual frontier runs onto Codex OAuth when the runtime resolver prefers it", async () => {
defaultQaRuntimeModelForMode.mockImplementation((mode, options) =>
mode === "live-frontier"
? "openai/gpt-5.6-luna"
: defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
);
defaultQaRuntimeModelForMode.mockImplementation((mode, options) => {
if (mode === "live-frontier" && !options?.alternate) {
return "openai/gpt-5.6-luna";
}
return defaultQaProviderModelForMode(mode as QaProviderModeInput, options);
});
await runQaManualLaneCommand({
repoRoot: "/tmp/openclaw-repo",
@@ -3306,7 +3331,7 @@ describe("qa cli runtime", () => {
transportId: "qa-channel",
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.6-luna",
alternateModel: "openai/gpt-5.6-luna",
alternateModel: "openai/gpt-5.6-sol",
fastMode: undefined,
message: "read qa kickoff and reply short",
timeoutMs: undefined,
+9 -11
View File
@@ -46,6 +46,7 @@ import {
import { startQaLabServer } from "./lab-server.js";
import { listLiveTransportQaAdapterFactories } from "./live-transports/cli.js";
import { runQaManualLane } from "./manual-lane.runtime.js";
import { resolveQaRuntimeModelPair } from "./model-selection.runtime.js";
import { runQaMultipass } from "./multipass.runtime.js";
import { qaProfileEvidencePlan, type QaProfileEvidencePlan } from "./profile-evidence-plan.js";
import {
@@ -193,17 +194,14 @@ function resolveQaManualLaneModels(opts: {
primaryModel?: string;
alternateModel?: string;
}) {
const primaryModel = opts.primaryModel?.trim() || defaultQaModelForMode(opts.providerMode);
const alternateModel = opts.alternateModel?.trim();
return {
primaryModel,
alternateModel:
alternateModel && alternateModel.length > 0
? alternateModel
: opts.primaryModel?.trim()
? primaryModel
: defaultQaModelForMode(opts.providerMode, true),
};
// `qa manual --model` is a one-model probe unless the operator also supplies
// `--alt-model`; materialize that contract before shared pair resolution.
const explicitPrimaryModel = opts.primaryModel?.trim();
return resolveQaRuntimeModelPair({
...opts,
primaryModel: explicitPrimaryModel,
alternateModel: opts.alternateModel?.trim() || explicitPrimaryModel,
});
}
function parseQaThinkingLevel(
+8 -4
View File
@@ -1084,15 +1084,19 @@ describe("buildQaRuntimeEnv", () => {
expect(configProfile.provider).toBe("openai");
expect(configProfile.mode).toBe("api_key");
expect(configProfile.displayName).toBe("QA live openai env credential");
expect(Object.values(cfg.auth?.profiles ?? {})).not.toContainEqual(
expect.objectContaining({ provider: "anthropic" }),
);
for (const agentId of ["main", "qa"]) {
const storeProfile = requireAuthProfile(
readAuthProfileStore(stateDir, agentId).profiles,
"qa-live-openai-env",
);
const profiles = readAuthProfileStore(stateDir, agentId).profiles;
const storeProfile = requireAuthProfile(profiles, "qa-live-openai-env");
expect(storeProfile.type).toBe("api_key");
expect(storeProfile.provider).toBe("openai");
expect(storeProfile.key).toBe("qa-live-not-a-real-key");
expect(Object.values(profiles)).not.toContainEqual(
expect.objectContaining({ provider: "anthropic" }),
);
}
});
@@ -18,7 +18,10 @@ vi.mock("openclaw/plugin-sdk/agent-runtime", () => ({
listProfilesForProvider,
}));
import { defaultQaRuntimeModelForMode } from "./model-selection.runtime.js";
import {
defaultQaRuntimeModelForMode,
resolveQaRuntimeModelPair,
} from "./model-selection.runtime.js";
describe("qa model selection runtime", () => {
beforeEach(() => {
@@ -34,26 +37,36 @@ describe("qa model selection runtime", () => {
resolveEnvApiKey.mockReturnValue({ apiKey: "sk-test" });
expect(defaultQaRuntimeModelForMode("live-frontier")).toBe("openai/gpt-5.6");
expect(resolveQaRuntimeModelPair({ providerMode: "live-frontier" })).toEqual({
primaryModel: "openai/gpt-5.6",
alternateModel: "openai/gpt-5.6-luna",
});
expect(loadAuthProfileStoreForRuntime).not.toHaveBeenCalled();
});
it("prefers the Codex OAuth live default when only Codex auth profiles are available", () => {
loadAuthProfileStoreForRuntime.mockReturnValue({
profiles: {
"openai:user@example.com": {
provider: "openai",
type: "oauth",
it.each(["oauth", "token"] as const)(
"prefers the Codex live default for a stored %s profile",
(type) => {
loadAuthProfileStoreForRuntime.mockReturnValue({
profiles: {
"openai:user@example.com": {
provider: "openai",
type,
},
},
},
});
});
expect(defaultQaRuntimeModelForMode("live-frontier")).toBe("openai/gpt-5.6-luna");
expect(loadAuthProfileStoreForRuntime).toHaveBeenCalledWith(undefined, {
readOnly: true,
allowKeychainPrompt: false,
externalCliProviderIds: ["openai"],
});
});
expect(resolveQaRuntimeModelPair({ providerMode: "live-frontier" })).toEqual({
primaryModel: "openai/gpt-5.6-luna",
alternateModel: "openai/gpt-5.6-sol",
});
expect(loadAuthProfileStoreForRuntime).toHaveBeenCalledWith(undefined, {
readOnly: true,
allowKeychainPrompt: false,
externalCliProviderIds: ["openai"],
});
},
);
it("keeps the OpenAI live default when stored OpenAI profiles are available", () => {
loadAuthProfileStoreForRuntime.mockReturnValue({
@@ -68,6 +81,66 @@ describe("qa model selection runtime", () => {
expect(defaultQaRuntimeModelForMode("live-frontier")).toBe("openai/gpt-5.6");
});
it.each(["openai/gpt-5.6", "openai/gpt-5.6-sol"])(
"derives Luna after explicit Sol primary %s",
(primaryModel) => {
expect(resolveQaRuntimeModelPair({ providerMode: "live-frontier", primaryModel })).toEqual({
primaryModel,
alternateModel: "openai/gpt-5.6-luna",
});
},
);
it("derives Sol after an explicit Luna primary", () => {
expect(
resolveQaRuntimeModelPair({
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.6-luna",
}),
).toEqual({
primaryModel: "openai/gpt-5.6-luna",
alternateModel: "openai/gpt-5.6-sol",
});
});
it("falls back through the provider default for an unmapped primary", () => {
expect(
resolveQaRuntimeModelPair({
providerMode: "live-frontier",
primaryModel: "anthropic/claude-sonnet-4-6",
}),
).toEqual({
primaryModel: "anthropic/claude-sonnet-4-6",
alternateModel: "openai/gpt-5.6",
});
});
it("preserves an explicit alternate model", () => {
expect(
resolveQaRuntimeModelPair({
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.6",
alternateModel: "openai/gpt-5.6-terra",
}),
).toEqual({
primaryModel: "openai/gpt-5.6",
alternateModel: "openai/gpt-5.6-terra",
});
});
it.each([
["openai/gpt-5.4", "openai/gpt-5.4"],
["openai/gpt-5.6", "openai/gpt-5.6-sol"],
])("preserves the explicit model pair %s / %s", (primaryModel, alternateModel) => {
expect(
resolveQaRuntimeModelPair({
providerMode: "live-frontier",
primaryModel,
alternateModel,
}),
).toEqual({ primaryModel, alternateModel });
});
it("leaves mock defaults unchanged", () => {
expect(defaultQaRuntimeModelForMode("mock-openai")).toBe("mock-openai/gpt-5.6-luna");
expect(defaultQaRuntimeModelForMode("mock-openai", { alternate: true })).toBe(
@@ -2,14 +2,14 @@
import {
defaultQaModelForMode,
normalizeQaProviderMode,
type QaProviderMode,
type QaProviderModeInput,
} from "./model-selection.js";
import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "./providers/index.js";
import { resolveQaLiveFrontierPreferredModel } from "./providers/live-frontier/model-selection.runtime.js";
function resolveQaPreferredLiveModel() {
return resolveQaLiveFrontierPreferredModel();
}
import {
resolveQaLiveFrontierAlternateModel,
resolveQaLiveFrontierPreferredModel,
} from "./providers/live-frontier/model-selection.runtime.js";
export function defaultQaRuntimeModelForMode(
mode: QaProviderModeInput,
@@ -21,10 +21,32 @@ export function defaultQaRuntimeModelForMode(
const preferredLiveModel =
options?.preferredLiveModel ??
(normalizeQaProviderMode(mode) === DEFAULT_QA_LIVE_PROVIDER_MODE
? resolveQaPreferredLiveModel()
? resolveQaLiveFrontierPreferredModel()
: undefined);
return defaultQaModelForMode(mode, {
...options,
preferredLiveModel,
});
}
export function resolveQaRuntimeModelPair(params: {
providerMode: QaProviderModeInput;
primaryModel?: string;
alternateModel?: string;
resolveDefaultModel?: (mode: QaProviderMode, alternate?: boolean) => string;
}) {
const providerMode = normalizeQaProviderMode(params.providerMode);
const normalizeModel = (model: string | undefined) => model?.trim() || undefined;
const resolveDefaultModel =
params.resolveDefaultModel ??
((mode: QaProviderModeInput, alternate = false) =>
defaultQaRuntimeModelForMode(mode, alternate ? { alternate: true } : undefined));
const primaryModel = normalizeModel(params.primaryModel) ?? resolveDefaultModel(providerMode);
const alternateModel =
normalizeModel(params.alternateModel) ??
(providerMode === DEFAULT_QA_LIVE_PROVIDER_MODE
? (resolveQaLiveFrontierAlternateModel(primaryModel) ??
resolveDefaultModel(providerMode, true))
: resolveDefaultModel(providerMode, true));
return { primaryModel, alternateModel };
}
@@ -7,6 +7,16 @@ import { resolveEnvApiKey } from "openclaw/plugin-sdk/provider-auth";
const QA_CODEX_OAUTH_LIVE_MODEL = "openai/gpt-5.6-luna";
export function resolveQaLiveFrontierAlternateModel(primaryModel: string) {
const normalized = primaryModel.toLowerCase();
if (normalized === QA_CODEX_OAUTH_LIVE_MODEL) {
return "openai/gpt-5.6-sol";
}
return normalized === "openai/gpt-5.6" || normalized === "openai/gpt-5.6-sol"
? QA_CODEX_OAUTH_LIVE_MODEL
: undefined;
}
export function resolveQaLiveFrontierPreferredModel() {
if (resolveEnvApiKey("openai")?.apiKey) {
return undefined;
@@ -270,23 +270,45 @@ describe("buildQaGatewayConfig", () => {
providerMode: "live-frontier",
fastMode: true,
primaryModel: "openai/gpt-5.6-luna",
alternateModel: "openai/gpt-5.6-luna",
alternateModel: "openai/gpt-5.6-sol",
...createQaChannelTransportParams(),
});
expect(getPrimaryModel(cfg.agents?.defaults?.model)).toBe("openai/gpt-5.6-luna");
expect(getPrimaryModel(cfg.agents?.entries?.qa?.model)).toBe("openai/gpt-5.6-luna");
expect(getModelFallbacks(cfg.agents?.defaults?.model)).toBeUndefined();
expect(getModelFallbacks(cfg.agents?.entries?.qa?.model)).toBeUndefined();
expect(getModelFallbacks(cfg.agents?.defaults?.model)).toEqual(["openai/gpt-5.6-sol"]);
expect(getModelFallbacks(cfg.agents?.entries?.qa?.model)).toEqual(["openai/gpt-5.6-sol"]);
expect(cfg.models).toBeUndefined();
expect(cfg.memory?.search?.remote).toBeUndefined();
expect(cfg.plugins?.allow).toEqual(["acpx", "memory-core", "qa-lab", "openai", "qa-channel"]);
expect(cfg.plugins?.allow).not.toContain("anthropic");
expect(cfg.plugins?.entries?.openai).toEqual({ enabled: true });
expect(cfg.agents?.defaults?.models?.["openai/gpt-5.6-luna"]).toEqual({
params: { transport: "sse", openaiWsWarmup: false, fastMode: true },
});
});
it.each([
["openai/gpt-5.6", "openai/gpt-5.6-luna"],
["openai/gpt-5.6-sol", "openai/gpt-5.6-luna"],
["openai/gpt-5.6-luna", "openai/gpt-5.6-sol"],
])("keeps an omitted live alternate on OpenAI for %s", (primary, alternate) => {
const cfg = buildQaGatewayConfig({
bind: "loopback",
gatewayPort: 18789,
gatewayToken: "token",
workspaceDir: "/tmp/qa-workspace",
providerMode: "live-frontier",
primaryModel: primary,
...createQaChannelTransportParams(),
});
expect(getPrimaryModel(cfg.agents?.defaults?.model)).toBe(primary);
expect(getModelFallbacks(cfg.agents?.defaults?.model)).toEqual([alternate]);
expect(cfg.plugins?.allow).toContain("openai");
expect(cfg.plugins?.allow).not.toContain("anthropic");
});
it("keeps inferred live providers when scenarios require additional plugins", () => {
const cfg = buildQaGatewayConfig({
bind: "loopback",
@@ -350,7 +372,7 @@ describe("buildQaGatewayConfig", () => {
providerMode,
forcedRuntime: "codex",
primaryModel: "openai/gpt-5.6-luna",
alternateModel: "openai/gpt-5.6-luna",
alternateModel: "openai/gpt-5.6-sol",
});
expect(cfg.plugins?.allow).toContain("codex");
@@ -442,7 +464,7 @@ describe("buildQaGatewayConfig", () => {
workspaceDir: "/tmp/qa-workspace",
providerMode: "live-frontier",
primaryModel: "codex-cli/test-model",
alternateModel: "codex-cli/test-model",
alternateModel: "codex-cli/test-model-alt",
imageGenerationModel: null,
enabledPluginIds: ["openai"],
...createQaChannelTransportParams(),
@@ -462,7 +484,7 @@ describe("buildQaGatewayConfig", () => {
workspaceDir: "/tmp/qa-workspace",
providerMode: "live-frontier",
primaryModel: "custom-openai/model-a",
alternateModel: "custom-openai/model-a",
alternateModel: "custom-openai/model-b",
imageGenerationModel: null,
enabledPluginIds: ["openai"],
...createQaChannelTransportParams(),
@@ -500,7 +522,7 @@ describe("buildQaGatewayConfig", () => {
workspaceDir: "/tmp/qa-workspace",
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.6-luna",
alternateModel: "openai/gpt-5.6-luna",
alternateModel: "openai/gpt-5.6-sol",
thinkingDefault: "xhigh",
...createQaChannelTransportParams(),
});
@@ -509,6 +531,22 @@ describe("buildQaGatewayConfig", () => {
expect(cfg.agents?.defaults?.models?.["openai/gpt-5.6-luna"]?.params?.thinking).toBe("xhigh");
});
it("preserves an intentional explicit same-model pair without a fallback", () => {
const cfg = buildQaGatewayConfig({
bind: "loopback",
gatewayPort: 18789,
gatewayToken: "token",
workspaceDir: "/tmp/qa-workspace",
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.4",
alternateModel: "openai/gpt-5.4",
...createQaChannelTransportParams(),
});
expect(getPrimaryModel(cfg.agents?.defaults?.model)).toBe("openai/gpt-5.4");
expect(getModelFallbacks(cfg.agents?.defaults?.model)).toBeUndefined();
});
it("can disable control ui for suite-only gateway children", () => {
const cfg = buildQaGatewayConfig({
bind: "loopback",
+7 -14
View File
@@ -4,11 +4,11 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
import {
defaultQaModelForMode,
normalizeQaProviderMode,
splitQaModelRef,
type QaProviderMode,
} from "./model-selection.js";
import { resolveQaRuntimeModelPair } from "./model-selection.runtime.js";
import { getQaProvider } from "./providers/index.js";
import { DEFAULT_QA_PROVIDER_MODE } from "./providers/index.js";
import { QA_FRONTIER_PROVIDER_IDS } from "./providers/live-frontier/catalog.js";
@@ -37,11 +37,6 @@ export function mergeQaControlUiAllowedOrigins(extraOrigins?: string[]) {
return uniqueStrings([...DEFAULT_QA_CONTROL_UI_ALLOWED_ORIGINS, ...normalizedExtra]);
}
function normalizeQaGatewayModelRef(input: string | undefined, fallback: string) {
const model = input?.trim();
return model && model.length > 0 ? model : fallback;
}
function remapQaMockModelRefForCodex(modelRef: string) {
const split = splitQaModelRef(modelRef);
return split?.provider === "mock-openai" ? `openai/${split.model}` : modelRef;
@@ -78,14 +73,12 @@ export function buildQaGatewayConfig(params: {
const providerMode = normalizeQaProviderMode(params.providerMode ?? DEFAULT_QA_PROVIDER_MODE);
const provider = getQaProvider(providerMode);
const usesCodexMockAppServer = params.forcedRuntime === "codex" && providerMode === "mock-openai";
const normalizedPrimaryModel = normalizeQaGatewayModelRef(
params.primaryModel,
defaultQaModelForMode(providerMode),
);
const normalizedAlternateModel = normalizeQaGatewayModelRef(
params.alternateModel,
defaultQaModelForMode(providerMode, { alternate: true }),
);
const { primaryModel: normalizedPrimaryModel, alternateModel: normalizedAlternateModel } =
resolveQaRuntimeModelPair({
providerMode,
primaryModel: params.primaryModel,
alternateModel: params.alternateModel,
});
const primaryModel = usesCodexMockAppServer
? remapQaMockModelRefForCodex(normalizedPrimaryModel)
: normalizedPrimaryModel;
+33 -9
View File
@@ -2,19 +2,22 @@
import path from "node:path";
import { beforeEach, describe, expect, it, vi } from "vitest";
const { defaultQaRuntimeModelForMode } = vi.hoisted(() => ({
const { defaultQaRuntimeModelForMode, resolveQaRuntimeModelPair } = vi.hoisted(() => ({
defaultQaRuntimeModelForMode:
vi.fn<(mode: string, options?: { alternate?: boolean }) => string>(),
resolveQaRuntimeModelPair: vi.fn(),
}));
vi.mock("./model-selection.runtime.js", () => ({
defaultQaRuntimeModelForMode,
resolveQaRuntimeModelPair,
}));
import { defaultQaModelForMode as defaultQaProviderModelForMode } from "./model-selection.js";
import {
resolveQaRunProfileExecutionSelection,
resolveQaRunProfileMembership,
} from "./profile-planning.js";
import { resolveQaLiveFrontierAlternateModel } from "./providers/live-frontier/model-selection.runtime.js";
import {
createIdleQaRunnerSnapshot,
createQaRunOutputDir,
@@ -28,7 +31,26 @@ import {
type QaScorecardTaxonomyReport,
} from "./scorecard-taxonomy.js";
const DEFAULT_LIVE_FRONTIER_MODEL = defaultQaProviderModelForMode("live-frontier");
function resolveMockQaRuntimeModelPair(params: {
providerMode: string;
primaryModel?: string;
alternateModel?: string;
resolveDefaultModel?: (mode: string, alternate?: boolean) => string;
}) {
const resolveDefaultModel =
params.resolveDefaultModel ??
((mode: string, alternate = false) =>
defaultQaRuntimeModelForMode(mode, alternate ? { alternate: true } : undefined));
const primaryModel = params.primaryModel?.trim() || resolveDefaultModel(params.providerMode);
const alternateModel =
params.alternateModel?.trim() ||
(params.providerMode === "live-frontier"
? (resolveQaLiveFrontierAlternateModel(primaryModel) ??
resolveDefaultModel(params.providerMode, true))
: resolveDefaultModel(params.providerMode, true));
return { primaryModel, alternateModel };
}
const profiles: QaScorecardTaxonomyReport["profiles"] = [
{
id: "smoke-ci",
@@ -92,6 +114,7 @@ describe("qa run config", () => {
(mode: string, options?: { alternate?: boolean }) =>
defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
);
resolveQaRuntimeModelPair.mockImplementation(resolveMockQaRuntimeModelPair);
});
it("creates a canonical smoke-profile request without copying profile membership", () => {
@@ -132,7 +155,7 @@ describe("qa run config", () => {
evidenceMode: "full",
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.6-luna",
alternateModel: DEFAULT_LIVE_FRONTIER_MODEL,
alternateModel: "openai/gpt-5.6-sol",
fastMode: true,
runtimePair: null,
runtimePairLane: null,
@@ -781,11 +804,12 @@ describe("qa run config", () => {
});
it("prefers the Codex OAuth default when the runtime resolver says it is available", () => {
defaultQaRuntimeModelForMode.mockImplementation((mode, options) =>
mode === "live-frontier"
? "openai/gpt-5.6-luna"
: defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
);
defaultQaRuntimeModelForMode.mockImplementation((mode, options) => {
if (mode === "live-frontier" && !options?.alternate) {
return "openai/gpt-5.6-luna";
}
return defaultQaProviderModelForMode(mode as QaProviderModeInput, options);
});
expect(normalizeQaRunSelection({ profile: "release" }, scenarios, profiles)).toEqual({
profile: "release",
@@ -794,7 +818,7 @@ describe("qa run config", () => {
evidenceMode: "full",
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.6-luna",
alternateModel: "openai/gpt-5.6-luna",
alternateModel: "openai/gpt-5.6-sol",
fastMode: true,
runtimePair: null,
runtimePairLane: null,
+11 -13
View File
@@ -9,7 +9,10 @@ import type {
QaLabRunSelection,
} from "../runner-contract.js";
import { defaultQaModelForMode as defaultStaticQaModelForMode } from "./model-selection.js";
import { defaultQaRuntimeModelForMode } from "./model-selection.runtime.js";
import {
defaultQaRuntimeModelForMode,
resolveQaRuntimeModelPair,
} from "./model-selection.runtime.js";
import {
resolveQaRunProfileExecutionSelection,
resolveQaRunProfileMembership,
@@ -73,8 +76,7 @@ function createDefaultQaRunSelection(
channelDriver: profile.channelDriver,
evidenceMode: profile.evidenceMode,
providerMode,
primaryModel: resolveDefaultModel(providerMode),
alternateModel: resolveDefaultModel(providerMode, true),
...resolveQaRuntimeModelPair({ providerMode, resolveDefaultModel }),
fastMode: getQaProvider(providerMode).kind === "live",
runtimePair: null,
runtimePairLane: null,
@@ -93,11 +95,6 @@ export function normalizeQaProviderMode(input: unknown): QaProviderMode {
throw new Error(`unknown QA provider mode${details}`);
}
function normalizeModel(input: unknown, fallback: string) {
const value = typeof input === "string" ? input.trim() : "";
return value || fallback;
}
function normalizeScenarioIds(input: unknown, scenarios: QaSeedScenario[]): string[] | null {
if (input === undefined || input === null) {
return null;
@@ -230,17 +227,18 @@ export function normalizeQaRunSelection(
const providerMode = normalizeQaProviderMode(
payload.providerMode ?? (profile === "smoke-ci" ? "mock-openai" : undefined),
);
const models = resolveQaRuntimeModelPair({
providerMode,
primaryModel: typeof payload.primaryModel === "string" ? payload.primaryModel : undefined,
alternateModel: typeof payload.alternateModel === "string" ? payload.alternateModel : undefined,
});
return {
profile,
channel: normalizeQaChannel(payload.channel),
channelDriver: normalizeQaChannelDriver(payload.channelDriver, profileDefaults.channelDriver),
evidenceMode: normalizeQaEvidenceMode(payload.evidenceMode, profileDefaults.evidenceMode),
providerMode,
primaryModel: normalizeModel(payload.primaryModel, defaultQaModelForMode(providerMode)),
alternateModel: normalizeModel(
payload.alternateModel,
defaultQaModelForMode(providerMode, true),
),
...models,
fastMode: getQaProvider(providerMode).kind === "live" || payload.fastMode === true,
runtimePair: normalizeQaRuntimePair(payload.runtimePair),
runtimePairLane: normalizeQaRuntimePairLane(payload.runtimePairLane),
@@ -13,6 +13,7 @@ import {
type QaEvidenceSummaryJson,
} from "./evidence-summary.js";
import { isQaFastModeEnabled } from "./model-selection.js";
import { resolveQaRuntimeModelPair } from "./model-selection.runtime.js";
import { DEFAULT_QA_PROVIDER_MODE } from "./providers/index.js";
import {
defaultQaSuiteConcurrencyForTransport,
@@ -765,10 +766,11 @@ async function runUnifiedQaSuite(params: {
})
: undefined;
progress?.start();
const primaryModel =
params.runParams?.primaryModel?.trim() || defaultQaModelForMode(providerMode);
const alternateModel =
params.runParams?.alternateModel?.trim() || defaultQaModelForMode(providerMode, true);
const { primaryModel, alternateModel } = resolveQaRuntimeModelPair({
providerMode,
primaryModel: params.runParams?.primaryModel,
alternateModel: params.runParams?.alternateModel,
});
const fastMode =
typeof params.runParams?.fastMode === "boolean"
? params.runParams.fastMode
@@ -0,0 +1,32 @@
// QA Lab tests cover suite model-pair resolution.
import { describe, expect, it } from "vitest";
import { resolveRequestedQaSuiteModels } from "./suite-model-selection.js";
describe("resolveRequestedQaSuiteModels", () => {
it("derives Luna after an explicit Sol primary", () => {
expect(
resolveRequestedQaSuiteModels({
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.6-sol",
scenarios: [],
}),
).toMatchObject({
primaryModel: "openai/gpt-5.6-sol",
alternateModel: "openai/gpt-5.6-luna",
});
});
it("preserves an explicit alternate", () => {
expect(
resolveRequestedQaSuiteModels({
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.6-luna",
alternateModel: "openai/gpt-5.6-terra",
scenarios: [],
}),
).toMatchObject({
primaryModel: "openai/gpt-5.6-luna",
alternateModel: "openai/gpt-5.6-terra",
});
});
});
+6 -14
View File
@@ -4,18 +4,13 @@ import {
normalizeQaProviderMode,
type QaProviderMode,
} from "./model-selection.js";
import { resolveQaRuntimeModelPair } from "./model-selection.runtime.js";
import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "./providers/index.js";
import { defaultQaModelForMode } from "./run-config.js";
import {
resolveQaScenarioRequiredProviderMode,
type QaSeedScenarioWithSource,
} from "./scenario-catalog.js";
function normalizeQaSuiteModelRef(input: string | undefined, fallback: string) {
const model = input?.trim();
return model && model.length > 0 ? model : fallback;
}
export function resolveRequestedQaSuiteModels(params: {
alternateModel?: string;
fastMode?: boolean;
@@ -36,14 +31,11 @@ export function resolveRequestedQaSuiteModels(params: {
const providerMode = normalizeQaProviderMode(
params.providerMode ?? selectedProviderMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE,
);
const primaryModel = normalizeQaSuiteModelRef(
params.primaryModel,
defaultQaModelForMode(providerMode),
);
const alternateModel = normalizeQaSuiteModelRef(
params.alternateModel,
defaultQaModelForMode(providerMode, true),
);
const { primaryModel, alternateModel } = resolveQaRuntimeModelPair({
providerMode,
primaryModel: params.primaryModel,
alternateModel: params.alternateModel,
});
return {
alternateModel,
fastMode: params.fastMode ?? isQaFastModeEnabled({ primaryModel, alternateModel }),
@@ -140,9 +140,10 @@ describe("qa suite provider selection", () => {
};
expect(summary.run).toMatchObject({
providerMode: "live-frontier",
primaryModel: expect.stringMatching(/^openai\//),
alternateModel: expect.stringMatching(/^openai\//),
primaryModel: "openai/gpt-5.6",
alternateModel: "openai/gpt-5.6-luna",
});
expect(summary.run.primaryModel).not.toBe(summary.run.alternateModel);
} finally {
await rm(repoRoot, { recursive: true, force: true });
}