mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(qa): resolve live model pairs after auth selection (#121065)
Centralize QA model-pair construction after resolving the actual primary, preserve explicit and manual single-model selections, and keep omitted live-frontier alternates on the provider-owned OpenAI Sol/Luna policy. Punchcard-Session: amber-workshop-workshop-36 Co-authored-by: Dallin Romney <6581799+RomneyDa@users.noreply.github.com>
This commit is contained in:
@@ -18,6 +18,7 @@ const {
|
||||
buildQaDockerHarnessImage,
|
||||
runQaDockerUp,
|
||||
defaultQaRuntimeModelForMode,
|
||||
resolveQaRuntimeModelPair,
|
||||
readQaScenarioPack,
|
||||
} = vi.hoisted(() => ({
|
||||
runQaManualLane: vi.fn(),
|
||||
@@ -32,6 +33,7 @@ const {
|
||||
runQaDockerUp: vi.fn(),
|
||||
defaultQaRuntimeModelForMode:
|
||||
vi.fn<(mode: string, options?: { alternate?: boolean }) => string>(),
|
||||
resolveQaRuntimeModelPair: vi.fn(),
|
||||
readQaScenarioPack: vi.fn<() => QaScenarioPack>(),
|
||||
}));
|
||||
|
||||
@@ -76,6 +78,7 @@ vi.mock("./docker-up.runtime.js", () => ({
|
||||
|
||||
vi.mock("./model-selection.runtime.js", () => ({
|
||||
defaultQaRuntimeModelForMode,
|
||||
resolveQaRuntimeModelPair,
|
||||
}));
|
||||
|
||||
vi.mock("./scenario-catalog.js", async (importOriginal) => {
|
||||
@@ -106,12 +109,29 @@ import { QaSuiteInfraError } from "./errors.js";
|
||||
import { QA_EVIDENCE_FILENAME } from "./evidence-summary.js";
|
||||
import { runQaTelegramCommand } from "./live-transports/telegram/cli.runtime.js";
|
||||
import { defaultQaModelForMode as defaultQaProviderModelForMode } from "./model-selection.js";
|
||||
import { resolveQaLiveFrontierAlternateModel } from "./providers/live-frontier/model-selection.runtime.js";
|
||||
import type { QaTransportAdapterFactory } from "./qa-transport-registry.js";
|
||||
import type { QaProviderModeInput } from "./run-config.js";
|
||||
import { expandQaScenarioExecutionCells } from "./scenario-lane.js";
|
||||
import type { QaSuiteRunParams } from "./suite.js";
|
||||
|
||||
const DEFAULT_LIVE_FRONTIER_MODEL = defaultQaProviderModelForMode("live-frontier");
|
||||
function resolveMockQaRuntimeModelPair(params: {
|
||||
providerMode: string;
|
||||
primaryModel?: string;
|
||||
alternateModel?: string;
|
||||
}) {
|
||||
const primaryModel =
|
||||
params.primaryModel?.trim() || defaultQaRuntimeModelForMode(params.providerMode);
|
||||
const alternateModel =
|
||||
params.alternateModel?.trim() ||
|
||||
(params.providerMode === "live-frontier"
|
||||
? (resolveQaLiveFrontierAlternateModel(primaryModel) ??
|
||||
defaultQaRuntimeModelForMode(params.providerMode, { alternate: true }))
|
||||
: defaultQaRuntimeModelForMode(params.providerMode, { alternate: true }));
|
||||
return { primaryModel, alternateModel };
|
||||
}
|
||||
|
||||
const QA_PASSING_SUITE_SCENARIO = {
|
||||
name: "channel chat baseline",
|
||||
status: "pass" as const,
|
||||
@@ -340,6 +360,7 @@ describe("qa cli runtime", () => {
|
||||
(mode: string, options?: { alternate?: boolean }) =>
|
||||
defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
|
||||
);
|
||||
resolveQaRuntimeModelPair.mockImplementation(resolveMockQaRuntimeModelPair);
|
||||
readQaScenarioPack.mockClear();
|
||||
runQaSuite.mockImplementation(async (params) => {
|
||||
const observedCells = executionCellsForSuiteParams(params);
|
||||
@@ -3262,39 +3283,43 @@ describe("qa cli runtime", () => {
|
||||
transportId: "qa-channel",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: DEFAULT_LIVE_FRONTIER_MODEL,
|
||||
alternateModel: DEFAULT_LIVE_FRONTIER_MODEL,
|
||||
alternateModel: "openai/gpt-5.6-luna",
|
||||
fastMode: undefined,
|
||||
message: "read qa kickoff and reply short",
|
||||
timeoutMs: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps an explicit manual primary model as the alternate default", async () => {
|
||||
await runQaManualLaneCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "anthropic/claude-sonnet-4-6",
|
||||
message: "read qa kickoff and reply short",
|
||||
});
|
||||
it.each(["anthropic/claude-sonnet-4-6", "openai/gpt-5.6-sol"])(
|
||||
"keeps explicit manual primary %s single-model when the alternate is omitted",
|
||||
async (primaryModel) => {
|
||||
await runQaManualLaneCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel,
|
||||
message: "read qa kickoff and reply short",
|
||||
});
|
||||
|
||||
expect(runQaManualLane).toHaveBeenCalledWith({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
transportId: "qa-channel",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "anthropic/claude-sonnet-4-6",
|
||||
alternateModel: "anthropic/claude-sonnet-4-6",
|
||||
fastMode: undefined,
|
||||
message: "read qa kickoff and reply short",
|
||||
timeoutMs: undefined,
|
||||
});
|
||||
});
|
||||
expect(runQaManualLane).toHaveBeenCalledWith({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
transportId: "qa-channel",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel,
|
||||
alternateModel: primaryModel,
|
||||
fastMode: undefined,
|
||||
message: "read qa kickoff and reply short",
|
||||
timeoutMs: undefined,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("defaults manual frontier runs onto Codex OAuth when the runtime resolver prefers it", async () => {
|
||||
defaultQaRuntimeModelForMode.mockImplementation((mode, options) =>
|
||||
mode === "live-frontier"
|
||||
? "openai/gpt-5.6-luna"
|
||||
: defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
|
||||
);
|
||||
defaultQaRuntimeModelForMode.mockImplementation((mode, options) => {
|
||||
if (mode === "live-frontier" && !options?.alternate) {
|
||||
return "openai/gpt-5.6-luna";
|
||||
}
|
||||
return defaultQaProviderModelForMode(mode as QaProviderModeInput, options);
|
||||
});
|
||||
|
||||
await runQaManualLaneCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
@@ -3306,7 +3331,7 @@ describe("qa cli runtime", () => {
|
||||
transportId: "qa-channel",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: "openai/gpt-5.6-sol",
|
||||
fastMode: undefined,
|
||||
message: "read qa kickoff and reply short",
|
||||
timeoutMs: undefined,
|
||||
|
||||
@@ -46,6 +46,7 @@ import {
|
||||
import { startQaLabServer } from "./lab-server.js";
|
||||
import { listLiveTransportQaAdapterFactories } from "./live-transports/cli.js";
|
||||
import { runQaManualLane } from "./manual-lane.runtime.js";
|
||||
import { resolveQaRuntimeModelPair } from "./model-selection.runtime.js";
|
||||
import { runQaMultipass } from "./multipass.runtime.js";
|
||||
import { qaProfileEvidencePlan, type QaProfileEvidencePlan } from "./profile-evidence-plan.js";
|
||||
import {
|
||||
@@ -193,17 +194,14 @@ function resolveQaManualLaneModels(opts: {
|
||||
primaryModel?: string;
|
||||
alternateModel?: string;
|
||||
}) {
|
||||
const primaryModel = opts.primaryModel?.trim() || defaultQaModelForMode(opts.providerMode);
|
||||
const alternateModel = opts.alternateModel?.trim();
|
||||
return {
|
||||
primaryModel,
|
||||
alternateModel:
|
||||
alternateModel && alternateModel.length > 0
|
||||
? alternateModel
|
||||
: opts.primaryModel?.trim()
|
||||
? primaryModel
|
||||
: defaultQaModelForMode(opts.providerMode, true),
|
||||
};
|
||||
// `qa manual --model` is a one-model probe unless the operator also supplies
|
||||
// `--alt-model`; materialize that contract before shared pair resolution.
|
||||
const explicitPrimaryModel = opts.primaryModel?.trim();
|
||||
return resolveQaRuntimeModelPair({
|
||||
...opts,
|
||||
primaryModel: explicitPrimaryModel,
|
||||
alternateModel: opts.alternateModel?.trim() || explicitPrimaryModel,
|
||||
});
|
||||
}
|
||||
|
||||
function parseQaThinkingLevel(
|
||||
|
||||
@@ -1084,15 +1084,19 @@ describe("buildQaRuntimeEnv", () => {
|
||||
expect(configProfile.provider).toBe("openai");
|
||||
expect(configProfile.mode).toBe("api_key");
|
||||
expect(configProfile.displayName).toBe("QA live openai env credential");
|
||||
expect(Object.values(cfg.auth?.profiles ?? {})).not.toContainEqual(
|
||||
expect.objectContaining({ provider: "anthropic" }),
|
||||
);
|
||||
|
||||
for (const agentId of ["main", "qa"]) {
|
||||
const storeProfile = requireAuthProfile(
|
||||
readAuthProfileStore(stateDir, agentId).profiles,
|
||||
"qa-live-openai-env",
|
||||
);
|
||||
const profiles = readAuthProfileStore(stateDir, agentId).profiles;
|
||||
const storeProfile = requireAuthProfile(profiles, "qa-live-openai-env");
|
||||
expect(storeProfile.type).toBe("api_key");
|
||||
expect(storeProfile.provider).toBe("openai");
|
||||
expect(storeProfile.key).toBe("qa-live-not-a-real-key");
|
||||
expect(Object.values(profiles)).not.toContainEqual(
|
||||
expect.objectContaining({ provider: "anthropic" }),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -18,7 +18,10 @@ vi.mock("openclaw/plugin-sdk/agent-runtime", () => ({
|
||||
listProfilesForProvider,
|
||||
}));
|
||||
|
||||
import { defaultQaRuntimeModelForMode } from "./model-selection.runtime.js";
|
||||
import {
|
||||
defaultQaRuntimeModelForMode,
|
||||
resolveQaRuntimeModelPair,
|
||||
} from "./model-selection.runtime.js";
|
||||
|
||||
describe("qa model selection runtime", () => {
|
||||
beforeEach(() => {
|
||||
@@ -34,26 +37,36 @@ describe("qa model selection runtime", () => {
|
||||
resolveEnvApiKey.mockReturnValue({ apiKey: "sk-test" });
|
||||
|
||||
expect(defaultQaRuntimeModelForMode("live-frontier")).toBe("openai/gpt-5.6");
|
||||
expect(resolveQaRuntimeModelPair({ providerMode: "live-frontier" })).toEqual({
|
||||
primaryModel: "openai/gpt-5.6",
|
||||
alternateModel: "openai/gpt-5.6-luna",
|
||||
});
|
||||
expect(loadAuthProfileStoreForRuntime).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("prefers the Codex OAuth live default when only Codex auth profiles are available", () => {
|
||||
loadAuthProfileStoreForRuntime.mockReturnValue({
|
||||
profiles: {
|
||||
"openai:user@example.com": {
|
||||
provider: "openai",
|
||||
type: "oauth",
|
||||
it.each(["oauth", "token"] as const)(
|
||||
"prefers the Codex live default for a stored %s profile",
|
||||
(type) => {
|
||||
loadAuthProfileStoreForRuntime.mockReturnValue({
|
||||
profiles: {
|
||||
"openai:user@example.com": {
|
||||
provider: "openai",
|
||||
type,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
expect(defaultQaRuntimeModelForMode("live-frontier")).toBe("openai/gpt-5.6-luna");
|
||||
expect(loadAuthProfileStoreForRuntime).toHaveBeenCalledWith(undefined, {
|
||||
readOnly: true,
|
||||
allowKeychainPrompt: false,
|
||||
externalCliProviderIds: ["openai"],
|
||||
});
|
||||
});
|
||||
expect(resolveQaRuntimeModelPair({ providerMode: "live-frontier" })).toEqual({
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: "openai/gpt-5.6-sol",
|
||||
});
|
||||
expect(loadAuthProfileStoreForRuntime).toHaveBeenCalledWith(undefined, {
|
||||
readOnly: true,
|
||||
allowKeychainPrompt: false,
|
||||
externalCliProviderIds: ["openai"],
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps the OpenAI live default when stored OpenAI profiles are available", () => {
|
||||
loadAuthProfileStoreForRuntime.mockReturnValue({
|
||||
@@ -68,6 +81,66 @@ describe("qa model selection runtime", () => {
|
||||
expect(defaultQaRuntimeModelForMode("live-frontier")).toBe("openai/gpt-5.6");
|
||||
});
|
||||
|
||||
it.each(["openai/gpt-5.6", "openai/gpt-5.6-sol"])(
|
||||
"derives Luna after explicit Sol primary %s",
|
||||
(primaryModel) => {
|
||||
expect(resolveQaRuntimeModelPair({ providerMode: "live-frontier", primaryModel })).toEqual({
|
||||
primaryModel,
|
||||
alternateModel: "openai/gpt-5.6-luna",
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("derives Sol after an explicit Luna primary", () => {
|
||||
expect(
|
||||
resolveQaRuntimeModelPair({
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
}),
|
||||
).toEqual({
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: "openai/gpt-5.6-sol",
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back through the provider default for an unmapped primary", () => {
|
||||
expect(
|
||||
resolveQaRuntimeModelPair({
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "anthropic/claude-sonnet-4-6",
|
||||
}),
|
||||
).toEqual({
|
||||
primaryModel: "anthropic/claude-sonnet-4-6",
|
||||
alternateModel: "openai/gpt-5.6",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves an explicit alternate model", () => {
|
||||
expect(
|
||||
resolveQaRuntimeModelPair({
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.6",
|
||||
alternateModel: "openai/gpt-5.6-terra",
|
||||
}),
|
||||
).toEqual({
|
||||
primaryModel: "openai/gpt-5.6",
|
||||
alternateModel: "openai/gpt-5.6-terra",
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["openai/gpt-5.4", "openai/gpt-5.4"],
|
||||
["openai/gpt-5.6", "openai/gpt-5.6-sol"],
|
||||
])("preserves the explicit model pair %s / %s", (primaryModel, alternateModel) => {
|
||||
expect(
|
||||
resolveQaRuntimeModelPair({
|
||||
providerMode: "live-frontier",
|
||||
primaryModel,
|
||||
alternateModel,
|
||||
}),
|
||||
).toEqual({ primaryModel, alternateModel });
|
||||
});
|
||||
|
||||
it("leaves mock defaults unchanged", () => {
|
||||
expect(defaultQaRuntimeModelForMode("mock-openai")).toBe("mock-openai/gpt-5.6-luna");
|
||||
expect(defaultQaRuntimeModelForMode("mock-openai", { alternate: true })).toBe(
|
||||
|
||||
@@ -2,14 +2,14 @@
|
||||
import {
|
||||
defaultQaModelForMode,
|
||||
normalizeQaProviderMode,
|
||||
type QaProviderMode,
|
||||
type QaProviderModeInput,
|
||||
} from "./model-selection.js";
|
||||
import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "./providers/index.js";
|
||||
import { resolveQaLiveFrontierPreferredModel } from "./providers/live-frontier/model-selection.runtime.js";
|
||||
|
||||
function resolveQaPreferredLiveModel() {
|
||||
return resolveQaLiveFrontierPreferredModel();
|
||||
}
|
||||
import {
|
||||
resolveQaLiveFrontierAlternateModel,
|
||||
resolveQaLiveFrontierPreferredModel,
|
||||
} from "./providers/live-frontier/model-selection.runtime.js";
|
||||
|
||||
export function defaultQaRuntimeModelForMode(
|
||||
mode: QaProviderModeInput,
|
||||
@@ -21,10 +21,32 @@ export function defaultQaRuntimeModelForMode(
|
||||
const preferredLiveModel =
|
||||
options?.preferredLiveModel ??
|
||||
(normalizeQaProviderMode(mode) === DEFAULT_QA_LIVE_PROVIDER_MODE
|
||||
? resolveQaPreferredLiveModel()
|
||||
? resolveQaLiveFrontierPreferredModel()
|
||||
: undefined);
|
||||
return defaultQaModelForMode(mode, {
|
||||
...options,
|
||||
preferredLiveModel,
|
||||
});
|
||||
}
|
||||
|
||||
export function resolveQaRuntimeModelPair(params: {
|
||||
providerMode: QaProviderModeInput;
|
||||
primaryModel?: string;
|
||||
alternateModel?: string;
|
||||
resolveDefaultModel?: (mode: QaProviderMode, alternate?: boolean) => string;
|
||||
}) {
|
||||
const providerMode = normalizeQaProviderMode(params.providerMode);
|
||||
const normalizeModel = (model: string | undefined) => model?.trim() || undefined;
|
||||
const resolveDefaultModel =
|
||||
params.resolveDefaultModel ??
|
||||
((mode: QaProviderModeInput, alternate = false) =>
|
||||
defaultQaRuntimeModelForMode(mode, alternate ? { alternate: true } : undefined));
|
||||
const primaryModel = normalizeModel(params.primaryModel) ?? resolveDefaultModel(providerMode);
|
||||
const alternateModel =
|
||||
normalizeModel(params.alternateModel) ??
|
||||
(providerMode === DEFAULT_QA_LIVE_PROVIDER_MODE
|
||||
? (resolveQaLiveFrontierAlternateModel(primaryModel) ??
|
||||
resolveDefaultModel(providerMode, true))
|
||||
: resolveDefaultModel(providerMode, true));
|
||||
return { primaryModel, alternateModel };
|
||||
}
|
||||
|
||||
@@ -7,6 +7,16 @@ import { resolveEnvApiKey } from "openclaw/plugin-sdk/provider-auth";
|
||||
|
||||
const QA_CODEX_OAUTH_LIVE_MODEL = "openai/gpt-5.6-luna";
|
||||
|
||||
export function resolveQaLiveFrontierAlternateModel(primaryModel: string) {
|
||||
const normalized = primaryModel.toLowerCase();
|
||||
if (normalized === QA_CODEX_OAUTH_LIVE_MODEL) {
|
||||
return "openai/gpt-5.6-sol";
|
||||
}
|
||||
return normalized === "openai/gpt-5.6" || normalized === "openai/gpt-5.6-sol"
|
||||
? QA_CODEX_OAUTH_LIVE_MODEL
|
||||
: undefined;
|
||||
}
|
||||
|
||||
export function resolveQaLiveFrontierPreferredModel() {
|
||||
if (resolveEnvApiKey("openai")?.apiKey) {
|
||||
return undefined;
|
||||
|
||||
@@ -270,23 +270,45 @@ describe("buildQaGatewayConfig", () => {
|
||||
providerMode: "live-frontier",
|
||||
fastMode: true,
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: "openai/gpt-5.6-sol",
|
||||
...createQaChannelTransportParams(),
|
||||
});
|
||||
|
||||
expect(getPrimaryModel(cfg.agents?.defaults?.model)).toBe("openai/gpt-5.6-luna");
|
||||
expect(getPrimaryModel(cfg.agents?.entries?.qa?.model)).toBe("openai/gpt-5.6-luna");
|
||||
expect(getModelFallbacks(cfg.agents?.defaults?.model)).toBeUndefined();
|
||||
expect(getModelFallbacks(cfg.agents?.entries?.qa?.model)).toBeUndefined();
|
||||
expect(getModelFallbacks(cfg.agents?.defaults?.model)).toEqual(["openai/gpt-5.6-sol"]);
|
||||
expect(getModelFallbacks(cfg.agents?.entries?.qa?.model)).toEqual(["openai/gpt-5.6-sol"]);
|
||||
expect(cfg.models).toBeUndefined();
|
||||
expect(cfg.memory?.search?.remote).toBeUndefined();
|
||||
expect(cfg.plugins?.allow).toEqual(["acpx", "memory-core", "qa-lab", "openai", "qa-channel"]);
|
||||
expect(cfg.plugins?.allow).not.toContain("anthropic");
|
||||
expect(cfg.plugins?.entries?.openai).toEqual({ enabled: true });
|
||||
expect(cfg.agents?.defaults?.models?.["openai/gpt-5.6-luna"]).toEqual({
|
||||
params: { transport: "sse", openaiWsWarmup: false, fastMode: true },
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["openai/gpt-5.6", "openai/gpt-5.6-luna"],
|
||||
["openai/gpt-5.6-sol", "openai/gpt-5.6-luna"],
|
||||
["openai/gpt-5.6-luna", "openai/gpt-5.6-sol"],
|
||||
])("keeps an omitted live alternate on OpenAI for %s", (primary, alternate) => {
|
||||
const cfg = buildQaGatewayConfig({
|
||||
bind: "loopback",
|
||||
gatewayPort: 18789,
|
||||
gatewayToken: "token",
|
||||
workspaceDir: "/tmp/qa-workspace",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: primary,
|
||||
...createQaChannelTransportParams(),
|
||||
});
|
||||
|
||||
expect(getPrimaryModel(cfg.agents?.defaults?.model)).toBe(primary);
|
||||
expect(getModelFallbacks(cfg.agents?.defaults?.model)).toEqual([alternate]);
|
||||
expect(cfg.plugins?.allow).toContain("openai");
|
||||
expect(cfg.plugins?.allow).not.toContain("anthropic");
|
||||
});
|
||||
|
||||
it("keeps inferred live providers when scenarios require additional plugins", () => {
|
||||
const cfg = buildQaGatewayConfig({
|
||||
bind: "loopback",
|
||||
@@ -350,7 +372,7 @@ describe("buildQaGatewayConfig", () => {
|
||||
providerMode,
|
||||
forcedRuntime: "codex",
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: "openai/gpt-5.6-sol",
|
||||
});
|
||||
|
||||
expect(cfg.plugins?.allow).toContain("codex");
|
||||
@@ -442,7 +464,7 @@ describe("buildQaGatewayConfig", () => {
|
||||
workspaceDir: "/tmp/qa-workspace",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "codex-cli/test-model",
|
||||
alternateModel: "codex-cli/test-model",
|
||||
alternateModel: "codex-cli/test-model-alt",
|
||||
imageGenerationModel: null,
|
||||
enabledPluginIds: ["openai"],
|
||||
...createQaChannelTransportParams(),
|
||||
@@ -462,7 +484,7 @@ describe("buildQaGatewayConfig", () => {
|
||||
workspaceDir: "/tmp/qa-workspace",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "custom-openai/model-a",
|
||||
alternateModel: "custom-openai/model-a",
|
||||
alternateModel: "custom-openai/model-b",
|
||||
imageGenerationModel: null,
|
||||
enabledPluginIds: ["openai"],
|
||||
...createQaChannelTransportParams(),
|
||||
@@ -500,7 +522,7 @@ describe("buildQaGatewayConfig", () => {
|
||||
workspaceDir: "/tmp/qa-workspace",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: "openai/gpt-5.6-sol",
|
||||
thinkingDefault: "xhigh",
|
||||
...createQaChannelTransportParams(),
|
||||
});
|
||||
@@ -509,6 +531,22 @@ describe("buildQaGatewayConfig", () => {
|
||||
expect(cfg.agents?.defaults?.models?.["openai/gpt-5.6-luna"]?.params?.thinking).toBe("xhigh");
|
||||
});
|
||||
|
||||
it("preserves an intentional explicit same-model pair without a fallback", () => {
|
||||
const cfg = buildQaGatewayConfig({
|
||||
bind: "loopback",
|
||||
gatewayPort: 18789,
|
||||
gatewayToken: "token",
|
||||
workspaceDir: "/tmp/qa-workspace",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.4",
|
||||
alternateModel: "openai/gpt-5.4",
|
||||
...createQaChannelTransportParams(),
|
||||
});
|
||||
|
||||
expect(getPrimaryModel(cfg.agents?.defaults?.model)).toBe("openai/gpt-5.4");
|
||||
expect(getModelFallbacks(cfg.agents?.defaults?.model)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("can disable control ui for suite-only gateway children", () => {
|
||||
const cfg = buildQaGatewayConfig({
|
||||
bind: "loopback",
|
||||
|
||||
@@ -4,11 +4,11 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import {
|
||||
defaultQaModelForMode,
|
||||
normalizeQaProviderMode,
|
||||
splitQaModelRef,
|
||||
type QaProviderMode,
|
||||
} from "./model-selection.js";
|
||||
import { resolveQaRuntimeModelPair } from "./model-selection.runtime.js";
|
||||
import { getQaProvider } from "./providers/index.js";
|
||||
import { DEFAULT_QA_PROVIDER_MODE } from "./providers/index.js";
|
||||
import { QA_FRONTIER_PROVIDER_IDS } from "./providers/live-frontier/catalog.js";
|
||||
@@ -37,11 +37,6 @@ export function mergeQaControlUiAllowedOrigins(extraOrigins?: string[]) {
|
||||
return uniqueStrings([...DEFAULT_QA_CONTROL_UI_ALLOWED_ORIGINS, ...normalizedExtra]);
|
||||
}
|
||||
|
||||
function normalizeQaGatewayModelRef(input: string | undefined, fallback: string) {
|
||||
const model = input?.trim();
|
||||
return model && model.length > 0 ? model : fallback;
|
||||
}
|
||||
|
||||
function remapQaMockModelRefForCodex(modelRef: string) {
|
||||
const split = splitQaModelRef(modelRef);
|
||||
return split?.provider === "mock-openai" ? `openai/${split.model}` : modelRef;
|
||||
@@ -78,14 +73,12 @@ export function buildQaGatewayConfig(params: {
|
||||
const providerMode = normalizeQaProviderMode(params.providerMode ?? DEFAULT_QA_PROVIDER_MODE);
|
||||
const provider = getQaProvider(providerMode);
|
||||
const usesCodexMockAppServer = params.forcedRuntime === "codex" && providerMode === "mock-openai";
|
||||
const normalizedPrimaryModel = normalizeQaGatewayModelRef(
|
||||
params.primaryModel,
|
||||
defaultQaModelForMode(providerMode),
|
||||
);
|
||||
const normalizedAlternateModel = normalizeQaGatewayModelRef(
|
||||
params.alternateModel,
|
||||
defaultQaModelForMode(providerMode, { alternate: true }),
|
||||
);
|
||||
const { primaryModel: normalizedPrimaryModel, alternateModel: normalizedAlternateModel } =
|
||||
resolveQaRuntimeModelPair({
|
||||
providerMode,
|
||||
primaryModel: params.primaryModel,
|
||||
alternateModel: params.alternateModel,
|
||||
});
|
||||
const primaryModel = usesCodexMockAppServer
|
||||
? remapQaMockModelRefForCodex(normalizedPrimaryModel)
|
||||
: normalizedPrimaryModel;
|
||||
|
||||
@@ -2,19 +2,22 @@
|
||||
import path from "node:path";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { defaultQaRuntimeModelForMode } = vi.hoisted(() => ({
|
||||
const { defaultQaRuntimeModelForMode, resolveQaRuntimeModelPair } = vi.hoisted(() => ({
|
||||
defaultQaRuntimeModelForMode:
|
||||
vi.fn<(mode: string, options?: { alternate?: boolean }) => string>(),
|
||||
resolveQaRuntimeModelPair: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./model-selection.runtime.js", () => ({
|
||||
defaultQaRuntimeModelForMode,
|
||||
resolveQaRuntimeModelPair,
|
||||
}));
|
||||
import { defaultQaModelForMode as defaultQaProviderModelForMode } from "./model-selection.js";
|
||||
import {
|
||||
resolveQaRunProfileExecutionSelection,
|
||||
resolveQaRunProfileMembership,
|
||||
} from "./profile-planning.js";
|
||||
import { resolveQaLiveFrontierAlternateModel } from "./providers/live-frontier/model-selection.runtime.js";
|
||||
import {
|
||||
createIdleQaRunnerSnapshot,
|
||||
createQaRunOutputDir,
|
||||
@@ -28,7 +31,26 @@ import {
|
||||
type QaScorecardTaxonomyReport,
|
||||
} from "./scorecard-taxonomy.js";
|
||||
|
||||
const DEFAULT_LIVE_FRONTIER_MODEL = defaultQaProviderModelForMode("live-frontier");
|
||||
function resolveMockQaRuntimeModelPair(params: {
|
||||
providerMode: string;
|
||||
primaryModel?: string;
|
||||
alternateModel?: string;
|
||||
resolveDefaultModel?: (mode: string, alternate?: boolean) => string;
|
||||
}) {
|
||||
const resolveDefaultModel =
|
||||
params.resolveDefaultModel ??
|
||||
((mode: string, alternate = false) =>
|
||||
defaultQaRuntimeModelForMode(mode, alternate ? { alternate: true } : undefined));
|
||||
const primaryModel = params.primaryModel?.trim() || resolveDefaultModel(params.providerMode);
|
||||
const alternateModel =
|
||||
params.alternateModel?.trim() ||
|
||||
(params.providerMode === "live-frontier"
|
||||
? (resolveQaLiveFrontierAlternateModel(primaryModel) ??
|
||||
resolveDefaultModel(params.providerMode, true))
|
||||
: resolveDefaultModel(params.providerMode, true));
|
||||
return { primaryModel, alternateModel };
|
||||
}
|
||||
|
||||
const profiles: QaScorecardTaxonomyReport["profiles"] = [
|
||||
{
|
||||
id: "smoke-ci",
|
||||
@@ -92,6 +114,7 @@ describe("qa run config", () => {
|
||||
(mode: string, options?: { alternate?: boolean }) =>
|
||||
defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
|
||||
);
|
||||
resolveQaRuntimeModelPair.mockImplementation(resolveMockQaRuntimeModelPair);
|
||||
});
|
||||
|
||||
it("creates a canonical smoke-profile request without copying profile membership", () => {
|
||||
@@ -132,7 +155,7 @@ describe("qa run config", () => {
|
||||
evidenceMode: "full",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: DEFAULT_LIVE_FRONTIER_MODEL,
|
||||
alternateModel: "openai/gpt-5.6-sol",
|
||||
fastMode: true,
|
||||
runtimePair: null,
|
||||
runtimePairLane: null,
|
||||
@@ -781,11 +804,12 @@ describe("qa run config", () => {
|
||||
});
|
||||
|
||||
it("prefers the Codex OAuth default when the runtime resolver says it is available", () => {
|
||||
defaultQaRuntimeModelForMode.mockImplementation((mode, options) =>
|
||||
mode === "live-frontier"
|
||||
? "openai/gpt-5.6-luna"
|
||||
: defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
|
||||
);
|
||||
defaultQaRuntimeModelForMode.mockImplementation((mode, options) => {
|
||||
if (mode === "live-frontier" && !options?.alternate) {
|
||||
return "openai/gpt-5.6-luna";
|
||||
}
|
||||
return defaultQaProviderModelForMode(mode as QaProviderModeInput, options);
|
||||
});
|
||||
|
||||
expect(normalizeQaRunSelection({ profile: "release" }, scenarios, profiles)).toEqual({
|
||||
profile: "release",
|
||||
@@ -794,7 +818,7 @@ describe("qa run config", () => {
|
||||
evidenceMode: "full",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: "openai/gpt-5.6-sol",
|
||||
fastMode: true,
|
||||
runtimePair: null,
|
||||
runtimePairLane: null,
|
||||
|
||||
@@ -9,7 +9,10 @@ import type {
|
||||
QaLabRunSelection,
|
||||
} from "../runner-contract.js";
|
||||
import { defaultQaModelForMode as defaultStaticQaModelForMode } from "./model-selection.js";
|
||||
import { defaultQaRuntimeModelForMode } from "./model-selection.runtime.js";
|
||||
import {
|
||||
defaultQaRuntimeModelForMode,
|
||||
resolveQaRuntimeModelPair,
|
||||
} from "./model-selection.runtime.js";
|
||||
import {
|
||||
resolveQaRunProfileExecutionSelection,
|
||||
resolveQaRunProfileMembership,
|
||||
@@ -73,8 +76,7 @@ function createDefaultQaRunSelection(
|
||||
channelDriver: profile.channelDriver,
|
||||
evidenceMode: profile.evidenceMode,
|
||||
providerMode,
|
||||
primaryModel: resolveDefaultModel(providerMode),
|
||||
alternateModel: resolveDefaultModel(providerMode, true),
|
||||
...resolveQaRuntimeModelPair({ providerMode, resolveDefaultModel }),
|
||||
fastMode: getQaProvider(providerMode).kind === "live",
|
||||
runtimePair: null,
|
||||
runtimePairLane: null,
|
||||
@@ -93,11 +95,6 @@ export function normalizeQaProviderMode(input: unknown): QaProviderMode {
|
||||
throw new Error(`unknown QA provider mode${details}`);
|
||||
}
|
||||
|
||||
function normalizeModel(input: unknown, fallback: string) {
|
||||
const value = typeof input === "string" ? input.trim() : "";
|
||||
return value || fallback;
|
||||
}
|
||||
|
||||
function normalizeScenarioIds(input: unknown, scenarios: QaSeedScenario[]): string[] | null {
|
||||
if (input === undefined || input === null) {
|
||||
return null;
|
||||
@@ -230,17 +227,18 @@ export function normalizeQaRunSelection(
|
||||
const providerMode = normalizeQaProviderMode(
|
||||
payload.providerMode ?? (profile === "smoke-ci" ? "mock-openai" : undefined),
|
||||
);
|
||||
const models = resolveQaRuntimeModelPair({
|
||||
providerMode,
|
||||
primaryModel: typeof payload.primaryModel === "string" ? payload.primaryModel : undefined,
|
||||
alternateModel: typeof payload.alternateModel === "string" ? payload.alternateModel : undefined,
|
||||
});
|
||||
return {
|
||||
profile,
|
||||
channel: normalizeQaChannel(payload.channel),
|
||||
channelDriver: normalizeQaChannelDriver(payload.channelDriver, profileDefaults.channelDriver),
|
||||
evidenceMode: normalizeQaEvidenceMode(payload.evidenceMode, profileDefaults.evidenceMode),
|
||||
providerMode,
|
||||
primaryModel: normalizeModel(payload.primaryModel, defaultQaModelForMode(providerMode)),
|
||||
alternateModel: normalizeModel(
|
||||
payload.alternateModel,
|
||||
defaultQaModelForMode(providerMode, true),
|
||||
),
|
||||
...models,
|
||||
fastMode: getQaProvider(providerMode).kind === "live" || payload.fastMode === true,
|
||||
runtimePair: normalizeQaRuntimePair(payload.runtimePair),
|
||||
runtimePairLane: normalizeQaRuntimePairLane(payload.runtimePairLane),
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
type QaEvidenceSummaryJson,
|
||||
} from "./evidence-summary.js";
|
||||
import { isQaFastModeEnabled } from "./model-selection.js";
|
||||
import { resolveQaRuntimeModelPair } from "./model-selection.runtime.js";
|
||||
import { DEFAULT_QA_PROVIDER_MODE } from "./providers/index.js";
|
||||
import {
|
||||
defaultQaSuiteConcurrencyForTransport,
|
||||
@@ -765,10 +766,11 @@ async function runUnifiedQaSuite(params: {
|
||||
})
|
||||
: undefined;
|
||||
progress?.start();
|
||||
const primaryModel =
|
||||
params.runParams?.primaryModel?.trim() || defaultQaModelForMode(providerMode);
|
||||
const alternateModel =
|
||||
params.runParams?.alternateModel?.trim() || defaultQaModelForMode(providerMode, true);
|
||||
const { primaryModel, alternateModel } = resolveQaRuntimeModelPair({
|
||||
providerMode,
|
||||
primaryModel: params.runParams?.primaryModel,
|
||||
alternateModel: params.runParams?.alternateModel,
|
||||
});
|
||||
const fastMode =
|
||||
typeof params.runParams?.fastMode === "boolean"
|
||||
? params.runParams.fastMode
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
// QA Lab tests cover suite model-pair resolution.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveRequestedQaSuiteModels } from "./suite-model-selection.js";
|
||||
|
||||
describe("resolveRequestedQaSuiteModels", () => {
|
||||
it("derives Luna after an explicit Sol primary", () => {
|
||||
expect(
|
||||
resolveRequestedQaSuiteModels({
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.6-sol",
|
||||
scenarios: [],
|
||||
}),
|
||||
).toMatchObject({
|
||||
primaryModel: "openai/gpt-5.6-sol",
|
||||
alternateModel: "openai/gpt-5.6-luna",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves an explicit alternate", () => {
|
||||
expect(
|
||||
resolveRequestedQaSuiteModels({
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: "openai/gpt-5.6-terra",
|
||||
scenarios: [],
|
||||
}),
|
||||
).toMatchObject({
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: "openai/gpt-5.6-terra",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -4,18 +4,13 @@ import {
|
||||
normalizeQaProviderMode,
|
||||
type QaProviderMode,
|
||||
} from "./model-selection.js";
|
||||
import { resolveQaRuntimeModelPair } from "./model-selection.runtime.js";
|
||||
import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "./providers/index.js";
|
||||
import { defaultQaModelForMode } from "./run-config.js";
|
||||
import {
|
||||
resolveQaScenarioRequiredProviderMode,
|
||||
type QaSeedScenarioWithSource,
|
||||
} from "./scenario-catalog.js";
|
||||
|
||||
function normalizeQaSuiteModelRef(input: string | undefined, fallback: string) {
|
||||
const model = input?.trim();
|
||||
return model && model.length > 0 ? model : fallback;
|
||||
}
|
||||
|
||||
export function resolveRequestedQaSuiteModels(params: {
|
||||
alternateModel?: string;
|
||||
fastMode?: boolean;
|
||||
@@ -36,14 +31,11 @@ export function resolveRequestedQaSuiteModels(params: {
|
||||
const providerMode = normalizeQaProviderMode(
|
||||
params.providerMode ?? selectedProviderMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE,
|
||||
);
|
||||
const primaryModel = normalizeQaSuiteModelRef(
|
||||
params.primaryModel,
|
||||
defaultQaModelForMode(providerMode),
|
||||
);
|
||||
const alternateModel = normalizeQaSuiteModelRef(
|
||||
params.alternateModel,
|
||||
defaultQaModelForMode(providerMode, true),
|
||||
);
|
||||
const { primaryModel, alternateModel } = resolveQaRuntimeModelPair({
|
||||
providerMode,
|
||||
primaryModel: params.primaryModel,
|
||||
alternateModel: params.alternateModel,
|
||||
});
|
||||
return {
|
||||
alternateModel,
|
||||
fastMode: params.fastMode ?? isQaFastModeEnabled({ primaryModel, alternateModel }),
|
||||
|
||||
@@ -140,9 +140,10 @@ describe("qa suite provider selection", () => {
|
||||
};
|
||||
expect(summary.run).toMatchObject({
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: expect.stringMatching(/^openai\//),
|
||||
alternateModel: expect.stringMatching(/^openai\//),
|
||||
primaryModel: "openai/gpt-5.6",
|
||||
alternateModel: "openai/gpt-5.6-luna",
|
||||
});
|
||||
expect(summary.run.primaryModel).not.toBe(summary.run.alternateModel);
|
||||
} finally {
|
||||
await rm(repoRoot, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user