fix(qa): preserve effective fast mode (#124762)

Amp-Thread-ID: https://ampcode.com/threads/T-01a00b6b-e4e9-74af-bb31-30363fae6c89

Co-authored-by: Amp <amp@ampcode.com>
This commit is contained in:
Peter Steinberger
2026-08-16 12:16:48 -07:00
committed by GitHub
parent 08786d32b2
commit 22dd3d4ed0
7 changed files with 66 additions and 7 deletions
+19 -1
View File
@@ -49,6 +49,7 @@ const {
runQaCoverageReportCommand,
runQaJsonlReplayCommand,
runQaLabSelfCheckCommand,
runQaManualLaneCommand,
runQaProfileCommand,
runQaProviderServerCommand,
runQaSuiteCommand,
@@ -65,6 +66,7 @@ const {
runQaCoverageReportCommand: vi.fn(),
runQaJsonlReplayCommand: vi.fn(),
runQaLabSelfCheckCommand: vi.fn(),
runQaManualLaneCommand: vi.fn(),
runQaProfileCommand: vi.fn(),
runQaProviderServerCommand: vi.fn(),
runQaSuiteCommand: vi.fn(),
@@ -123,6 +125,7 @@ vi.mock("./cli.runtime.js", () => ({
runQaCoverageReportCommand,
runQaJsonlReplayCommand,
runQaLabSelfCheckCommand,
runQaManualLaneCommand,
runQaProfileCommand,
runQaProviderServerCommand,
runQaSuiteCommand,
@@ -141,6 +144,7 @@ describe("qa cli registration", () => {
runQaCoverageReportCommand.mockReset();
runQaJsonlReplayCommand.mockReset();
runQaLabSelfCheckCommand.mockReset();
runQaManualLaneCommand.mockReset();
runQaProfileCommand.mockReset();
runQaProviderServerCommand.mockReset();
runQaSuiteCommand.mockReset();
@@ -880,7 +884,7 @@ describe("qa cli registration", () => {
providerMode: "live-frontier",
primaryModel: undefined,
alternateModel: undefined,
fastMode: false,
fastMode: undefined,
allowFailures: false,
scenarioIds: [],
listScenarios: false,
@@ -890,6 +894,20 @@ describe("qa cli registration", () => {
});
});
it.each([
["suite", ["qa", "suite"]],
["profile", ["qa", "run", "--qa-profile", "smoke-ci"]],
["manual", ["qa", "manual", "--message", "hello"]],
])("preserves omitted --fast intent for %s runs", async (_name, args) => {
await program.parseAsync(["node", "openclaw", ...args]);
const call =
runQaSuiteCommand.mock.calls[0]?.[0] ??
runQaProfileCommand.mock.calls[0]?.[0] ??
runQaManualLaneCommand.mock.calls[0]?.[0];
expect(call?.fastMode).toBeUndefined();
});
it("forwards --list-scenarios for telegram runs", async () => {
await program.parseAsync(["node", "openclaw", "qa", "telegram", "--list-scenarios"]);
+3 -3
View File
@@ -435,7 +435,7 @@ export function registerQaLabCli(program: Command) {
false,
)
.option("--fail-fast", "Stop after the first failed QA scenario")
.option("--fast", "Enable provider fast mode where supported", false);
.option("--fast", "Enable provider fast mode where supported");
qaRun.action(async (opts: QaRunCliOptions, command: Command) => {
validateQaRunMode(opts, command);
if (opts.qaProfile?.trim()) {
@@ -497,7 +497,7 @@ export function registerQaLabCli(program: Command) {
false,
)
.option("--fail-fast", "Stop after the first failed QA scenario")
.option("--fast", "Enable provider fast mode where supported", false)
.option("--fast", "Enable provider fast mode where supported")
.option(
"--thinking <level>",
"Suite thinking default: off|minimal|low|medium|high|xhigh|adaptive|max",
@@ -738,7 +738,7 @@ export function registerQaLabCli(program: Command) {
.option("--provider-mode <mode>", formatQaProviderModeHelp(), DEFAULT_QA_LIVE_PROVIDER_MODE)
.option("--model <ref>", "Primary provider/model ref (defaults by provider mode)")
.option("--alt-model <ref>", "Alternate provider/model ref")
.option("--fast", "Enable provider fast mode where supported", false)
.option("--fast", "Enable provider fast mode where supported")
.option("--timeout-ms <ms>", "Override agent.wait timeout", (value: string) =>
parseQaCliPositiveIntegerOption(value, "--timeout-ms"),
)
@@ -107,7 +107,7 @@ function createSharedLiveTransportQaCliRegistration(
.option("--model <ref>", "Primary provider/model ref")
.option("--alt-model <ref>", "Alternate provider/model ref")
.option("--scenario <id>", params.scenarioHelp, collectStringOption, [])
.option("--fast", "Enable provider fast mode where supported", false);
.option("--fast", "Enable provider fast mode where supported");
if (params.allowFailuresHelp) {
command.option("--allow-failures", params.allowFailuresHelp, false);
@@ -39,7 +39,7 @@ export const liveFrontierProviderDefinition: QaProviderDefinition = {
resolveModelParams: ({ modelRef, fastMode, thinkingDefault }) => ({
transport: "sse",
openaiWsWarmup: false,
...(fastMode === true || isQaFastModeModelRef(modelRef) ? { fastMode: true } : {}),
...((fastMode ?? isQaFastModeModelRef(modelRef)) ? { fastMode: true } : {}),
...(thinkingDefault ? { thinking: thinkingDefault } : {}),
}),
resolveTurnTimeoutMs: ({ fallbackMs, modelRef }) => {
@@ -1,5 +1,7 @@
// QA Lab tests cover suite model-pair resolution.
import { describe, expect, it } from "vitest";
import { buildQaGatewayConfig } from "./qa-gateway-config.js";
import { buildQaSuiteSummaryJson } from "./suite-artifacts.js";
import { resolveRequestedQaSuiteModels } from "./suite-model-selection.js";
describe("resolveRequestedQaSuiteModels", () => {
@@ -29,4 +31,39 @@ describe("resolveRequestedQaSuiteModels", () => {
alternateModel: "openai/gpt-5.6-terra",
});
});
it.each([
[undefined, true],
[true, true],
[false, false],
] as const)(
"keeps effective fast mode aligned between live-frontier config and summary for input %s",
(fastMode, expectedFastMode) => {
const selection = resolveRequestedQaSuiteModels({
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.6-sol",
fastMode,
scenarios: [],
});
const config = buildQaGatewayConfig({
bind: "loopback",
gatewayPort: 18789,
gatewayToken: "test-token",
workspaceDir: "/tmp/qa-workspace",
...selection,
});
const summary = buildQaSuiteSummaryJson({
...selection,
scenarios: [],
startedAt: new Date("2026-08-16T00:00:00.000Z"),
finishedAt: new Date("2026-08-16T00:00:01.000Z"),
concurrency: 1,
});
expect(config.agents?.defaults?.models?.[selection.primaryModel]?.params?.fastMode).toBe(
expectedFastMode ? true : undefined,
);
expect(summary.run.fastMode).toBe(expectedFastMode);
},
);
});
+1 -1
View File
@@ -336,7 +336,7 @@ function registerLiveTransportQaCli(
.option("--model <ref>", "Primary provider/model ref")
.option("--alt-model <ref>", "Alternate provider/model ref")
.option("--scenario <id>", params.scenarioHelp, collectLiveTransportQaStringOption, [])
.option("--fast", "Enable provider fast mode where supported", false);
.option("--fast", "Enable provider fast mode where supported");
if (params.allowFailuresHelp) {
command.option("--allow-failures", params.allowFailuresHelp, false);
+4
View File
@@ -181,6 +181,10 @@ describe("plugin-sdk qa-runtime", () => {
})
.register(qa);
await qa.parseAsync(["node", "openclaw", "telegram"]);
expect(run).toHaveBeenCalledWith(expect.objectContaining({ fastMode: undefined }));
run.mockClear();
await qa.parseAsync([
"node",
"openclaw",