mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-27 12:56:01 -06:00
fix(qa): preserve effective fast mode (#124762)
Amp-Thread-ID: https://ampcode.com/threads/T-01a00b6b-e4e9-74af-bb31-30363fae6c89 Co-authored-by: Amp <amp@ampcode.com>
This commit is contained in:
committed by
GitHub
parent
08786d32b2
commit
22dd3d4ed0
@@ -49,6 +49,7 @@ const {
|
||||
runQaCoverageReportCommand,
|
||||
runQaJsonlReplayCommand,
|
||||
runQaLabSelfCheckCommand,
|
||||
runQaManualLaneCommand,
|
||||
runQaProfileCommand,
|
||||
runQaProviderServerCommand,
|
||||
runQaSuiteCommand,
|
||||
@@ -65,6 +66,7 @@ const {
|
||||
runQaCoverageReportCommand: vi.fn(),
|
||||
runQaJsonlReplayCommand: vi.fn(),
|
||||
runQaLabSelfCheckCommand: vi.fn(),
|
||||
runQaManualLaneCommand: vi.fn(),
|
||||
runQaProfileCommand: vi.fn(),
|
||||
runQaProviderServerCommand: vi.fn(),
|
||||
runQaSuiteCommand: vi.fn(),
|
||||
@@ -123,6 +125,7 @@ vi.mock("./cli.runtime.js", () => ({
|
||||
runQaCoverageReportCommand,
|
||||
runQaJsonlReplayCommand,
|
||||
runQaLabSelfCheckCommand,
|
||||
runQaManualLaneCommand,
|
||||
runQaProfileCommand,
|
||||
runQaProviderServerCommand,
|
||||
runQaSuiteCommand,
|
||||
@@ -141,6 +144,7 @@ describe("qa cli registration", () => {
|
||||
runQaCoverageReportCommand.mockReset();
|
||||
runQaJsonlReplayCommand.mockReset();
|
||||
runQaLabSelfCheckCommand.mockReset();
|
||||
runQaManualLaneCommand.mockReset();
|
||||
runQaProfileCommand.mockReset();
|
||||
runQaProviderServerCommand.mockReset();
|
||||
runQaSuiteCommand.mockReset();
|
||||
@@ -880,7 +884,7 @@ describe("qa cli registration", () => {
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: undefined,
|
||||
alternateModel: undefined,
|
||||
fastMode: false,
|
||||
fastMode: undefined,
|
||||
allowFailures: false,
|
||||
scenarioIds: [],
|
||||
listScenarios: false,
|
||||
@@ -890,6 +894,20 @@ describe("qa cli registration", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["suite", ["qa", "suite"]],
|
||||
["profile", ["qa", "run", "--qa-profile", "smoke-ci"]],
|
||||
["manual", ["qa", "manual", "--message", "hello"]],
|
||||
])("preserves omitted --fast intent for %s runs", async (_name, args) => {
|
||||
await program.parseAsync(["node", "openclaw", ...args]);
|
||||
|
||||
const call =
|
||||
runQaSuiteCommand.mock.calls[0]?.[0] ??
|
||||
runQaProfileCommand.mock.calls[0]?.[0] ??
|
||||
runQaManualLaneCommand.mock.calls[0]?.[0];
|
||||
expect(call?.fastMode).toBeUndefined();
|
||||
});
|
||||
|
||||
it("forwards --list-scenarios for telegram runs", async () => {
|
||||
await program.parseAsync(["node", "openclaw", "qa", "telegram", "--list-scenarios"]);
|
||||
|
||||
|
||||
@@ -435,7 +435,7 @@ export function registerQaLabCli(program: Command) {
|
||||
false,
|
||||
)
|
||||
.option("--fail-fast", "Stop after the first failed QA scenario")
|
||||
.option("--fast", "Enable provider fast mode where supported", false);
|
||||
.option("--fast", "Enable provider fast mode where supported");
|
||||
qaRun.action(async (opts: QaRunCliOptions, command: Command) => {
|
||||
validateQaRunMode(opts, command);
|
||||
if (opts.qaProfile?.trim()) {
|
||||
@@ -497,7 +497,7 @@ export function registerQaLabCli(program: Command) {
|
||||
false,
|
||||
)
|
||||
.option("--fail-fast", "Stop after the first failed QA scenario")
|
||||
.option("--fast", "Enable provider fast mode where supported", false)
|
||||
.option("--fast", "Enable provider fast mode where supported")
|
||||
.option(
|
||||
"--thinking <level>",
|
||||
"Suite thinking default: off|minimal|low|medium|high|xhigh|adaptive|max",
|
||||
@@ -738,7 +738,7 @@ export function registerQaLabCli(program: Command) {
|
||||
.option("--provider-mode <mode>", formatQaProviderModeHelp(), DEFAULT_QA_LIVE_PROVIDER_MODE)
|
||||
.option("--model <ref>", "Primary provider/model ref (defaults by provider mode)")
|
||||
.option("--alt-model <ref>", "Alternate provider/model ref")
|
||||
.option("--fast", "Enable provider fast mode where supported", false)
|
||||
.option("--fast", "Enable provider fast mode where supported")
|
||||
.option("--timeout-ms <ms>", "Override agent.wait timeout", (value: string) =>
|
||||
parseQaCliPositiveIntegerOption(value, "--timeout-ms"),
|
||||
)
|
||||
|
||||
@@ -107,7 +107,7 @@ function createSharedLiveTransportQaCliRegistration(
|
||||
.option("--model <ref>", "Primary provider/model ref")
|
||||
.option("--alt-model <ref>", "Alternate provider/model ref")
|
||||
.option("--scenario <id>", params.scenarioHelp, collectStringOption, [])
|
||||
.option("--fast", "Enable provider fast mode where supported", false);
|
||||
.option("--fast", "Enable provider fast mode where supported");
|
||||
|
||||
if (params.allowFailuresHelp) {
|
||||
command.option("--allow-failures", params.allowFailuresHelp, false);
|
||||
|
||||
@@ -39,7 +39,7 @@ export const liveFrontierProviderDefinition: QaProviderDefinition = {
|
||||
resolveModelParams: ({ modelRef, fastMode, thinkingDefault }) => ({
|
||||
transport: "sse",
|
||||
openaiWsWarmup: false,
|
||||
...(fastMode === true || isQaFastModeModelRef(modelRef) ? { fastMode: true } : {}),
|
||||
...((fastMode ?? isQaFastModeModelRef(modelRef)) ? { fastMode: true } : {}),
|
||||
...(thinkingDefault ? { thinking: thinkingDefault } : {}),
|
||||
}),
|
||||
resolveTurnTimeoutMs: ({ fallbackMs, modelRef }) => {
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
// QA Lab tests cover suite model-pair resolution.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildQaGatewayConfig } from "./qa-gateway-config.js";
|
||||
import { buildQaSuiteSummaryJson } from "./suite-artifacts.js";
|
||||
import { resolveRequestedQaSuiteModels } from "./suite-model-selection.js";
|
||||
|
||||
describe("resolveRequestedQaSuiteModels", () => {
|
||||
@@ -29,4 +31,39 @@ describe("resolveRequestedQaSuiteModels", () => {
|
||||
alternateModel: "openai/gpt-5.6-terra",
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
[undefined, true],
|
||||
[true, true],
|
||||
[false, false],
|
||||
] as const)(
|
||||
"keeps effective fast mode aligned between live-frontier config and summary for input %s",
|
||||
(fastMode, expectedFastMode) => {
|
||||
const selection = resolveRequestedQaSuiteModels({
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.6-sol",
|
||||
fastMode,
|
||||
scenarios: [],
|
||||
});
|
||||
const config = buildQaGatewayConfig({
|
||||
bind: "loopback",
|
||||
gatewayPort: 18789,
|
||||
gatewayToken: "test-token",
|
||||
workspaceDir: "/tmp/qa-workspace",
|
||||
...selection,
|
||||
});
|
||||
const summary = buildQaSuiteSummaryJson({
|
||||
...selection,
|
||||
scenarios: [],
|
||||
startedAt: new Date("2026-08-16T00:00:00.000Z"),
|
||||
finishedAt: new Date("2026-08-16T00:00:01.000Z"),
|
||||
concurrency: 1,
|
||||
});
|
||||
|
||||
expect(config.agents?.defaults?.models?.[selection.primaryModel]?.params?.fastMode).toBe(
|
||||
expectedFastMode ? true : undefined,
|
||||
);
|
||||
expect(summary.run.fastMode).toBe(expectedFastMode);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
@@ -336,7 +336,7 @@ function registerLiveTransportQaCli(
|
||||
.option("--model <ref>", "Primary provider/model ref")
|
||||
.option("--alt-model <ref>", "Alternate provider/model ref")
|
||||
.option("--scenario <id>", params.scenarioHelp, collectLiveTransportQaStringOption, [])
|
||||
.option("--fast", "Enable provider fast mode where supported", false);
|
||||
.option("--fast", "Enable provider fast mode where supported");
|
||||
|
||||
if (params.allowFailuresHelp) {
|
||||
command.option("--allow-failures", params.allowFailuresHelp, false);
|
||||
|
||||
@@ -181,6 +181,10 @@ describe("plugin-sdk qa-runtime", () => {
|
||||
})
|
||||
.register(qa);
|
||||
|
||||
await qa.parseAsync(["node", "openclaw", "telegram"]);
|
||||
expect(run).toHaveBeenCalledWith(expect.objectContaining({ fastMode: undefined }));
|
||||
run.mockClear();
|
||||
|
||||
await qa.parseAsync([
|
||||
"node",
|
||||
"openclaw",
|
||||
|
||||
Reference in New Issue
Block a user