refactor(qa): derive scenario runtime API from deps (#119974)

This commit is contained in:
Peter Steinberger
2026-08-06 08:40:53 -07:00
committed by GitHub
parent 75e5c2694f
commit 9213792cdf
4 changed files with 154 additions and 443 deletions
@@ -1,7 +1,4 @@
// Qa Lab tests cover scenario runtime api plugin behavior.
import { randomUUID } from "node:crypto";
import * as fs from "node:fs/promises";
import path from "node:path";
import { describe, expect, it, vi } from "vitest";
import { createQaBusState } from "./bus-state.js";
import type { QaTransportAdapter } from "./qa-transport.js";
@@ -9,87 +6,6 @@ import { createQaScenarioRuntimeApi } from "./scenario-runtime-api.js";
type CreateQaScenarioRuntimeApiParams = Parameters<typeof createQaScenarioRuntimeApi>[0];
type QaScenarioRuntimeConstants = CreateQaScenarioRuntimeApiParams["constants"];
type QaScenarioRuntimeDeps = CreateQaScenarioRuntimeApiParams["deps"];
function createDeps(overrides?: Partial<QaScenarioRuntimeDeps>): QaScenarioRuntimeDeps {
const fn = vi.fn();
return {
fs,
path,
sleep: vi.fn(async () => undefined),
randomUUID,
runScenario: fn,
waitForOutboundMessage: fn,
waitForNoOutbound: fn,
waitForNoTransportOutbound: fn,
recentOutboundSummary: fn,
formatConversationTranscript: fn,
readTransportTranscript: fn,
formatTransportTranscript: fn,
fetchJson: fn,
waitForGatewayHealthy: fn,
waitForTransportReady: fn,
browserRequest: fn,
waitForBrowserReady: fn,
browserOpenTab: fn,
browserSnapshot: fn,
browserAct: fn,
webOpenPage: fn,
webWait: fn,
webType: fn,
webSnapshot: fn,
webEvaluate: fn,
waitForConfigRestartSettle: fn,
patchConfig: fn,
applyConfig: fn,
readConfigSnapshot: fn,
restartGatewayWithConfigPatch: fn,
createSession: fn,
readEffectiveTools: fn,
readSkillStatus: fn,
readRawQaSessionStore: fn,
seedQaSessionTranscript: fn,
readGatewayLogs: fn,
markGatewayLogCursor: fn,
scanGatewayLogSentinels: fn,
assertNoGatewayLogSentinels: fn,
readSessionTranscriptSummary: fn,
runQaCli: fn,
extractMediaPathFromText: fn,
resolveGeneratedImagePath: fn,
startAgentRun: fn,
waitForAgentRun: fn,
waitForAgentHistoryReply: fn,
listCronJobs: fn,
waitForCronRunCompletion: fn,
findManagedDreamingCronJob: fn,
readDoctorMemoryStatus: fn,
forceMemoryIndex: fn,
findSkill: fn,
writeWorkspaceSkill: fn,
callPluginToolsMcp: fn,
runAgentPrompt: fn,
ensureImageGenerationConfigured: fn,
handleQaAction: fn,
runRuntimeToolFixture: fn,
extractQaToolPayload: fn,
formatMemoryDreamingDay: fn,
resolveSessionTranscriptsDirForAgent: fn,
activeMemoryToggleKey: fn,
setActiveMemorySessionDisabled: fn,
buildAgentSessionKey: fn,
normalizeLowercaseStringOrEmpty: fn,
formatErrorMessage: fn,
liveTurnTimeoutMs: fn,
resolveQaLiveTurnTimeoutMs: fn,
splitModelRef: fn,
hasDiscoveryLabels: fn,
reportsDiscoveryScopeLeak: fn,
reportsMissingDiscoveryFiles: fn,
hasModelSwitchContinuitySignal: fn,
...overrides,
};
}
const constants: QaScenarioRuntimeConstants = {
imageUnderstandingPngBase64: "png-small",
@@ -97,19 +13,6 @@ const constants: QaScenarioRuntimeConstants = {
imageUnderstandingValidPngBase64: "png-valid",
};
const browserAndWebRuntimeTools = [
"browserRequest",
"waitForBrowserReady",
"browserOpenTab",
"browserSnapshot",
"browserAct",
"webOpenPage",
"webWait",
"webType",
"webSnapshot",
"webEvaluate",
] as const;
describe("createQaScenarioRuntimeApi", () => {
it("builds a markdown-flow runtime surface from the transport adapter", async () => {
const state = createQaBusState();
@@ -173,7 +76,12 @@ describe("createQaScenarioRuntimeApi", () => {
},
},
};
const deps = createDeps({ sleep });
const deps = {
sleep,
waitForTransportReady: vi.fn(),
waitForAgentHistoryReply: vi.fn(),
browserRequest: vi.fn(),
};
const api = createQaScenarioRuntimeApi({
env,
@@ -188,13 +96,8 @@ describe("createQaScenarioRuntimeApi", () => {
expect(api.waitForCondition).toBe(waitForCondition);
expect(api.waitForChannelReady).toBe(api.waitForTransportReady);
expect(api.waitForQaChannelReady).toBe(api.waitForTransportReady);
expect(api.waitForAgentHistoryReply).toBe(deps.waitForAgentHistoryReply);
expect(api.markGatewayLogCursor).toBe(deps.markGatewayLogCursor);
expect(api.assertNoGatewayLogSentinels).toBe(deps.assertNoGatewayLogSentinels);
expect(api.readSessionTranscriptSummary).toBe(deps.readSessionTranscriptSummary);
expect(api.seedQaSessionTranscript).toBe(deps.seedQaSessionTranscript);
for (const toolName of browserAndWebRuntimeTools) {
expect(api[toolName]).toBe(deps[toolName]);
for (const name of Object.keys(deps) as Array<keyof typeof deps>) {
expect(api[name]).toBe(deps[name]);
}
expect(api.getTransportSnapshot()).toEqual(state.getSnapshot());
expect(api.imageUnderstandingPngBase64).toBe("png-small");
+27 -44
View File
@@ -25,7 +25,7 @@ export type QaScenarioRuntimeEnv<
transport: TTransport;
};
type QaScenarioRuntimeDeps = {
export type QaScenarioRuntimeDeps = {
fs: typeof NodeFs;
path: typeof NodePath;
sleep: (ms?: number) => Promise<unknown>;
@@ -101,6 +101,8 @@ type QaScenarioRuntimeDeps = {
hasModelSwitchContinuitySignal: QaScenarioRuntimeFunction;
};
type QaScenarioRuntimeApiDeps = Pick<QaScenarioRuntimeDeps, "sleep" | "waitForTransportReady">;
type QaScenarioRuntimeConstants = {
imageUnderstandingPngBase64: string;
imageUnderstandingLargePngBase64: string;
@@ -109,7 +111,7 @@ type QaScenarioRuntimeConstants = {
type QaScenarioRuntimeApi<
TEnv extends QaScenarioRuntimeEnv = QaScenarioRuntimeEnv,
TDeps extends QaScenarioRuntimeDeps = QaScenarioRuntimeDeps,
TDeps extends QaScenarioRuntimeApiDeps = QaScenarioRuntimeDeps,
> = TDeps & {
env: TEnv;
lab: TEnv["lab"];
@@ -134,7 +136,7 @@ type QaScenarioRuntimeApi<
export function createQaScenarioRuntimeApi<
TEnv extends QaScenarioRuntimeEnv,
TDeps extends QaScenarioRuntimeDeps,
TDeps extends QaScenarioRuntimeApiDeps,
>(params: {
env: TEnv;
scenario: QaSeedScenarioWithSource;
@@ -148,45 +150,26 @@ export function createQaScenarioRuntimeApi<
await params.deps.sleep(100);
};
return Object.assign(
{
env: params.env,
lab: params.env.lab,
transport,
state: transportState,
scenario: params.scenario,
config: params.scenario.execution.config ?? {},
fs: params.deps.fs,
path: params.deps.path,
sleep: params.deps.sleep,
randomUUID: params.deps.randomUUID,
runScenario: params.deps.runScenario,
waitForCondition: transport.waitForCondition,
waitForOutboundMessage: params.deps.waitForOutboundMessage,
waitForNoOutbound: params.deps.waitForNoOutbound,
waitForNoTransportOutbound: params.deps.waitForNoTransportOutbound,
recentOutboundSummary: params.deps.recentOutboundSummary,
formatConversationTranscript: params.deps.formatConversationTranscript,
readTransportTranscript: params.deps.readTransportTranscript,
formatTransportTranscript: params.deps.formatTransportTranscript,
fetchJson: params.deps.fetchJson,
waitForGatewayHealthy: params.deps.waitForGatewayHealthy,
waitForTransportReady: params.deps.waitForTransportReady,
waitForChannelReady: params.deps.waitForTransportReady,
waitForQaChannelReady: params.deps.waitForTransportReady,
},
params.deps,
{
imageUnderstandingPngBase64: params.constants.imageUnderstandingPngBase64,
imageUnderstandingLargePngBase64: params.constants.imageUnderstandingLargePngBase64,
imageUnderstandingValidPngBase64: params.constants.imageUnderstandingValidPngBase64,
getTransportSnapshot: transportState.getSnapshot.bind(transportState),
resetTransport: resetTransportState,
injectInboundMessage: transportState.addInboundMessage.bind(transportState),
injectOutboundMessage: transportState.addOutboundMessage.bind(transportState),
readTransportMessage: transportState.readMessage.bind(transportState),
resetBus: resetTransportState,
reset: resetTransportState,
},
);
return {
...params.deps,
env: params.env,
lab: params.env.lab,
transport,
state: transportState,
scenario: params.scenario,
config: params.scenario.execution.config ?? {},
waitForCondition: transport.waitForCondition,
waitForChannelReady: params.deps.waitForTransportReady,
waitForQaChannelReady: params.deps.waitForTransportReady,
imageUnderstandingPngBase64: params.constants.imageUnderstandingPngBase64,
imageUnderstandingLargePngBase64: params.constants.imageUnderstandingLargePngBase64,
imageUnderstandingValidPngBase64: params.constants.imageUnderstandingValidPngBase64,
getTransportSnapshot: transportState.getSnapshot.bind(transportState),
resetTransport: resetTransportState,
injectInboundMessage: transportState.addInboundMessage.bind(transportState),
injectOutboundMessage: transportState.addOutboundMessage.bind(transportState),
readTransportMessage: transportState.readMessage.bind(transportState),
resetBus: resetTransportState,
reset: resetTransportState,
};
}
+54 -155
View File
@@ -4,61 +4,8 @@ import { describe, expect, it, vi } from "vitest";
const createQaScenarioRuntimeApi = vi.hoisted(() => vi.fn());
const runScenarioFlow = vi.hoisted(() => vi.fn(async (params: { api: unknown }) => params.api));
const waitForOutboundMessage = vi.hoisted(() => vi.fn());
const waitForNoOutbound = vi.hoisted(() => vi.fn());
const waitForNoTransportOutbound = vi.hoisted(() => vi.fn());
const recentOutboundSummary = vi.hoisted(() => vi.fn());
const formatConversationTranscript = vi.hoisted(() => vi.fn());
const readTransportTranscript = vi.hoisted(() => vi.fn());
const formatTransportTranscript = vi.hoisted(() => vi.fn());
const fetchJson = vi.hoisted(() => vi.fn());
const waitForGatewayHealthy = vi.hoisted(() => vi.fn());
const waitForTransportReady = vi.hoisted(() => vi.fn());
const patchConfig = vi.hoisted(() => vi.fn());
const applyConfig = vi.hoisted(() => vi.fn());
const readConfigSnapshot = vi.hoisted(() => vi.fn());
const restartGatewayWithConfigPatch = vi.hoisted(() => vi.fn());
const waitForConfigRestartSettle = vi.hoisted(() => vi.fn());
const createSession = vi.hoisted(() => vi.fn());
const readEffectiveTools = vi.hoisted(() => vi.fn());
const readSkillStatus = vi.hoisted(() => vi.fn());
const readRawQaSessionStore = vi.hoisted(() => vi.fn());
const seedQaSessionTranscript = vi.hoisted(() => vi.fn());
const readSessionTranscriptSummary = vi.hoisted(() => vi.fn());
const runQaCli = vi.hoisted(() => vi.fn());
const extractMediaPathFromText = vi.hoisted(() => vi.fn());
const resolveGeneratedImagePath = vi.hoisted(() => vi.fn());
const startAgentRun = vi.hoisted(() => vi.fn());
const waitForAgentRun = vi.hoisted(() => vi.fn());
const waitForAgentHistoryReply = vi.hoisted(() => vi.fn());
const listCronJobs = vi.hoisted(() => vi.fn());
const findManagedDreamingCronJob = vi.hoisted(() => vi.fn());
const waitForCronRunCompletion = vi.hoisted(() => vi.fn());
const readDoctorMemoryStatus = vi.hoisted(() => vi.fn());
const forceMemoryIndex = vi.hoisted(() => vi.fn());
const findSkill = vi.hoisted(() => vi.fn());
const writeWorkspaceSkill = vi.hoisted(() => vi.fn());
const callPluginToolsMcp = vi.hoisted(() => vi.fn());
const runAgentPrompt = vi.hoisted(() => vi.fn());
const ensureImageGenerationConfigured = vi.hoisted(() => vi.fn());
const handleQaAction = vi.hoisted(() => vi.fn());
const runRuntimeToolFixture = vi.hoisted(() => vi.fn());
const extractQaToolPayload = vi.hoisted(() => vi.fn());
const browserRequest = vi.hoisted(() => vi.fn());
const waitForBrowserReady = vi.hoisted(() => vi.fn());
const browserOpenTab = vi.hoisted(() => vi.fn());
const browserSnapshot = vi.hoisted(() => vi.fn());
const browserAct = vi.hoisted(() => vi.fn());
const webOpenPage = vi.hoisted(() => vi.fn(async () => ({ pageId: "page-1" })));
const webWait = vi.hoisted(() => vi.fn());
const webType = vi.hoisted(() => vi.fn());
const webSnapshot = vi.hoisted(() => vi.fn());
const webEvaluate = vi.hoisted(() => vi.fn());
const hasDiscoveryLabels = vi.hoisted(() => vi.fn());
const reportsDiscoveryScopeLeak = vi.hoisted(() => vi.fn());
const reportsMissingDiscoveryFiles = vi.hoisted(() => vi.fn());
const hasModelSwitchContinuitySignal = vi.hoisted(() => vi.fn());
const scanGatewayLogSentinels = vi.hoisted(() => vi.fn());
const assertNoGatewayLogSentinels = vi.hoisted(() => vi.fn());
vi.mock("./scenario-runtime-api.js", () => ({
createQaScenarioRuntimeApi,
@@ -68,98 +15,34 @@ vi.mock("./scenario-flow-runner.js", () => ({
runScenarioFlow,
}));
vi.mock("./suite-runtime-transport.js", () => ({
vi.mock("./suite-runtime-transport.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./suite-runtime-transport.js")>()),
waitForOutboundMessage,
waitForNoOutbound,
waitForNoTransportOutbound,
recentOutboundSummary,
formatConversationTranscript,
readTransportTranscript,
formatTransportTranscript,
}));
vi.mock("./suite-runtime-gateway.js", () => ({
fetchJson,
waitForGatewayHealthy,
waitForTransportReady,
waitForConfigRestartSettle,
patchConfig,
applyConfig,
readConfigSnapshot,
restartGatewayWithConfigPatch,
}));
vi.mock("./suite-runtime-agent.js", () => ({
createSession,
readEffectiveTools,
readSkillStatus,
readRawQaSessionStore,
seedQaSessionTranscript,
readSessionTranscriptSummary,
runQaCli,
extractMediaPathFromText,
resolveGeneratedImagePath,
startAgentRun,
waitForAgentRun,
waitForAgentHistoryReply,
listCronJobs,
findManagedDreamingCronJob,
readDoctorMemoryStatus,
forceMemoryIndex,
findSkill,
writeWorkspaceSkill,
callPluginToolsMcp,
runAgentPrompt,
ensureImageGenerationConfigured,
handleQaAction,
}));
vi.mock("./browser-runtime.js", () => ({
callQaBrowserRequest: browserRequest,
waitForQaBrowserReady: waitForBrowserReady,
qaBrowserOpenTab: browserOpenTab,
qaBrowserSnapshot: browserSnapshot,
qaBrowserAct: browserAct,
}));
vi.mock("./web-runtime.js", () => ({
vi.mock("./web-runtime.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./web-runtime.js")>()),
qaWebOpenPage: webOpenPage,
qaWebWait: webWait,
qaWebType: webType,
qaWebSnapshot: webSnapshot,
qaWebEvaluate: webEvaluate,
}));
vi.mock("./cron-run-wait.js", () => ({
waitForCronRunCompletion,
}));
vi.mock("./discovery-eval.js", () => ({
hasDiscoveryLabels,
reportsDiscoveryScopeLeak,
reportsMissingDiscoveryFiles,
}));
vi.mock("./extract-tool-payload.js", () => ({
extractQaToolPayload,
}));
vi.mock("./runtime-tool-fixture.js", () => ({
vi.mock("./runtime-tool-fixture.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./runtime-tool-fixture.js")>()),
runRuntimeToolFixture,
}));
vi.mock("./model-switch-eval.js", () => ({
hasModelSwitchContinuitySignal,
}));
vi.mock("./gateway-log-sentinel.js", () => ({
scanGatewayLogSentinels,
assertNoGatewayLogSentinels,
}));
import * as browserRuntime from "./browser-runtime.js";
import * as cronRunWait from "./cron-run-wait.js";
import * as discoveryEval from "./discovery-eval.js";
import { QaSuiteScenarioSkipError } from "./errors.js";
import * as extractToolPayload from "./extract-tool-payload.js";
import * as modelSwitchEval from "./model-switch-eval.js";
import type { QaScenarioRuntimeDeps } from "./scenario-runtime-api.js";
import * as suiteRuntimeAgent from "./suite-runtime-agent.js";
import { runQaSuiteScenarioDefinition, runQaSuiteScenarioSteps } from "./suite-runtime-flow.js";
import * as suiteRuntimeGateway from "./suite-runtime-gateway.js";
import * as suiteRuntimeTransport from "./suite-runtime-transport.js";
import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js";
import * as webRuntime from "./web-runtime.js";
describe("qa suite runtime flow", () => {
it("records intentional scenario skips without running later steps", async () => {
@@ -269,18 +152,10 @@ describe("qa suite runtime flow", () => {
const call = createQaScenarioRuntimeApi.mock.calls[0]?.[0] as {
env: typeof env;
scenario: typeof scenario;
deps: {
runScenario: typeof runScenario;
waitForTransportReady: typeof waitForTransportReady;
deps: QaScenarioRuntimeDeps & {
waitForOutboundMessage: typeof waitForOutboundMessage;
markGatewayLogCursor: () => number;
assertNoGatewayLogSentinels: typeof assertNoGatewayLogSentinels;
readSessionTranscriptSummary: typeof readSessionTranscriptSummary;
seedQaSessionTranscript: typeof seedQaSessionTranscript;
findManagedDreamingCronJob: typeof findManagedDreamingCronJob;
forceMemoryIndex: typeof forceMemoryIndex;
runAgentPrompt: typeof runAgentPrompt;
waitForAgentHistoryReply: typeof waitForAgentHistoryReply;
assertNoGatewayLogSentinels: () => void;
runRuntimeToolFixture: (
envArg: typeof env,
configArg: Record<string, unknown>,
@@ -296,7 +171,37 @@ describe("qa suite runtime flow", () => {
expect(call.env).toBe(env);
expect(call.scenario).toBe(scenario);
expect(call.deps.runScenario).toBe(runScenario);
expect(call.deps.waitForTransportReady).toBe(waitForTransportReady);
for (const dependencyModule of [
suiteRuntimeAgent,
suiteRuntimeGateway,
cronRunWait,
discoveryEval,
extractToolPayload,
modelSwitchEval,
]) {
for (const [name, helper] of Object.entries(dependencyModule)) {
expect((call.deps as Record<string, unknown>)[name]).toBe(helper);
}
}
for (const [name, helper] of Object.entries(suiteRuntimeTransport)) {
if (name !== "waitForOutboundMessage") {
expect((call.deps as Record<string, unknown>)[name]).toBe(helper);
}
}
const aliasedDependencies = {
browserRequest: browserRuntime.callQaBrowserRequest,
waitForBrowserReady: browserRuntime.waitForQaBrowserReady,
browserOpenTab: browserRuntime.qaBrowserOpenTab,
browserSnapshot: browserRuntime.qaBrowserSnapshot,
browserAct: browserRuntime.qaBrowserAct,
webWait: webRuntime.qaWebWait,
webType: webRuntime.qaWebType,
webSnapshot: webRuntime.qaWebSnapshot,
webEvaluate: webRuntime.qaWebEvaluate,
};
for (const [name, helper] of Object.entries(aliasedDependencies)) {
expect((call.deps as Record<string, unknown>)[name]).toBe(helper);
}
expect(call.deps.waitForOutboundMessage).toBeTypeOf("function");
const outboundPredicate = vi.fn();
call.deps.waitForOutboundMessage(env.transport.state, outboundPredicate, 123);
@@ -308,22 +213,16 @@ describe("qa suite runtime flow", () => {
);
expect(call.deps.markGatewayLogCursor()).toBe(0);
expect(() => call.deps.assertNoGatewayLogSentinels()).not.toThrow();
expect(call.deps.readSessionTranscriptSummary).toBe(readSessionTranscriptSummary);
expect(call.deps.seedQaSessionTranscript).toBe(seedQaSessionTranscript);
expect(call.deps.findManagedDreamingCronJob).toBe(findManagedDreamingCronJob);
expect(call.deps.forceMemoryIndex).toBe(forceMemoryIndex);
expect(call.deps.waitForAgentHistoryReply).toBe(waitForAgentHistoryReply);
expect(call.deps.runAgentPrompt).toBe(runAgentPrompt);
await call.deps.runRuntimeToolFixture(env, { toolName: "read" });
expect(runRuntimeToolFixture).toHaveBeenCalledWith(
env,
{ toolName: "read" },
{
createSession,
readEffectiveTools,
runAgentPrompt,
fetchJson,
ensureImageGenerationConfigured,
createSession: suiteRuntimeAgent.createSession,
readEffectiveTools: suiteRuntimeAgent.readEffectiveTools,
runAgentPrompt: suiteRuntimeAgent.runAgentPrompt,
fetchJson: suiteRuntimeGateway.fetchJson,
ensureImageGenerationConfigured: suiteRuntimeAgent.ensureImageGenerationConfigured,
},
);
expect(call.constants).toEqual({
+65 -139
View File
@@ -9,79 +9,27 @@ import { resolveSessionTranscriptsDirForAgent } from "openclaw/plugin-sdk/memory
import { buildAgentSessionKey } from "openclaw/plugin-sdk/routing";
import { createPluginStateSyncKeyedStore } from "openclaw/plugin-sdk/runtime-doctor";
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime";
import {
callQaBrowserRequest,
qaBrowserAct,
qaBrowserOpenTab,
qaBrowserSnapshot,
waitForQaBrowserReady,
} from "./browser-runtime.js";
import { waitForCronRunCompletion } from "./cron-run-wait.js";
import {
hasDiscoveryLabels,
reportsDiscoveryScopeLeak,
reportsMissingDiscoveryFiles,
} from "./discovery-eval.js";
import * as browserRuntime from "./browser-runtime.js";
import * as cronRunWait from "./cron-run-wait.js";
import * as discoveryEval from "./discovery-eval.js";
import { QaSuiteScenarioSkipError } from "./errors.js";
import { extractQaToolPayload } from "./extract-tool-payload.js";
import * as extractToolPayload from "./extract-tool-payload.js";
import { assertNoGatewayLogSentinels, scanGatewayLogSentinels } from "./gateway-log-sentinel.js";
import { resolveQaLiveTurnTimeoutMs } from "./live-timeout.js";
import { hasModelSwitchContinuitySignal } from "./model-switch-eval.js";
import { runRuntimeToolFixture } from "./runtime-tool-fixture.js";
import * as modelSwitchEval from "./model-switch-eval.js";
import * as runtimeToolFixture from "./runtime-tool-fixture.js";
import type { QaSeedScenarioWithSource } from "./scenario-catalog.js";
import { runScenarioFlow } from "./scenario-flow-runner.js";
import { createQaScenarioRuntimeApi, type QaScenarioRuntimeEnv } from "./scenario-runtime-api.js";
import {
callPluginToolsMcp,
createSession,
ensureImageGenerationConfigured,
extractMediaPathFromText,
findSkill,
forceMemoryIndex,
findManagedDreamingCronJob,
handleQaAction,
listCronJobs,
readDoctorMemoryStatus,
readEffectiveTools,
readRawQaSessionStore,
readSessionTranscriptSummary,
readSkillStatus,
resolveGeneratedImagePath,
runAgentPrompt,
runQaCli,
seedQaSessionTranscript,
startAgentRun,
waitForAgentHistoryReply,
waitForAgentRun,
writeWorkspaceSkill,
} from "./suite-runtime-agent.js";
import {
applyConfig,
fetchJson,
patchConfig,
readConfigSnapshot,
restartGatewayWithConfigPatch,
waitForConfigRestartSettle,
waitForGatewayHealthy,
waitForTransportReady,
} from "./suite-runtime-gateway.js";
import {
formatConversationTranscript,
formatTransportTranscript,
readTransportTranscript,
recentOutboundSummary,
waitForNoOutbound,
waitForNoTransportOutbound,
waitForOutboundMessage,
} from "./suite-runtime-transport.js";
createQaScenarioRuntimeApi,
type QaScenarioRuntimeDeps,
type QaScenarioRuntimeEnv,
} from "./scenario-runtime-api.js";
import * as suiteRuntimeAgent from "./suite-runtime-agent.js";
import * as suiteRuntimeGateway from "./suite-runtime-gateway.js";
import * as suiteRuntimeTransport from "./suite-runtime-transport.js";
import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js";
import {
qaWebEvaluate,
qaWebOpenPage,
qaWebSnapshot,
qaWebType,
qaWebWait,
} from "./web-runtime.js";
import * as webRuntime from "./web-runtime.js";
type QaSuiteScenarioFlowEnv = {
lab: unknown;
@@ -122,6 +70,28 @@ function setActiveMemorySessionDisabled(
store.delete(key);
}
const qaSuiteScenarioIdentityDeps = {
fs,
path,
sleep,
randomUUID,
...suiteRuntimeAgent,
...suiteRuntimeGateway,
...suiteRuntimeTransport,
...extractToolPayload,
waitForCronRunCompletion: cronRunWait.waitForCronRunCompletion,
hasDiscoveryLabels: discoveryEval.hasDiscoveryLabels,
reportsDiscoveryScopeLeak: discoveryEval.reportsDiscoveryScopeLeak,
reportsMissingDiscoveryFiles: discoveryEval.reportsMissingDiscoveryFiles,
hasModelSwitchContinuitySignal: modelSwitchEval.hasModelSwitchContinuitySignal,
formatMemoryDreamingDay,
resolveSessionTranscriptsDirForAgent,
activeMemoryToggleKey,
setActiveMemorySessionDisabled,
buildAgentSessionKey,
normalizeLowercaseStringOrEmpty,
};
type QaSuiteStep = {
name: string;
run: () => Promise<string | void>;
@@ -198,107 +168,59 @@ type QaSuiteScenarioFlowApiParams = QaSuiteScenarioDepsParams & {
};
function createQaSuiteScenarioDeps(params: QaSuiteScenarioDepsParams) {
const waitForAccountOutboundMessage: typeof waitForOutboundMessage = (
const waitForAccountOutboundMessage: typeof suiteRuntimeTransport.waitForOutboundMessage = (
state,
predicate,
timeoutMs,
options,
) =>
waitForOutboundMessage(state, predicate, timeoutMs, {
suiteRuntimeTransport.waitForOutboundMessage(state, predicate, timeoutMs, {
...options,
accountId: params.env.transport.accountId,
});
return {
fs,
path,
sleep,
randomUUID,
...qaSuiteScenarioIdentityDeps,
runScenario: params.runScenario,
waitForOutboundMessage: waitForAccountOutboundMessage,
waitForNoOutbound,
waitForNoTransportOutbound,
recentOutboundSummary,
formatConversationTranscript,
readTransportTranscript,
formatTransportTranscript,
fetchJson,
waitForGatewayHealthy,
waitForTransportReady,
browserRequest: callQaBrowserRequest,
waitForBrowserReady: waitForQaBrowserReady,
browserOpenTab: qaBrowserOpenTab,
browserSnapshot: qaBrowserSnapshot,
browserAct: qaBrowserAct,
webOpenPage: async (webParams: Parameters<typeof qaWebOpenPage>[0]) => {
const opened = await qaWebOpenPage({ ...webParams, repoRoot: params.env.repoRoot });
browserRequest: browserRuntime.callQaBrowserRequest,
waitForBrowserReady: browserRuntime.waitForQaBrowserReady,
browserOpenTab: browserRuntime.qaBrowserOpenTab,
browserSnapshot: browserRuntime.qaBrowserSnapshot,
browserAct: browserRuntime.qaBrowserAct,
webOpenPage: async (webParams: Parameters<typeof webRuntime.qaWebOpenPage>[0]) => {
const opened = await webRuntime.qaWebOpenPage({
...webParams,
repoRoot: params.env.repoRoot,
});
params.env.webSessionIds.add(opened.pageId);
return opened;
},
webWait: qaWebWait,
webType: qaWebType,
webSnapshot: qaWebSnapshot,
webEvaluate: qaWebEvaluate,
waitForConfigRestartSettle,
patchConfig,
applyConfig,
readConfigSnapshot,
restartGatewayWithConfigPatch,
createSession,
readEffectiveTools,
readSkillStatus,
readRawQaSessionStore,
seedQaSessionTranscript,
webWait: webRuntime.qaWebWait,
webType: webRuntime.qaWebType,
webSnapshot: webRuntime.qaWebSnapshot,
webEvaluate: webRuntime.qaWebEvaluate,
readGatewayLogs: () => params.env.gateway.logs?.() ?? "",
markGatewayLogCursor: () => (params.env.gateway.logs?.() ?? "").length,
scanGatewayLogSentinels: (options?: Parameters<typeof scanGatewayLogSentinels>[1]) =>
scanGatewayLogSentinels(params.env.gateway.logs?.(), options),
assertNoGatewayLogSentinels: (options?: Parameters<typeof assertNoGatewayLogSentinels>[1]) =>
assertNoGatewayLogSentinels(params.env.gateway.logs?.(), options),
readSessionTranscriptSummary,
runQaCli,
extractMediaPathFromText,
resolveGeneratedImagePath,
startAgentRun,
waitForAgentRun,
waitForAgentHistoryReply,
listCronJobs,
findManagedDreamingCronJob,
waitForCronRunCompletion,
readDoctorMemoryStatus,
forceMemoryIndex,
findSkill,
writeWorkspaceSkill,
callPluginToolsMcp,
runAgentPrompt,
ensureImageGenerationConfigured,
handleQaAction,
runRuntimeToolFixture: async (
envArg: QaSuiteScenarioFlowEnv,
configArg: Record<string, unknown>,
) =>
runRuntimeToolFixture(envArg, configArg, {
createSession,
readEffectiveTools,
runAgentPrompt,
fetchJson,
ensureImageGenerationConfigured,
runtimeToolFixture.runRuntimeToolFixture(envArg, configArg, {
createSession: suiteRuntimeAgent.createSession,
readEffectiveTools: suiteRuntimeAgent.readEffectiveTools,
runAgentPrompt: suiteRuntimeAgent.runAgentPrompt,
fetchJson: suiteRuntimeGateway.fetchJson,
ensureImageGenerationConfigured: suiteRuntimeAgent.ensureImageGenerationConfigured,
}),
extractQaToolPayload,
formatMemoryDreamingDay,
resolveSessionTranscriptsDirForAgent,
activeMemoryToggleKey,
setActiveMemorySessionDisabled,
buildAgentSessionKey,
normalizeLowercaseStringOrEmpty,
formatErrorMessage: params.formatErrorMessage,
liveTurnTimeoutMs: params.liveTurnTimeoutMs,
resolveQaLiveTurnTimeoutMs: params.resolveQaLiveTurnTimeoutMs,
splitModelRef: params.splitModelRef,
hasDiscoveryLabels,
reportsDiscoveryScopeLeak,
reportsMissingDiscoveryFiles,
hasModelSwitchContinuitySignal,
};
} satisfies QaScenarioRuntimeDeps;
}
function createQaSuiteScenarioFlowApi(params: QaSuiteScenarioFlowApiParams) {
@@ -348,7 +270,11 @@ export function createQaSuiteScenarioStepRunner(
scenarioTitle: scenario.title,
timeoutMs: execution.timeoutMs ?? deps.liveTurnTimeoutMs(env, 60_000),
waitForConfigRestartSettle: async (options) =>
await waitForConfigRestartSettle(env, options?.restartDelayMs, options?.timeoutMs),
await suiteRuntimeGateway.waitForConfigRestartSettle(
env,
options?.restartDelayMs,
options?.timeoutMs,
),
});
if (prepared) {
Object.assign(vars, prepared);