diff --git a/extensions/qa-lab/src/scenario-runtime-api.test.ts b/extensions/qa-lab/src/scenario-runtime-api.test.ts index 9ddad794c543..6e031bf45cfd 100644 --- a/extensions/qa-lab/src/scenario-runtime-api.test.ts +++ b/extensions/qa-lab/src/scenario-runtime-api.test.ts @@ -1,7 +1,4 @@ // Qa Lab tests cover scenario runtime api plugin behavior. -import { randomUUID } from "node:crypto"; -import * as fs from "node:fs/promises"; -import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { createQaBusState } from "./bus-state.js"; import type { QaTransportAdapter } from "./qa-transport.js"; @@ -9,87 +6,6 @@ import { createQaScenarioRuntimeApi } from "./scenario-runtime-api.js"; type CreateQaScenarioRuntimeApiParams = Parameters[0]; type QaScenarioRuntimeConstants = CreateQaScenarioRuntimeApiParams["constants"]; -type QaScenarioRuntimeDeps = CreateQaScenarioRuntimeApiParams["deps"]; - -function createDeps(overrides?: Partial): QaScenarioRuntimeDeps { - const fn = vi.fn(); - return { - fs, - path, - sleep: vi.fn(async () => undefined), - randomUUID, - runScenario: fn, - waitForOutboundMessage: fn, - waitForNoOutbound: fn, - waitForNoTransportOutbound: fn, - recentOutboundSummary: fn, - formatConversationTranscript: fn, - readTransportTranscript: fn, - formatTransportTranscript: fn, - fetchJson: fn, - waitForGatewayHealthy: fn, - waitForTransportReady: fn, - browserRequest: fn, - waitForBrowserReady: fn, - browserOpenTab: fn, - browserSnapshot: fn, - browserAct: fn, - webOpenPage: fn, - webWait: fn, - webType: fn, - webSnapshot: fn, - webEvaluate: fn, - waitForConfigRestartSettle: fn, - patchConfig: fn, - applyConfig: fn, - readConfigSnapshot: fn, - restartGatewayWithConfigPatch: fn, - createSession: fn, - readEffectiveTools: fn, - readSkillStatus: fn, - readRawQaSessionStore: fn, - seedQaSessionTranscript: fn, - readGatewayLogs: fn, - markGatewayLogCursor: fn, - scanGatewayLogSentinels: fn, - assertNoGatewayLogSentinels: fn, - readSessionTranscriptSummary: fn, - runQaCli: fn, - extractMediaPathFromText: fn, - resolveGeneratedImagePath: fn, - startAgentRun: fn, - waitForAgentRun: fn, - waitForAgentHistoryReply: fn, - listCronJobs: fn, - waitForCronRunCompletion: fn, - findManagedDreamingCronJob: fn, - readDoctorMemoryStatus: fn, - forceMemoryIndex: fn, - findSkill: fn, - writeWorkspaceSkill: fn, - callPluginToolsMcp: fn, - runAgentPrompt: fn, - ensureImageGenerationConfigured: fn, - handleQaAction: fn, - runRuntimeToolFixture: fn, - extractQaToolPayload: fn, - formatMemoryDreamingDay: fn, - resolveSessionTranscriptsDirForAgent: fn, - activeMemoryToggleKey: fn, - setActiveMemorySessionDisabled: fn, - buildAgentSessionKey: fn, - normalizeLowercaseStringOrEmpty: fn, - formatErrorMessage: fn, - liveTurnTimeoutMs: fn, - resolveQaLiveTurnTimeoutMs: fn, - splitModelRef: fn, - hasDiscoveryLabels: fn, - reportsDiscoveryScopeLeak: fn, - reportsMissingDiscoveryFiles: fn, - hasModelSwitchContinuitySignal: fn, - ...overrides, - }; -} const constants: QaScenarioRuntimeConstants = { imageUnderstandingPngBase64: "png-small", @@ -97,19 +13,6 @@ const constants: QaScenarioRuntimeConstants = { imageUnderstandingValidPngBase64: "png-valid", }; -const browserAndWebRuntimeTools = [ - "browserRequest", - "waitForBrowserReady", - "browserOpenTab", - "browserSnapshot", - "browserAct", - "webOpenPage", - "webWait", - "webType", - "webSnapshot", - "webEvaluate", -] as const; - describe("createQaScenarioRuntimeApi", () => { it("builds a markdown-flow runtime surface from the transport adapter", async () => { const state = createQaBusState(); @@ -173,7 +76,12 @@ describe("createQaScenarioRuntimeApi", () => { }, }, }; - const deps = createDeps({ sleep }); + const deps = { + sleep, + waitForTransportReady: vi.fn(), + waitForAgentHistoryReply: vi.fn(), + browserRequest: vi.fn(), + }; const api = createQaScenarioRuntimeApi({ env, @@ -188,13 +96,8 @@ describe("createQaScenarioRuntimeApi", () => { expect(api.waitForCondition).toBe(waitForCondition); expect(api.waitForChannelReady).toBe(api.waitForTransportReady); expect(api.waitForQaChannelReady).toBe(api.waitForTransportReady); - expect(api.waitForAgentHistoryReply).toBe(deps.waitForAgentHistoryReply); - expect(api.markGatewayLogCursor).toBe(deps.markGatewayLogCursor); - expect(api.assertNoGatewayLogSentinels).toBe(deps.assertNoGatewayLogSentinels); - expect(api.readSessionTranscriptSummary).toBe(deps.readSessionTranscriptSummary); - expect(api.seedQaSessionTranscript).toBe(deps.seedQaSessionTranscript); - for (const toolName of browserAndWebRuntimeTools) { - expect(api[toolName]).toBe(deps[toolName]); + for (const name of Object.keys(deps) as Array) { + expect(api[name]).toBe(deps[name]); } expect(api.getTransportSnapshot()).toEqual(state.getSnapshot()); expect(api.imageUnderstandingPngBase64).toBe("png-small"); diff --git a/extensions/qa-lab/src/scenario-runtime-api.ts b/extensions/qa-lab/src/scenario-runtime-api.ts index 375e50bba9b4..a42db3cc0b1e 100644 --- a/extensions/qa-lab/src/scenario-runtime-api.ts +++ b/extensions/qa-lab/src/scenario-runtime-api.ts @@ -25,7 +25,7 @@ export type QaScenarioRuntimeEnv< transport: TTransport; }; -type QaScenarioRuntimeDeps = { +export type QaScenarioRuntimeDeps = { fs: typeof NodeFs; path: typeof NodePath; sleep: (ms?: number) => Promise; @@ -101,6 +101,8 @@ type QaScenarioRuntimeDeps = { hasModelSwitchContinuitySignal: QaScenarioRuntimeFunction; }; +type QaScenarioRuntimeApiDeps = Pick; + type QaScenarioRuntimeConstants = { imageUnderstandingPngBase64: string; imageUnderstandingLargePngBase64: string; @@ -109,7 +111,7 @@ type QaScenarioRuntimeConstants = { type QaScenarioRuntimeApi< TEnv extends QaScenarioRuntimeEnv = QaScenarioRuntimeEnv, - TDeps extends QaScenarioRuntimeDeps = QaScenarioRuntimeDeps, + TDeps extends QaScenarioRuntimeApiDeps = QaScenarioRuntimeDeps, > = TDeps & { env: TEnv; lab: TEnv["lab"]; @@ -134,7 +136,7 @@ type QaScenarioRuntimeApi< export function createQaScenarioRuntimeApi< TEnv extends QaScenarioRuntimeEnv, - TDeps extends QaScenarioRuntimeDeps, + TDeps extends QaScenarioRuntimeApiDeps, >(params: { env: TEnv; scenario: QaSeedScenarioWithSource; @@ -148,45 +150,26 @@ export function createQaScenarioRuntimeApi< await params.deps.sleep(100); }; - return Object.assign( - { - env: params.env, - lab: params.env.lab, - transport, - state: transportState, - scenario: params.scenario, - config: params.scenario.execution.config ?? {}, - fs: params.deps.fs, - path: params.deps.path, - sleep: params.deps.sleep, - randomUUID: params.deps.randomUUID, - runScenario: params.deps.runScenario, - waitForCondition: transport.waitForCondition, - waitForOutboundMessage: params.deps.waitForOutboundMessage, - waitForNoOutbound: params.deps.waitForNoOutbound, - waitForNoTransportOutbound: params.deps.waitForNoTransportOutbound, - recentOutboundSummary: params.deps.recentOutboundSummary, - formatConversationTranscript: params.deps.formatConversationTranscript, - readTransportTranscript: params.deps.readTransportTranscript, - formatTransportTranscript: params.deps.formatTransportTranscript, - fetchJson: params.deps.fetchJson, - waitForGatewayHealthy: params.deps.waitForGatewayHealthy, - waitForTransportReady: params.deps.waitForTransportReady, - waitForChannelReady: params.deps.waitForTransportReady, - waitForQaChannelReady: params.deps.waitForTransportReady, - }, - params.deps, - { - imageUnderstandingPngBase64: params.constants.imageUnderstandingPngBase64, - imageUnderstandingLargePngBase64: params.constants.imageUnderstandingLargePngBase64, - imageUnderstandingValidPngBase64: params.constants.imageUnderstandingValidPngBase64, - getTransportSnapshot: transportState.getSnapshot.bind(transportState), - resetTransport: resetTransportState, - injectInboundMessage: transportState.addInboundMessage.bind(transportState), - injectOutboundMessage: transportState.addOutboundMessage.bind(transportState), - readTransportMessage: transportState.readMessage.bind(transportState), - resetBus: resetTransportState, - reset: resetTransportState, - }, - ); + return { + ...params.deps, + env: params.env, + lab: params.env.lab, + transport, + state: transportState, + scenario: params.scenario, + config: params.scenario.execution.config ?? {}, + waitForCondition: transport.waitForCondition, + waitForChannelReady: params.deps.waitForTransportReady, + waitForQaChannelReady: params.deps.waitForTransportReady, + imageUnderstandingPngBase64: params.constants.imageUnderstandingPngBase64, + imageUnderstandingLargePngBase64: params.constants.imageUnderstandingLargePngBase64, + imageUnderstandingValidPngBase64: params.constants.imageUnderstandingValidPngBase64, + getTransportSnapshot: transportState.getSnapshot.bind(transportState), + resetTransport: resetTransportState, + injectInboundMessage: transportState.addInboundMessage.bind(transportState), + injectOutboundMessage: transportState.addOutboundMessage.bind(transportState), + readTransportMessage: transportState.readMessage.bind(transportState), + resetBus: resetTransportState, + reset: resetTransportState, + }; } diff --git a/extensions/qa-lab/src/suite-runtime-flow.test.ts b/extensions/qa-lab/src/suite-runtime-flow.test.ts index afbe3363fdbd..6d1279398332 100644 --- a/extensions/qa-lab/src/suite-runtime-flow.test.ts +++ b/extensions/qa-lab/src/suite-runtime-flow.test.ts @@ -4,61 +4,8 @@ import { describe, expect, it, vi } from "vitest"; const createQaScenarioRuntimeApi = vi.hoisted(() => vi.fn()); const runScenarioFlow = vi.hoisted(() => vi.fn(async (params: { api: unknown }) => params.api)); const waitForOutboundMessage = vi.hoisted(() => vi.fn()); -const waitForNoOutbound = vi.hoisted(() => vi.fn()); -const waitForNoTransportOutbound = vi.hoisted(() => vi.fn()); -const recentOutboundSummary = vi.hoisted(() => vi.fn()); -const formatConversationTranscript = vi.hoisted(() => vi.fn()); -const readTransportTranscript = vi.hoisted(() => vi.fn()); -const formatTransportTranscript = vi.hoisted(() => vi.fn()); -const fetchJson = vi.hoisted(() => vi.fn()); -const waitForGatewayHealthy = vi.hoisted(() => vi.fn()); -const waitForTransportReady = vi.hoisted(() => vi.fn()); -const patchConfig = vi.hoisted(() => vi.fn()); -const applyConfig = vi.hoisted(() => vi.fn()); -const readConfigSnapshot = vi.hoisted(() => vi.fn()); -const restartGatewayWithConfigPatch = vi.hoisted(() => vi.fn()); -const waitForConfigRestartSettle = vi.hoisted(() => vi.fn()); -const createSession = vi.hoisted(() => vi.fn()); -const readEffectiveTools = vi.hoisted(() => vi.fn()); -const readSkillStatus = vi.hoisted(() => vi.fn()); -const readRawQaSessionStore = vi.hoisted(() => vi.fn()); -const seedQaSessionTranscript = vi.hoisted(() => vi.fn()); -const readSessionTranscriptSummary = vi.hoisted(() => vi.fn()); -const runQaCli = vi.hoisted(() => vi.fn()); -const extractMediaPathFromText = vi.hoisted(() => vi.fn()); -const resolveGeneratedImagePath = vi.hoisted(() => vi.fn()); -const startAgentRun = vi.hoisted(() => vi.fn()); -const waitForAgentRun = vi.hoisted(() => vi.fn()); -const waitForAgentHistoryReply = vi.hoisted(() => vi.fn()); -const listCronJobs = vi.hoisted(() => vi.fn()); -const findManagedDreamingCronJob = vi.hoisted(() => vi.fn()); -const waitForCronRunCompletion = vi.hoisted(() => vi.fn()); -const readDoctorMemoryStatus = vi.hoisted(() => vi.fn()); -const forceMemoryIndex = vi.hoisted(() => vi.fn()); -const findSkill = vi.hoisted(() => vi.fn()); -const writeWorkspaceSkill = vi.hoisted(() => vi.fn()); -const callPluginToolsMcp = vi.hoisted(() => vi.fn()); -const runAgentPrompt = vi.hoisted(() => vi.fn()); -const ensureImageGenerationConfigured = vi.hoisted(() => vi.fn()); -const handleQaAction = vi.hoisted(() => vi.fn()); const runRuntimeToolFixture = vi.hoisted(() => vi.fn()); -const extractQaToolPayload = vi.hoisted(() => vi.fn()); -const browserRequest = vi.hoisted(() => vi.fn()); -const waitForBrowserReady = vi.hoisted(() => vi.fn()); -const browserOpenTab = vi.hoisted(() => vi.fn()); -const browserSnapshot = vi.hoisted(() => vi.fn()); -const browserAct = vi.hoisted(() => vi.fn()); const webOpenPage = vi.hoisted(() => vi.fn(async () => ({ pageId: "page-1" }))); -const webWait = vi.hoisted(() => vi.fn()); -const webType = vi.hoisted(() => vi.fn()); -const webSnapshot = vi.hoisted(() => vi.fn()); -const webEvaluate = vi.hoisted(() => vi.fn()); -const hasDiscoveryLabels = vi.hoisted(() => vi.fn()); -const reportsDiscoveryScopeLeak = vi.hoisted(() => vi.fn()); -const reportsMissingDiscoveryFiles = vi.hoisted(() => vi.fn()); -const hasModelSwitchContinuitySignal = vi.hoisted(() => vi.fn()); -const scanGatewayLogSentinels = vi.hoisted(() => vi.fn()); -const assertNoGatewayLogSentinels = vi.hoisted(() => vi.fn()); vi.mock("./scenario-runtime-api.js", () => ({ createQaScenarioRuntimeApi, @@ -68,98 +15,34 @@ vi.mock("./scenario-flow-runner.js", () => ({ runScenarioFlow, })); -vi.mock("./suite-runtime-transport.js", () => ({ +vi.mock("./suite-runtime-transport.js", async (importOriginal) => ({ + ...(await importOriginal()), waitForOutboundMessage, - waitForNoOutbound, - waitForNoTransportOutbound, - recentOutboundSummary, - formatConversationTranscript, - readTransportTranscript, - formatTransportTranscript, })); -vi.mock("./suite-runtime-gateway.js", () => ({ - fetchJson, - waitForGatewayHealthy, - waitForTransportReady, - waitForConfigRestartSettle, - patchConfig, - applyConfig, - readConfigSnapshot, - restartGatewayWithConfigPatch, -})); - -vi.mock("./suite-runtime-agent.js", () => ({ - createSession, - readEffectiveTools, - readSkillStatus, - readRawQaSessionStore, - seedQaSessionTranscript, - readSessionTranscriptSummary, - runQaCli, - extractMediaPathFromText, - resolveGeneratedImagePath, - startAgentRun, - waitForAgentRun, - waitForAgentHistoryReply, - listCronJobs, - findManagedDreamingCronJob, - readDoctorMemoryStatus, - forceMemoryIndex, - findSkill, - writeWorkspaceSkill, - callPluginToolsMcp, - runAgentPrompt, - ensureImageGenerationConfigured, - handleQaAction, -})); - -vi.mock("./browser-runtime.js", () => ({ - callQaBrowserRequest: browserRequest, - waitForQaBrowserReady: waitForBrowserReady, - qaBrowserOpenTab: browserOpenTab, - qaBrowserSnapshot: browserSnapshot, - qaBrowserAct: browserAct, -})); - -vi.mock("./web-runtime.js", () => ({ +vi.mock("./web-runtime.js", async (importOriginal) => ({ + ...(await importOriginal()), qaWebOpenPage: webOpenPage, - qaWebWait: webWait, - qaWebType: webType, - qaWebSnapshot: webSnapshot, - qaWebEvaluate: webEvaluate, })); -vi.mock("./cron-run-wait.js", () => ({ - waitForCronRunCompletion, -})); - -vi.mock("./discovery-eval.js", () => ({ - hasDiscoveryLabels, - reportsDiscoveryScopeLeak, - reportsMissingDiscoveryFiles, -})); - -vi.mock("./extract-tool-payload.js", () => ({ - extractQaToolPayload, -})); - -vi.mock("./runtime-tool-fixture.js", () => ({ +vi.mock("./runtime-tool-fixture.js", async (importOriginal) => ({ + ...(await importOriginal()), runRuntimeToolFixture, })); -vi.mock("./model-switch-eval.js", () => ({ - hasModelSwitchContinuitySignal, -})); - -vi.mock("./gateway-log-sentinel.js", () => ({ - scanGatewayLogSentinels, - assertNoGatewayLogSentinels, -})); - +import * as browserRuntime from "./browser-runtime.js"; +import * as cronRunWait from "./cron-run-wait.js"; +import * as discoveryEval from "./discovery-eval.js"; import { QaSuiteScenarioSkipError } from "./errors.js"; +import * as extractToolPayload from "./extract-tool-payload.js"; +import * as modelSwitchEval from "./model-switch-eval.js"; +import type { QaScenarioRuntimeDeps } from "./scenario-runtime-api.js"; +import * as suiteRuntimeAgent from "./suite-runtime-agent.js"; import { runQaSuiteScenarioDefinition, runQaSuiteScenarioSteps } from "./suite-runtime-flow.js"; +import * as suiteRuntimeGateway from "./suite-runtime-gateway.js"; +import * as suiteRuntimeTransport from "./suite-runtime-transport.js"; import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js"; +import * as webRuntime from "./web-runtime.js"; describe("qa suite runtime flow", () => { it("records intentional scenario skips without running later steps", async () => { @@ -269,18 +152,10 @@ describe("qa suite runtime flow", () => { const call = createQaScenarioRuntimeApi.mock.calls[0]?.[0] as { env: typeof env; scenario: typeof scenario; - deps: { - runScenario: typeof runScenario; - waitForTransportReady: typeof waitForTransportReady; + deps: QaScenarioRuntimeDeps & { waitForOutboundMessage: typeof waitForOutboundMessage; markGatewayLogCursor: () => number; - assertNoGatewayLogSentinels: typeof assertNoGatewayLogSentinels; - readSessionTranscriptSummary: typeof readSessionTranscriptSummary; - seedQaSessionTranscript: typeof seedQaSessionTranscript; - findManagedDreamingCronJob: typeof findManagedDreamingCronJob; - forceMemoryIndex: typeof forceMemoryIndex; - runAgentPrompt: typeof runAgentPrompt; - waitForAgentHistoryReply: typeof waitForAgentHistoryReply; + assertNoGatewayLogSentinels: () => void; runRuntimeToolFixture: ( envArg: typeof env, configArg: Record, @@ -296,7 +171,37 @@ describe("qa suite runtime flow", () => { expect(call.env).toBe(env); expect(call.scenario).toBe(scenario); expect(call.deps.runScenario).toBe(runScenario); - expect(call.deps.waitForTransportReady).toBe(waitForTransportReady); + for (const dependencyModule of [ + suiteRuntimeAgent, + suiteRuntimeGateway, + cronRunWait, + discoveryEval, + extractToolPayload, + modelSwitchEval, + ]) { + for (const [name, helper] of Object.entries(dependencyModule)) { + expect((call.deps as Record)[name]).toBe(helper); + } + } + for (const [name, helper] of Object.entries(suiteRuntimeTransport)) { + if (name !== "waitForOutboundMessage") { + expect((call.deps as Record)[name]).toBe(helper); + } + } + const aliasedDependencies = { + browserRequest: browserRuntime.callQaBrowserRequest, + waitForBrowserReady: browserRuntime.waitForQaBrowserReady, + browserOpenTab: browserRuntime.qaBrowserOpenTab, + browserSnapshot: browserRuntime.qaBrowserSnapshot, + browserAct: browserRuntime.qaBrowserAct, + webWait: webRuntime.qaWebWait, + webType: webRuntime.qaWebType, + webSnapshot: webRuntime.qaWebSnapshot, + webEvaluate: webRuntime.qaWebEvaluate, + }; + for (const [name, helper] of Object.entries(aliasedDependencies)) { + expect((call.deps as Record)[name]).toBe(helper); + } expect(call.deps.waitForOutboundMessage).toBeTypeOf("function"); const outboundPredicate = vi.fn(); call.deps.waitForOutboundMessage(env.transport.state, outboundPredicate, 123); @@ -308,22 +213,16 @@ describe("qa suite runtime flow", () => { ); expect(call.deps.markGatewayLogCursor()).toBe(0); expect(() => call.deps.assertNoGatewayLogSentinels()).not.toThrow(); - expect(call.deps.readSessionTranscriptSummary).toBe(readSessionTranscriptSummary); - expect(call.deps.seedQaSessionTranscript).toBe(seedQaSessionTranscript); - expect(call.deps.findManagedDreamingCronJob).toBe(findManagedDreamingCronJob); - expect(call.deps.forceMemoryIndex).toBe(forceMemoryIndex); - expect(call.deps.waitForAgentHistoryReply).toBe(waitForAgentHistoryReply); - expect(call.deps.runAgentPrompt).toBe(runAgentPrompt); await call.deps.runRuntimeToolFixture(env, { toolName: "read" }); expect(runRuntimeToolFixture).toHaveBeenCalledWith( env, { toolName: "read" }, { - createSession, - readEffectiveTools, - runAgentPrompt, - fetchJson, - ensureImageGenerationConfigured, + createSession: suiteRuntimeAgent.createSession, + readEffectiveTools: suiteRuntimeAgent.readEffectiveTools, + runAgentPrompt: suiteRuntimeAgent.runAgentPrompt, + fetchJson: suiteRuntimeGateway.fetchJson, + ensureImageGenerationConfigured: suiteRuntimeAgent.ensureImageGenerationConfigured, }, ); expect(call.constants).toEqual({ diff --git a/extensions/qa-lab/src/suite-runtime-flow.ts b/extensions/qa-lab/src/suite-runtime-flow.ts index 2b1c907084d6..98a68324fab4 100644 --- a/extensions/qa-lab/src/suite-runtime-flow.ts +++ b/extensions/qa-lab/src/suite-runtime-flow.ts @@ -9,79 +9,27 @@ import { resolveSessionTranscriptsDirForAgent } from "openclaw/plugin-sdk/memory import { buildAgentSessionKey } from "openclaw/plugin-sdk/routing"; import { createPluginStateSyncKeyedStore } from "openclaw/plugin-sdk/runtime-doctor"; import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime"; -import { - callQaBrowserRequest, - qaBrowserAct, - qaBrowserOpenTab, - qaBrowserSnapshot, - waitForQaBrowserReady, -} from "./browser-runtime.js"; -import { waitForCronRunCompletion } from "./cron-run-wait.js"; -import { - hasDiscoveryLabels, - reportsDiscoveryScopeLeak, - reportsMissingDiscoveryFiles, -} from "./discovery-eval.js"; +import * as browserRuntime from "./browser-runtime.js"; +import * as cronRunWait from "./cron-run-wait.js"; +import * as discoveryEval from "./discovery-eval.js"; import { QaSuiteScenarioSkipError } from "./errors.js"; -import { extractQaToolPayload } from "./extract-tool-payload.js"; +import * as extractToolPayload from "./extract-tool-payload.js"; import { assertNoGatewayLogSentinels, scanGatewayLogSentinels } from "./gateway-log-sentinel.js"; import { resolveQaLiveTurnTimeoutMs } from "./live-timeout.js"; -import { hasModelSwitchContinuitySignal } from "./model-switch-eval.js"; -import { runRuntimeToolFixture } from "./runtime-tool-fixture.js"; +import * as modelSwitchEval from "./model-switch-eval.js"; +import * as runtimeToolFixture from "./runtime-tool-fixture.js"; import type { QaSeedScenarioWithSource } from "./scenario-catalog.js"; import { runScenarioFlow } from "./scenario-flow-runner.js"; -import { createQaScenarioRuntimeApi, type QaScenarioRuntimeEnv } from "./scenario-runtime-api.js"; import { - callPluginToolsMcp, - createSession, - ensureImageGenerationConfigured, - extractMediaPathFromText, - findSkill, - forceMemoryIndex, - findManagedDreamingCronJob, - handleQaAction, - listCronJobs, - readDoctorMemoryStatus, - readEffectiveTools, - readRawQaSessionStore, - readSessionTranscriptSummary, - readSkillStatus, - resolveGeneratedImagePath, - runAgentPrompt, - runQaCli, - seedQaSessionTranscript, - startAgentRun, - waitForAgentHistoryReply, - waitForAgentRun, - writeWorkspaceSkill, -} from "./suite-runtime-agent.js"; -import { - applyConfig, - fetchJson, - patchConfig, - readConfigSnapshot, - restartGatewayWithConfigPatch, - waitForConfigRestartSettle, - waitForGatewayHealthy, - waitForTransportReady, -} from "./suite-runtime-gateway.js"; -import { - formatConversationTranscript, - formatTransportTranscript, - readTransportTranscript, - recentOutboundSummary, - waitForNoOutbound, - waitForNoTransportOutbound, - waitForOutboundMessage, -} from "./suite-runtime-transport.js"; + createQaScenarioRuntimeApi, + type QaScenarioRuntimeDeps, + type QaScenarioRuntimeEnv, +} from "./scenario-runtime-api.js"; +import * as suiteRuntimeAgent from "./suite-runtime-agent.js"; +import * as suiteRuntimeGateway from "./suite-runtime-gateway.js"; +import * as suiteRuntimeTransport from "./suite-runtime-transport.js"; import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js"; -import { - qaWebEvaluate, - qaWebOpenPage, - qaWebSnapshot, - qaWebType, - qaWebWait, -} from "./web-runtime.js"; +import * as webRuntime from "./web-runtime.js"; type QaSuiteScenarioFlowEnv = { lab: unknown; @@ -122,6 +70,28 @@ function setActiveMemorySessionDisabled( store.delete(key); } +const qaSuiteScenarioIdentityDeps = { + fs, + path, + sleep, + randomUUID, + ...suiteRuntimeAgent, + ...suiteRuntimeGateway, + ...suiteRuntimeTransport, + ...extractToolPayload, + waitForCronRunCompletion: cronRunWait.waitForCronRunCompletion, + hasDiscoveryLabels: discoveryEval.hasDiscoveryLabels, + reportsDiscoveryScopeLeak: discoveryEval.reportsDiscoveryScopeLeak, + reportsMissingDiscoveryFiles: discoveryEval.reportsMissingDiscoveryFiles, + hasModelSwitchContinuitySignal: modelSwitchEval.hasModelSwitchContinuitySignal, + formatMemoryDreamingDay, + resolveSessionTranscriptsDirForAgent, + activeMemoryToggleKey, + setActiveMemorySessionDisabled, + buildAgentSessionKey, + normalizeLowercaseStringOrEmpty, +}; + type QaSuiteStep = { name: string; run: () => Promise; @@ -198,107 +168,59 @@ type QaSuiteScenarioFlowApiParams = QaSuiteScenarioDepsParams & { }; function createQaSuiteScenarioDeps(params: QaSuiteScenarioDepsParams) { - const waitForAccountOutboundMessage: typeof waitForOutboundMessage = ( + const waitForAccountOutboundMessage: typeof suiteRuntimeTransport.waitForOutboundMessage = ( state, predicate, timeoutMs, options, ) => - waitForOutboundMessage(state, predicate, timeoutMs, { + suiteRuntimeTransport.waitForOutboundMessage(state, predicate, timeoutMs, { ...options, accountId: params.env.transport.accountId, }); return { - fs, - path, - sleep, - randomUUID, + ...qaSuiteScenarioIdentityDeps, runScenario: params.runScenario, waitForOutboundMessage: waitForAccountOutboundMessage, - waitForNoOutbound, - waitForNoTransportOutbound, - recentOutboundSummary, - formatConversationTranscript, - readTransportTranscript, - formatTransportTranscript, - fetchJson, - waitForGatewayHealthy, - waitForTransportReady, - browserRequest: callQaBrowserRequest, - waitForBrowserReady: waitForQaBrowserReady, - browserOpenTab: qaBrowserOpenTab, - browserSnapshot: qaBrowserSnapshot, - browserAct: qaBrowserAct, - webOpenPage: async (webParams: Parameters[0]) => { - const opened = await qaWebOpenPage({ ...webParams, repoRoot: params.env.repoRoot }); + browserRequest: browserRuntime.callQaBrowserRequest, + waitForBrowserReady: browserRuntime.waitForQaBrowserReady, + browserOpenTab: browserRuntime.qaBrowserOpenTab, + browserSnapshot: browserRuntime.qaBrowserSnapshot, + browserAct: browserRuntime.qaBrowserAct, + webOpenPage: async (webParams: Parameters[0]) => { + const opened = await webRuntime.qaWebOpenPage({ + ...webParams, + repoRoot: params.env.repoRoot, + }); params.env.webSessionIds.add(opened.pageId); return opened; }, - webWait: qaWebWait, - webType: qaWebType, - webSnapshot: qaWebSnapshot, - webEvaluate: qaWebEvaluate, - waitForConfigRestartSettle, - patchConfig, - applyConfig, - readConfigSnapshot, - restartGatewayWithConfigPatch, - createSession, - readEffectiveTools, - readSkillStatus, - readRawQaSessionStore, - seedQaSessionTranscript, + webWait: webRuntime.qaWebWait, + webType: webRuntime.qaWebType, + webSnapshot: webRuntime.qaWebSnapshot, + webEvaluate: webRuntime.qaWebEvaluate, readGatewayLogs: () => params.env.gateway.logs?.() ?? "", markGatewayLogCursor: () => (params.env.gateway.logs?.() ?? "").length, scanGatewayLogSentinels: (options?: Parameters[1]) => scanGatewayLogSentinels(params.env.gateway.logs?.(), options), assertNoGatewayLogSentinels: (options?: Parameters[1]) => assertNoGatewayLogSentinels(params.env.gateway.logs?.(), options), - readSessionTranscriptSummary, - runQaCli, - extractMediaPathFromText, - resolveGeneratedImagePath, - startAgentRun, - waitForAgentRun, - waitForAgentHistoryReply, - listCronJobs, - findManagedDreamingCronJob, - waitForCronRunCompletion, - readDoctorMemoryStatus, - forceMemoryIndex, - findSkill, - writeWorkspaceSkill, - callPluginToolsMcp, - runAgentPrompt, - ensureImageGenerationConfigured, - handleQaAction, runRuntimeToolFixture: async ( envArg: QaSuiteScenarioFlowEnv, configArg: Record, ) => - runRuntimeToolFixture(envArg, configArg, { - createSession, - readEffectiveTools, - runAgentPrompt, - fetchJson, - ensureImageGenerationConfigured, + runtimeToolFixture.runRuntimeToolFixture(envArg, configArg, { + createSession: suiteRuntimeAgent.createSession, + readEffectiveTools: suiteRuntimeAgent.readEffectiveTools, + runAgentPrompt: suiteRuntimeAgent.runAgentPrompt, + fetchJson: suiteRuntimeGateway.fetchJson, + ensureImageGenerationConfigured: suiteRuntimeAgent.ensureImageGenerationConfigured, }), - extractQaToolPayload, - formatMemoryDreamingDay, - resolveSessionTranscriptsDirForAgent, - activeMemoryToggleKey, - setActiveMemorySessionDisabled, - buildAgentSessionKey, - normalizeLowercaseStringOrEmpty, formatErrorMessage: params.formatErrorMessage, liveTurnTimeoutMs: params.liveTurnTimeoutMs, resolveQaLiveTurnTimeoutMs: params.resolveQaLiveTurnTimeoutMs, splitModelRef: params.splitModelRef, - hasDiscoveryLabels, - reportsDiscoveryScopeLeak, - reportsMissingDiscoveryFiles, - hasModelSwitchContinuitySignal, - }; + } satisfies QaScenarioRuntimeDeps; } function createQaSuiteScenarioFlowApi(params: QaSuiteScenarioFlowApiParams) { @@ -348,7 +270,11 @@ export function createQaSuiteScenarioStepRunner( scenarioTitle: scenario.title, timeoutMs: execution.timeoutMs ?? deps.liveTurnTimeoutMs(env, 60_000), waitForConfigRestartSettle: async (options) => - await waitForConfigRestartSettle(env, options?.restartDelayMs, options?.timeoutMs), + await suiteRuntimeGateway.waitForConfigRestartSettle( + env, + options?.restartDelayMs, + options?.timeoutMs, + ), }); if (prepared) { Object.assign(vars, prepared);