mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-14 14:43:16 -06:00
refactor(qa): derive scenario runtime API from deps (#119974)
This commit is contained in:
committed by
GitHub
parent
75e5c2694f
commit
9213792cdf
@@ -1,7 +1,4 @@
|
||||
// Qa Lab tests cover scenario runtime api plugin behavior.
|
||||
import { randomUUID } from "node:crypto";
|
||||
import * as fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createQaBusState } from "./bus-state.js";
|
||||
import type { QaTransportAdapter } from "./qa-transport.js";
|
||||
@@ -9,87 +6,6 @@ import { createQaScenarioRuntimeApi } from "./scenario-runtime-api.js";
|
||||
|
||||
type CreateQaScenarioRuntimeApiParams = Parameters<typeof createQaScenarioRuntimeApi>[0];
|
||||
type QaScenarioRuntimeConstants = CreateQaScenarioRuntimeApiParams["constants"];
|
||||
type QaScenarioRuntimeDeps = CreateQaScenarioRuntimeApiParams["deps"];
|
||||
|
||||
function createDeps(overrides?: Partial<QaScenarioRuntimeDeps>): QaScenarioRuntimeDeps {
|
||||
const fn = vi.fn();
|
||||
return {
|
||||
fs,
|
||||
path,
|
||||
sleep: vi.fn(async () => undefined),
|
||||
randomUUID,
|
||||
runScenario: fn,
|
||||
waitForOutboundMessage: fn,
|
||||
waitForNoOutbound: fn,
|
||||
waitForNoTransportOutbound: fn,
|
||||
recentOutboundSummary: fn,
|
||||
formatConversationTranscript: fn,
|
||||
readTransportTranscript: fn,
|
||||
formatTransportTranscript: fn,
|
||||
fetchJson: fn,
|
||||
waitForGatewayHealthy: fn,
|
||||
waitForTransportReady: fn,
|
||||
browserRequest: fn,
|
||||
waitForBrowserReady: fn,
|
||||
browserOpenTab: fn,
|
||||
browserSnapshot: fn,
|
||||
browserAct: fn,
|
||||
webOpenPage: fn,
|
||||
webWait: fn,
|
||||
webType: fn,
|
||||
webSnapshot: fn,
|
||||
webEvaluate: fn,
|
||||
waitForConfigRestartSettle: fn,
|
||||
patchConfig: fn,
|
||||
applyConfig: fn,
|
||||
readConfigSnapshot: fn,
|
||||
restartGatewayWithConfigPatch: fn,
|
||||
createSession: fn,
|
||||
readEffectiveTools: fn,
|
||||
readSkillStatus: fn,
|
||||
readRawQaSessionStore: fn,
|
||||
seedQaSessionTranscript: fn,
|
||||
readGatewayLogs: fn,
|
||||
markGatewayLogCursor: fn,
|
||||
scanGatewayLogSentinels: fn,
|
||||
assertNoGatewayLogSentinels: fn,
|
||||
readSessionTranscriptSummary: fn,
|
||||
runQaCli: fn,
|
||||
extractMediaPathFromText: fn,
|
||||
resolveGeneratedImagePath: fn,
|
||||
startAgentRun: fn,
|
||||
waitForAgentRun: fn,
|
||||
waitForAgentHistoryReply: fn,
|
||||
listCronJobs: fn,
|
||||
waitForCronRunCompletion: fn,
|
||||
findManagedDreamingCronJob: fn,
|
||||
readDoctorMemoryStatus: fn,
|
||||
forceMemoryIndex: fn,
|
||||
findSkill: fn,
|
||||
writeWorkspaceSkill: fn,
|
||||
callPluginToolsMcp: fn,
|
||||
runAgentPrompt: fn,
|
||||
ensureImageGenerationConfigured: fn,
|
||||
handleQaAction: fn,
|
||||
runRuntimeToolFixture: fn,
|
||||
extractQaToolPayload: fn,
|
||||
formatMemoryDreamingDay: fn,
|
||||
resolveSessionTranscriptsDirForAgent: fn,
|
||||
activeMemoryToggleKey: fn,
|
||||
setActiveMemorySessionDisabled: fn,
|
||||
buildAgentSessionKey: fn,
|
||||
normalizeLowercaseStringOrEmpty: fn,
|
||||
formatErrorMessage: fn,
|
||||
liveTurnTimeoutMs: fn,
|
||||
resolveQaLiveTurnTimeoutMs: fn,
|
||||
splitModelRef: fn,
|
||||
hasDiscoveryLabels: fn,
|
||||
reportsDiscoveryScopeLeak: fn,
|
||||
reportsMissingDiscoveryFiles: fn,
|
||||
hasModelSwitchContinuitySignal: fn,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
const constants: QaScenarioRuntimeConstants = {
|
||||
imageUnderstandingPngBase64: "png-small",
|
||||
@@ -97,19 +13,6 @@ const constants: QaScenarioRuntimeConstants = {
|
||||
imageUnderstandingValidPngBase64: "png-valid",
|
||||
};
|
||||
|
||||
const browserAndWebRuntimeTools = [
|
||||
"browserRequest",
|
||||
"waitForBrowserReady",
|
||||
"browserOpenTab",
|
||||
"browserSnapshot",
|
||||
"browserAct",
|
||||
"webOpenPage",
|
||||
"webWait",
|
||||
"webType",
|
||||
"webSnapshot",
|
||||
"webEvaluate",
|
||||
] as const;
|
||||
|
||||
describe("createQaScenarioRuntimeApi", () => {
|
||||
it("builds a markdown-flow runtime surface from the transport adapter", async () => {
|
||||
const state = createQaBusState();
|
||||
@@ -173,7 +76,12 @@ describe("createQaScenarioRuntimeApi", () => {
|
||||
},
|
||||
},
|
||||
};
|
||||
const deps = createDeps({ sleep });
|
||||
const deps = {
|
||||
sleep,
|
||||
waitForTransportReady: vi.fn(),
|
||||
waitForAgentHistoryReply: vi.fn(),
|
||||
browserRequest: vi.fn(),
|
||||
};
|
||||
|
||||
const api = createQaScenarioRuntimeApi({
|
||||
env,
|
||||
@@ -188,13 +96,8 @@ describe("createQaScenarioRuntimeApi", () => {
|
||||
expect(api.waitForCondition).toBe(waitForCondition);
|
||||
expect(api.waitForChannelReady).toBe(api.waitForTransportReady);
|
||||
expect(api.waitForQaChannelReady).toBe(api.waitForTransportReady);
|
||||
expect(api.waitForAgentHistoryReply).toBe(deps.waitForAgentHistoryReply);
|
||||
expect(api.markGatewayLogCursor).toBe(deps.markGatewayLogCursor);
|
||||
expect(api.assertNoGatewayLogSentinels).toBe(deps.assertNoGatewayLogSentinels);
|
||||
expect(api.readSessionTranscriptSummary).toBe(deps.readSessionTranscriptSummary);
|
||||
expect(api.seedQaSessionTranscript).toBe(deps.seedQaSessionTranscript);
|
||||
for (const toolName of browserAndWebRuntimeTools) {
|
||||
expect(api[toolName]).toBe(deps[toolName]);
|
||||
for (const name of Object.keys(deps) as Array<keyof typeof deps>) {
|
||||
expect(api[name]).toBe(deps[name]);
|
||||
}
|
||||
expect(api.getTransportSnapshot()).toEqual(state.getSnapshot());
|
||||
expect(api.imageUnderstandingPngBase64).toBe("png-small");
|
||||
|
||||
@@ -25,7 +25,7 @@ export type QaScenarioRuntimeEnv<
|
||||
transport: TTransport;
|
||||
};
|
||||
|
||||
type QaScenarioRuntimeDeps = {
|
||||
export type QaScenarioRuntimeDeps = {
|
||||
fs: typeof NodeFs;
|
||||
path: typeof NodePath;
|
||||
sleep: (ms?: number) => Promise<unknown>;
|
||||
@@ -101,6 +101,8 @@ type QaScenarioRuntimeDeps = {
|
||||
hasModelSwitchContinuitySignal: QaScenarioRuntimeFunction;
|
||||
};
|
||||
|
||||
type QaScenarioRuntimeApiDeps = Pick<QaScenarioRuntimeDeps, "sleep" | "waitForTransportReady">;
|
||||
|
||||
type QaScenarioRuntimeConstants = {
|
||||
imageUnderstandingPngBase64: string;
|
||||
imageUnderstandingLargePngBase64: string;
|
||||
@@ -109,7 +111,7 @@ type QaScenarioRuntimeConstants = {
|
||||
|
||||
type QaScenarioRuntimeApi<
|
||||
TEnv extends QaScenarioRuntimeEnv = QaScenarioRuntimeEnv,
|
||||
TDeps extends QaScenarioRuntimeDeps = QaScenarioRuntimeDeps,
|
||||
TDeps extends QaScenarioRuntimeApiDeps = QaScenarioRuntimeDeps,
|
||||
> = TDeps & {
|
||||
env: TEnv;
|
||||
lab: TEnv["lab"];
|
||||
@@ -134,7 +136,7 @@ type QaScenarioRuntimeApi<
|
||||
|
||||
export function createQaScenarioRuntimeApi<
|
||||
TEnv extends QaScenarioRuntimeEnv,
|
||||
TDeps extends QaScenarioRuntimeDeps,
|
||||
TDeps extends QaScenarioRuntimeApiDeps,
|
||||
>(params: {
|
||||
env: TEnv;
|
||||
scenario: QaSeedScenarioWithSource;
|
||||
@@ -148,45 +150,26 @@ export function createQaScenarioRuntimeApi<
|
||||
await params.deps.sleep(100);
|
||||
};
|
||||
|
||||
return Object.assign(
|
||||
{
|
||||
env: params.env,
|
||||
lab: params.env.lab,
|
||||
transport,
|
||||
state: transportState,
|
||||
scenario: params.scenario,
|
||||
config: params.scenario.execution.config ?? {},
|
||||
fs: params.deps.fs,
|
||||
path: params.deps.path,
|
||||
sleep: params.deps.sleep,
|
||||
randomUUID: params.deps.randomUUID,
|
||||
runScenario: params.deps.runScenario,
|
||||
waitForCondition: transport.waitForCondition,
|
||||
waitForOutboundMessage: params.deps.waitForOutboundMessage,
|
||||
waitForNoOutbound: params.deps.waitForNoOutbound,
|
||||
waitForNoTransportOutbound: params.deps.waitForNoTransportOutbound,
|
||||
recentOutboundSummary: params.deps.recentOutboundSummary,
|
||||
formatConversationTranscript: params.deps.formatConversationTranscript,
|
||||
readTransportTranscript: params.deps.readTransportTranscript,
|
||||
formatTransportTranscript: params.deps.formatTransportTranscript,
|
||||
fetchJson: params.deps.fetchJson,
|
||||
waitForGatewayHealthy: params.deps.waitForGatewayHealthy,
|
||||
waitForTransportReady: params.deps.waitForTransportReady,
|
||||
waitForChannelReady: params.deps.waitForTransportReady,
|
||||
waitForQaChannelReady: params.deps.waitForTransportReady,
|
||||
},
|
||||
params.deps,
|
||||
{
|
||||
imageUnderstandingPngBase64: params.constants.imageUnderstandingPngBase64,
|
||||
imageUnderstandingLargePngBase64: params.constants.imageUnderstandingLargePngBase64,
|
||||
imageUnderstandingValidPngBase64: params.constants.imageUnderstandingValidPngBase64,
|
||||
getTransportSnapshot: transportState.getSnapshot.bind(transportState),
|
||||
resetTransport: resetTransportState,
|
||||
injectInboundMessage: transportState.addInboundMessage.bind(transportState),
|
||||
injectOutboundMessage: transportState.addOutboundMessage.bind(transportState),
|
||||
readTransportMessage: transportState.readMessage.bind(transportState),
|
||||
resetBus: resetTransportState,
|
||||
reset: resetTransportState,
|
||||
},
|
||||
);
|
||||
return {
|
||||
...params.deps,
|
||||
env: params.env,
|
||||
lab: params.env.lab,
|
||||
transport,
|
||||
state: transportState,
|
||||
scenario: params.scenario,
|
||||
config: params.scenario.execution.config ?? {},
|
||||
waitForCondition: transport.waitForCondition,
|
||||
waitForChannelReady: params.deps.waitForTransportReady,
|
||||
waitForQaChannelReady: params.deps.waitForTransportReady,
|
||||
imageUnderstandingPngBase64: params.constants.imageUnderstandingPngBase64,
|
||||
imageUnderstandingLargePngBase64: params.constants.imageUnderstandingLargePngBase64,
|
||||
imageUnderstandingValidPngBase64: params.constants.imageUnderstandingValidPngBase64,
|
||||
getTransportSnapshot: transportState.getSnapshot.bind(transportState),
|
||||
resetTransport: resetTransportState,
|
||||
injectInboundMessage: transportState.addInboundMessage.bind(transportState),
|
||||
injectOutboundMessage: transportState.addOutboundMessage.bind(transportState),
|
||||
readTransportMessage: transportState.readMessage.bind(transportState),
|
||||
resetBus: resetTransportState,
|
||||
reset: resetTransportState,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -4,61 +4,8 @@ import { describe, expect, it, vi } from "vitest";
|
||||
const createQaScenarioRuntimeApi = vi.hoisted(() => vi.fn());
|
||||
const runScenarioFlow = vi.hoisted(() => vi.fn(async (params: { api: unknown }) => params.api));
|
||||
const waitForOutboundMessage = vi.hoisted(() => vi.fn());
|
||||
const waitForNoOutbound = vi.hoisted(() => vi.fn());
|
||||
const waitForNoTransportOutbound = vi.hoisted(() => vi.fn());
|
||||
const recentOutboundSummary = vi.hoisted(() => vi.fn());
|
||||
const formatConversationTranscript = vi.hoisted(() => vi.fn());
|
||||
const readTransportTranscript = vi.hoisted(() => vi.fn());
|
||||
const formatTransportTranscript = vi.hoisted(() => vi.fn());
|
||||
const fetchJson = vi.hoisted(() => vi.fn());
|
||||
const waitForGatewayHealthy = vi.hoisted(() => vi.fn());
|
||||
const waitForTransportReady = vi.hoisted(() => vi.fn());
|
||||
const patchConfig = vi.hoisted(() => vi.fn());
|
||||
const applyConfig = vi.hoisted(() => vi.fn());
|
||||
const readConfigSnapshot = vi.hoisted(() => vi.fn());
|
||||
const restartGatewayWithConfigPatch = vi.hoisted(() => vi.fn());
|
||||
const waitForConfigRestartSettle = vi.hoisted(() => vi.fn());
|
||||
const createSession = vi.hoisted(() => vi.fn());
|
||||
const readEffectiveTools = vi.hoisted(() => vi.fn());
|
||||
const readSkillStatus = vi.hoisted(() => vi.fn());
|
||||
const readRawQaSessionStore = vi.hoisted(() => vi.fn());
|
||||
const seedQaSessionTranscript = vi.hoisted(() => vi.fn());
|
||||
const readSessionTranscriptSummary = vi.hoisted(() => vi.fn());
|
||||
const runQaCli = vi.hoisted(() => vi.fn());
|
||||
const extractMediaPathFromText = vi.hoisted(() => vi.fn());
|
||||
const resolveGeneratedImagePath = vi.hoisted(() => vi.fn());
|
||||
const startAgentRun = vi.hoisted(() => vi.fn());
|
||||
const waitForAgentRun = vi.hoisted(() => vi.fn());
|
||||
const waitForAgentHistoryReply = vi.hoisted(() => vi.fn());
|
||||
const listCronJobs = vi.hoisted(() => vi.fn());
|
||||
const findManagedDreamingCronJob = vi.hoisted(() => vi.fn());
|
||||
const waitForCronRunCompletion = vi.hoisted(() => vi.fn());
|
||||
const readDoctorMemoryStatus = vi.hoisted(() => vi.fn());
|
||||
const forceMemoryIndex = vi.hoisted(() => vi.fn());
|
||||
const findSkill = vi.hoisted(() => vi.fn());
|
||||
const writeWorkspaceSkill = vi.hoisted(() => vi.fn());
|
||||
const callPluginToolsMcp = vi.hoisted(() => vi.fn());
|
||||
const runAgentPrompt = vi.hoisted(() => vi.fn());
|
||||
const ensureImageGenerationConfigured = vi.hoisted(() => vi.fn());
|
||||
const handleQaAction = vi.hoisted(() => vi.fn());
|
||||
const runRuntimeToolFixture = vi.hoisted(() => vi.fn());
|
||||
const extractQaToolPayload = vi.hoisted(() => vi.fn());
|
||||
const browserRequest = vi.hoisted(() => vi.fn());
|
||||
const waitForBrowserReady = vi.hoisted(() => vi.fn());
|
||||
const browserOpenTab = vi.hoisted(() => vi.fn());
|
||||
const browserSnapshot = vi.hoisted(() => vi.fn());
|
||||
const browserAct = vi.hoisted(() => vi.fn());
|
||||
const webOpenPage = vi.hoisted(() => vi.fn(async () => ({ pageId: "page-1" })));
|
||||
const webWait = vi.hoisted(() => vi.fn());
|
||||
const webType = vi.hoisted(() => vi.fn());
|
||||
const webSnapshot = vi.hoisted(() => vi.fn());
|
||||
const webEvaluate = vi.hoisted(() => vi.fn());
|
||||
const hasDiscoveryLabels = vi.hoisted(() => vi.fn());
|
||||
const reportsDiscoveryScopeLeak = vi.hoisted(() => vi.fn());
|
||||
const reportsMissingDiscoveryFiles = vi.hoisted(() => vi.fn());
|
||||
const hasModelSwitchContinuitySignal = vi.hoisted(() => vi.fn());
|
||||
const scanGatewayLogSentinels = vi.hoisted(() => vi.fn());
|
||||
const assertNoGatewayLogSentinels = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("./scenario-runtime-api.js", () => ({
|
||||
createQaScenarioRuntimeApi,
|
||||
@@ -68,98 +15,34 @@ vi.mock("./scenario-flow-runner.js", () => ({
|
||||
runScenarioFlow,
|
||||
}));
|
||||
|
||||
vi.mock("./suite-runtime-transport.js", () => ({
|
||||
vi.mock("./suite-runtime-transport.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("./suite-runtime-transport.js")>()),
|
||||
waitForOutboundMessage,
|
||||
waitForNoOutbound,
|
||||
waitForNoTransportOutbound,
|
||||
recentOutboundSummary,
|
||||
formatConversationTranscript,
|
||||
readTransportTranscript,
|
||||
formatTransportTranscript,
|
||||
}));
|
||||
|
||||
vi.mock("./suite-runtime-gateway.js", () => ({
|
||||
fetchJson,
|
||||
waitForGatewayHealthy,
|
||||
waitForTransportReady,
|
||||
waitForConfigRestartSettle,
|
||||
patchConfig,
|
||||
applyConfig,
|
||||
readConfigSnapshot,
|
||||
restartGatewayWithConfigPatch,
|
||||
}));
|
||||
|
||||
vi.mock("./suite-runtime-agent.js", () => ({
|
||||
createSession,
|
||||
readEffectiveTools,
|
||||
readSkillStatus,
|
||||
readRawQaSessionStore,
|
||||
seedQaSessionTranscript,
|
||||
readSessionTranscriptSummary,
|
||||
runQaCli,
|
||||
extractMediaPathFromText,
|
||||
resolveGeneratedImagePath,
|
||||
startAgentRun,
|
||||
waitForAgentRun,
|
||||
waitForAgentHistoryReply,
|
||||
listCronJobs,
|
||||
findManagedDreamingCronJob,
|
||||
readDoctorMemoryStatus,
|
||||
forceMemoryIndex,
|
||||
findSkill,
|
||||
writeWorkspaceSkill,
|
||||
callPluginToolsMcp,
|
||||
runAgentPrompt,
|
||||
ensureImageGenerationConfigured,
|
||||
handleQaAction,
|
||||
}));
|
||||
|
||||
vi.mock("./browser-runtime.js", () => ({
|
||||
callQaBrowserRequest: browserRequest,
|
||||
waitForQaBrowserReady: waitForBrowserReady,
|
||||
qaBrowserOpenTab: browserOpenTab,
|
||||
qaBrowserSnapshot: browserSnapshot,
|
||||
qaBrowserAct: browserAct,
|
||||
}));
|
||||
|
||||
vi.mock("./web-runtime.js", () => ({
|
||||
vi.mock("./web-runtime.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("./web-runtime.js")>()),
|
||||
qaWebOpenPage: webOpenPage,
|
||||
qaWebWait: webWait,
|
||||
qaWebType: webType,
|
||||
qaWebSnapshot: webSnapshot,
|
||||
qaWebEvaluate: webEvaluate,
|
||||
}));
|
||||
|
||||
vi.mock("./cron-run-wait.js", () => ({
|
||||
waitForCronRunCompletion,
|
||||
}));
|
||||
|
||||
vi.mock("./discovery-eval.js", () => ({
|
||||
hasDiscoveryLabels,
|
||||
reportsDiscoveryScopeLeak,
|
||||
reportsMissingDiscoveryFiles,
|
||||
}));
|
||||
|
||||
vi.mock("./extract-tool-payload.js", () => ({
|
||||
extractQaToolPayload,
|
||||
}));
|
||||
|
||||
vi.mock("./runtime-tool-fixture.js", () => ({
|
||||
vi.mock("./runtime-tool-fixture.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("./runtime-tool-fixture.js")>()),
|
||||
runRuntimeToolFixture,
|
||||
}));
|
||||
|
||||
vi.mock("./model-switch-eval.js", () => ({
|
||||
hasModelSwitchContinuitySignal,
|
||||
}));
|
||||
|
||||
vi.mock("./gateway-log-sentinel.js", () => ({
|
||||
scanGatewayLogSentinels,
|
||||
assertNoGatewayLogSentinels,
|
||||
}));
|
||||
|
||||
import * as browserRuntime from "./browser-runtime.js";
|
||||
import * as cronRunWait from "./cron-run-wait.js";
|
||||
import * as discoveryEval from "./discovery-eval.js";
|
||||
import { QaSuiteScenarioSkipError } from "./errors.js";
|
||||
import * as extractToolPayload from "./extract-tool-payload.js";
|
||||
import * as modelSwitchEval from "./model-switch-eval.js";
|
||||
import type { QaScenarioRuntimeDeps } from "./scenario-runtime-api.js";
|
||||
import * as suiteRuntimeAgent from "./suite-runtime-agent.js";
|
||||
import { runQaSuiteScenarioDefinition, runQaSuiteScenarioSteps } from "./suite-runtime-flow.js";
|
||||
import * as suiteRuntimeGateway from "./suite-runtime-gateway.js";
|
||||
import * as suiteRuntimeTransport from "./suite-runtime-transport.js";
|
||||
import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js";
|
||||
import * as webRuntime from "./web-runtime.js";
|
||||
|
||||
describe("qa suite runtime flow", () => {
|
||||
it("records intentional scenario skips without running later steps", async () => {
|
||||
@@ -269,18 +152,10 @@ describe("qa suite runtime flow", () => {
|
||||
const call = createQaScenarioRuntimeApi.mock.calls[0]?.[0] as {
|
||||
env: typeof env;
|
||||
scenario: typeof scenario;
|
||||
deps: {
|
||||
runScenario: typeof runScenario;
|
||||
waitForTransportReady: typeof waitForTransportReady;
|
||||
deps: QaScenarioRuntimeDeps & {
|
||||
waitForOutboundMessage: typeof waitForOutboundMessage;
|
||||
markGatewayLogCursor: () => number;
|
||||
assertNoGatewayLogSentinels: typeof assertNoGatewayLogSentinels;
|
||||
readSessionTranscriptSummary: typeof readSessionTranscriptSummary;
|
||||
seedQaSessionTranscript: typeof seedQaSessionTranscript;
|
||||
findManagedDreamingCronJob: typeof findManagedDreamingCronJob;
|
||||
forceMemoryIndex: typeof forceMemoryIndex;
|
||||
runAgentPrompt: typeof runAgentPrompt;
|
||||
waitForAgentHistoryReply: typeof waitForAgentHistoryReply;
|
||||
assertNoGatewayLogSentinels: () => void;
|
||||
runRuntimeToolFixture: (
|
||||
envArg: typeof env,
|
||||
configArg: Record<string, unknown>,
|
||||
@@ -296,7 +171,37 @@ describe("qa suite runtime flow", () => {
|
||||
expect(call.env).toBe(env);
|
||||
expect(call.scenario).toBe(scenario);
|
||||
expect(call.deps.runScenario).toBe(runScenario);
|
||||
expect(call.deps.waitForTransportReady).toBe(waitForTransportReady);
|
||||
for (const dependencyModule of [
|
||||
suiteRuntimeAgent,
|
||||
suiteRuntimeGateway,
|
||||
cronRunWait,
|
||||
discoveryEval,
|
||||
extractToolPayload,
|
||||
modelSwitchEval,
|
||||
]) {
|
||||
for (const [name, helper] of Object.entries(dependencyModule)) {
|
||||
expect((call.deps as Record<string, unknown>)[name]).toBe(helper);
|
||||
}
|
||||
}
|
||||
for (const [name, helper] of Object.entries(suiteRuntimeTransport)) {
|
||||
if (name !== "waitForOutboundMessage") {
|
||||
expect((call.deps as Record<string, unknown>)[name]).toBe(helper);
|
||||
}
|
||||
}
|
||||
const aliasedDependencies = {
|
||||
browserRequest: browserRuntime.callQaBrowserRequest,
|
||||
waitForBrowserReady: browserRuntime.waitForQaBrowserReady,
|
||||
browserOpenTab: browserRuntime.qaBrowserOpenTab,
|
||||
browserSnapshot: browserRuntime.qaBrowserSnapshot,
|
||||
browserAct: browserRuntime.qaBrowserAct,
|
||||
webWait: webRuntime.qaWebWait,
|
||||
webType: webRuntime.qaWebType,
|
||||
webSnapshot: webRuntime.qaWebSnapshot,
|
||||
webEvaluate: webRuntime.qaWebEvaluate,
|
||||
};
|
||||
for (const [name, helper] of Object.entries(aliasedDependencies)) {
|
||||
expect((call.deps as Record<string, unknown>)[name]).toBe(helper);
|
||||
}
|
||||
expect(call.deps.waitForOutboundMessage).toBeTypeOf("function");
|
||||
const outboundPredicate = vi.fn();
|
||||
call.deps.waitForOutboundMessage(env.transport.state, outboundPredicate, 123);
|
||||
@@ -308,22 +213,16 @@ describe("qa suite runtime flow", () => {
|
||||
);
|
||||
expect(call.deps.markGatewayLogCursor()).toBe(0);
|
||||
expect(() => call.deps.assertNoGatewayLogSentinels()).not.toThrow();
|
||||
expect(call.deps.readSessionTranscriptSummary).toBe(readSessionTranscriptSummary);
|
||||
expect(call.deps.seedQaSessionTranscript).toBe(seedQaSessionTranscript);
|
||||
expect(call.deps.findManagedDreamingCronJob).toBe(findManagedDreamingCronJob);
|
||||
expect(call.deps.forceMemoryIndex).toBe(forceMemoryIndex);
|
||||
expect(call.deps.waitForAgentHistoryReply).toBe(waitForAgentHistoryReply);
|
||||
expect(call.deps.runAgentPrompt).toBe(runAgentPrompt);
|
||||
await call.deps.runRuntimeToolFixture(env, { toolName: "read" });
|
||||
expect(runRuntimeToolFixture).toHaveBeenCalledWith(
|
||||
env,
|
||||
{ toolName: "read" },
|
||||
{
|
||||
createSession,
|
||||
readEffectiveTools,
|
||||
runAgentPrompt,
|
||||
fetchJson,
|
||||
ensureImageGenerationConfigured,
|
||||
createSession: suiteRuntimeAgent.createSession,
|
||||
readEffectiveTools: suiteRuntimeAgent.readEffectiveTools,
|
||||
runAgentPrompt: suiteRuntimeAgent.runAgentPrompt,
|
||||
fetchJson: suiteRuntimeGateway.fetchJson,
|
||||
ensureImageGenerationConfigured: suiteRuntimeAgent.ensureImageGenerationConfigured,
|
||||
},
|
||||
);
|
||||
expect(call.constants).toEqual({
|
||||
|
||||
@@ -9,79 +9,27 @@ import { resolveSessionTranscriptsDirForAgent } from "openclaw/plugin-sdk/memory
|
||||
import { buildAgentSessionKey } from "openclaw/plugin-sdk/routing";
|
||||
import { createPluginStateSyncKeyedStore } from "openclaw/plugin-sdk/runtime-doctor";
|
||||
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import {
|
||||
callQaBrowserRequest,
|
||||
qaBrowserAct,
|
||||
qaBrowserOpenTab,
|
||||
qaBrowserSnapshot,
|
||||
waitForQaBrowserReady,
|
||||
} from "./browser-runtime.js";
|
||||
import { waitForCronRunCompletion } from "./cron-run-wait.js";
|
||||
import {
|
||||
hasDiscoveryLabels,
|
||||
reportsDiscoveryScopeLeak,
|
||||
reportsMissingDiscoveryFiles,
|
||||
} from "./discovery-eval.js";
|
||||
import * as browserRuntime from "./browser-runtime.js";
|
||||
import * as cronRunWait from "./cron-run-wait.js";
|
||||
import * as discoveryEval from "./discovery-eval.js";
|
||||
import { QaSuiteScenarioSkipError } from "./errors.js";
|
||||
import { extractQaToolPayload } from "./extract-tool-payload.js";
|
||||
import * as extractToolPayload from "./extract-tool-payload.js";
|
||||
import { assertNoGatewayLogSentinels, scanGatewayLogSentinels } from "./gateway-log-sentinel.js";
|
||||
import { resolveQaLiveTurnTimeoutMs } from "./live-timeout.js";
|
||||
import { hasModelSwitchContinuitySignal } from "./model-switch-eval.js";
|
||||
import { runRuntimeToolFixture } from "./runtime-tool-fixture.js";
|
||||
import * as modelSwitchEval from "./model-switch-eval.js";
|
||||
import * as runtimeToolFixture from "./runtime-tool-fixture.js";
|
||||
import type { QaSeedScenarioWithSource } from "./scenario-catalog.js";
|
||||
import { runScenarioFlow } from "./scenario-flow-runner.js";
|
||||
import { createQaScenarioRuntimeApi, type QaScenarioRuntimeEnv } from "./scenario-runtime-api.js";
|
||||
import {
|
||||
callPluginToolsMcp,
|
||||
createSession,
|
||||
ensureImageGenerationConfigured,
|
||||
extractMediaPathFromText,
|
||||
findSkill,
|
||||
forceMemoryIndex,
|
||||
findManagedDreamingCronJob,
|
||||
handleQaAction,
|
||||
listCronJobs,
|
||||
readDoctorMemoryStatus,
|
||||
readEffectiveTools,
|
||||
readRawQaSessionStore,
|
||||
readSessionTranscriptSummary,
|
||||
readSkillStatus,
|
||||
resolveGeneratedImagePath,
|
||||
runAgentPrompt,
|
||||
runQaCli,
|
||||
seedQaSessionTranscript,
|
||||
startAgentRun,
|
||||
waitForAgentHistoryReply,
|
||||
waitForAgentRun,
|
||||
writeWorkspaceSkill,
|
||||
} from "./suite-runtime-agent.js";
|
||||
import {
|
||||
applyConfig,
|
||||
fetchJson,
|
||||
patchConfig,
|
||||
readConfigSnapshot,
|
||||
restartGatewayWithConfigPatch,
|
||||
waitForConfigRestartSettle,
|
||||
waitForGatewayHealthy,
|
||||
waitForTransportReady,
|
||||
} from "./suite-runtime-gateway.js";
|
||||
import {
|
||||
formatConversationTranscript,
|
||||
formatTransportTranscript,
|
||||
readTransportTranscript,
|
||||
recentOutboundSummary,
|
||||
waitForNoOutbound,
|
||||
waitForNoTransportOutbound,
|
||||
waitForOutboundMessage,
|
||||
} from "./suite-runtime-transport.js";
|
||||
createQaScenarioRuntimeApi,
|
||||
type QaScenarioRuntimeDeps,
|
||||
type QaScenarioRuntimeEnv,
|
||||
} from "./scenario-runtime-api.js";
|
||||
import * as suiteRuntimeAgent from "./suite-runtime-agent.js";
|
||||
import * as suiteRuntimeGateway from "./suite-runtime-gateway.js";
|
||||
import * as suiteRuntimeTransport from "./suite-runtime-transport.js";
|
||||
import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js";
|
||||
import {
|
||||
qaWebEvaluate,
|
||||
qaWebOpenPage,
|
||||
qaWebSnapshot,
|
||||
qaWebType,
|
||||
qaWebWait,
|
||||
} from "./web-runtime.js";
|
||||
import * as webRuntime from "./web-runtime.js";
|
||||
|
||||
type QaSuiteScenarioFlowEnv = {
|
||||
lab: unknown;
|
||||
@@ -122,6 +70,28 @@ function setActiveMemorySessionDisabled(
|
||||
store.delete(key);
|
||||
}
|
||||
|
||||
const qaSuiteScenarioIdentityDeps = {
|
||||
fs,
|
||||
path,
|
||||
sleep,
|
||||
randomUUID,
|
||||
...suiteRuntimeAgent,
|
||||
...suiteRuntimeGateway,
|
||||
...suiteRuntimeTransport,
|
||||
...extractToolPayload,
|
||||
waitForCronRunCompletion: cronRunWait.waitForCronRunCompletion,
|
||||
hasDiscoveryLabels: discoveryEval.hasDiscoveryLabels,
|
||||
reportsDiscoveryScopeLeak: discoveryEval.reportsDiscoveryScopeLeak,
|
||||
reportsMissingDiscoveryFiles: discoveryEval.reportsMissingDiscoveryFiles,
|
||||
hasModelSwitchContinuitySignal: modelSwitchEval.hasModelSwitchContinuitySignal,
|
||||
formatMemoryDreamingDay,
|
||||
resolveSessionTranscriptsDirForAgent,
|
||||
activeMemoryToggleKey,
|
||||
setActiveMemorySessionDisabled,
|
||||
buildAgentSessionKey,
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
};
|
||||
|
||||
type QaSuiteStep = {
|
||||
name: string;
|
||||
run: () => Promise<string | void>;
|
||||
@@ -198,107 +168,59 @@ type QaSuiteScenarioFlowApiParams = QaSuiteScenarioDepsParams & {
|
||||
};
|
||||
|
||||
function createQaSuiteScenarioDeps(params: QaSuiteScenarioDepsParams) {
|
||||
const waitForAccountOutboundMessage: typeof waitForOutboundMessage = (
|
||||
const waitForAccountOutboundMessage: typeof suiteRuntimeTransport.waitForOutboundMessage = (
|
||||
state,
|
||||
predicate,
|
||||
timeoutMs,
|
||||
options,
|
||||
) =>
|
||||
waitForOutboundMessage(state, predicate, timeoutMs, {
|
||||
suiteRuntimeTransport.waitForOutboundMessage(state, predicate, timeoutMs, {
|
||||
...options,
|
||||
accountId: params.env.transport.accountId,
|
||||
});
|
||||
return {
|
||||
fs,
|
||||
path,
|
||||
sleep,
|
||||
randomUUID,
|
||||
...qaSuiteScenarioIdentityDeps,
|
||||
runScenario: params.runScenario,
|
||||
waitForOutboundMessage: waitForAccountOutboundMessage,
|
||||
waitForNoOutbound,
|
||||
waitForNoTransportOutbound,
|
||||
recentOutboundSummary,
|
||||
formatConversationTranscript,
|
||||
readTransportTranscript,
|
||||
formatTransportTranscript,
|
||||
fetchJson,
|
||||
waitForGatewayHealthy,
|
||||
waitForTransportReady,
|
||||
browserRequest: callQaBrowserRequest,
|
||||
waitForBrowserReady: waitForQaBrowserReady,
|
||||
browserOpenTab: qaBrowserOpenTab,
|
||||
browserSnapshot: qaBrowserSnapshot,
|
||||
browserAct: qaBrowserAct,
|
||||
webOpenPage: async (webParams: Parameters<typeof qaWebOpenPage>[0]) => {
|
||||
const opened = await qaWebOpenPage({ ...webParams, repoRoot: params.env.repoRoot });
|
||||
browserRequest: browserRuntime.callQaBrowserRequest,
|
||||
waitForBrowserReady: browserRuntime.waitForQaBrowserReady,
|
||||
browserOpenTab: browserRuntime.qaBrowserOpenTab,
|
||||
browserSnapshot: browserRuntime.qaBrowserSnapshot,
|
||||
browserAct: browserRuntime.qaBrowserAct,
|
||||
webOpenPage: async (webParams: Parameters<typeof webRuntime.qaWebOpenPage>[0]) => {
|
||||
const opened = await webRuntime.qaWebOpenPage({
|
||||
...webParams,
|
||||
repoRoot: params.env.repoRoot,
|
||||
});
|
||||
params.env.webSessionIds.add(opened.pageId);
|
||||
return opened;
|
||||
},
|
||||
webWait: qaWebWait,
|
||||
webType: qaWebType,
|
||||
webSnapshot: qaWebSnapshot,
|
||||
webEvaluate: qaWebEvaluate,
|
||||
waitForConfigRestartSettle,
|
||||
patchConfig,
|
||||
applyConfig,
|
||||
readConfigSnapshot,
|
||||
restartGatewayWithConfigPatch,
|
||||
createSession,
|
||||
readEffectiveTools,
|
||||
readSkillStatus,
|
||||
readRawQaSessionStore,
|
||||
seedQaSessionTranscript,
|
||||
webWait: webRuntime.qaWebWait,
|
||||
webType: webRuntime.qaWebType,
|
||||
webSnapshot: webRuntime.qaWebSnapshot,
|
||||
webEvaluate: webRuntime.qaWebEvaluate,
|
||||
readGatewayLogs: () => params.env.gateway.logs?.() ?? "",
|
||||
markGatewayLogCursor: () => (params.env.gateway.logs?.() ?? "").length,
|
||||
scanGatewayLogSentinels: (options?: Parameters<typeof scanGatewayLogSentinels>[1]) =>
|
||||
scanGatewayLogSentinels(params.env.gateway.logs?.(), options),
|
||||
assertNoGatewayLogSentinels: (options?: Parameters<typeof assertNoGatewayLogSentinels>[1]) =>
|
||||
assertNoGatewayLogSentinels(params.env.gateway.logs?.(), options),
|
||||
readSessionTranscriptSummary,
|
||||
runQaCli,
|
||||
extractMediaPathFromText,
|
||||
resolveGeneratedImagePath,
|
||||
startAgentRun,
|
||||
waitForAgentRun,
|
||||
waitForAgentHistoryReply,
|
||||
listCronJobs,
|
||||
findManagedDreamingCronJob,
|
||||
waitForCronRunCompletion,
|
||||
readDoctorMemoryStatus,
|
||||
forceMemoryIndex,
|
||||
findSkill,
|
||||
writeWorkspaceSkill,
|
||||
callPluginToolsMcp,
|
||||
runAgentPrompt,
|
||||
ensureImageGenerationConfigured,
|
||||
handleQaAction,
|
||||
runRuntimeToolFixture: async (
|
||||
envArg: QaSuiteScenarioFlowEnv,
|
||||
configArg: Record<string, unknown>,
|
||||
) =>
|
||||
runRuntimeToolFixture(envArg, configArg, {
|
||||
createSession,
|
||||
readEffectiveTools,
|
||||
runAgentPrompt,
|
||||
fetchJson,
|
||||
ensureImageGenerationConfigured,
|
||||
runtimeToolFixture.runRuntimeToolFixture(envArg, configArg, {
|
||||
createSession: suiteRuntimeAgent.createSession,
|
||||
readEffectiveTools: suiteRuntimeAgent.readEffectiveTools,
|
||||
runAgentPrompt: suiteRuntimeAgent.runAgentPrompt,
|
||||
fetchJson: suiteRuntimeGateway.fetchJson,
|
||||
ensureImageGenerationConfigured: suiteRuntimeAgent.ensureImageGenerationConfigured,
|
||||
}),
|
||||
extractQaToolPayload,
|
||||
formatMemoryDreamingDay,
|
||||
resolveSessionTranscriptsDirForAgent,
|
||||
activeMemoryToggleKey,
|
||||
setActiveMemorySessionDisabled,
|
||||
buildAgentSessionKey,
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
formatErrorMessage: params.formatErrorMessage,
|
||||
liveTurnTimeoutMs: params.liveTurnTimeoutMs,
|
||||
resolveQaLiveTurnTimeoutMs: params.resolveQaLiveTurnTimeoutMs,
|
||||
splitModelRef: params.splitModelRef,
|
||||
hasDiscoveryLabels,
|
||||
reportsDiscoveryScopeLeak,
|
||||
reportsMissingDiscoveryFiles,
|
||||
hasModelSwitchContinuitySignal,
|
||||
};
|
||||
} satisfies QaScenarioRuntimeDeps;
|
||||
}
|
||||
|
||||
function createQaSuiteScenarioFlowApi(params: QaSuiteScenarioFlowApiParams) {
|
||||
@@ -348,7 +270,11 @@ export function createQaSuiteScenarioStepRunner(
|
||||
scenarioTitle: scenario.title,
|
||||
timeoutMs: execution.timeoutMs ?? deps.liveTurnTimeoutMs(env, 60_000),
|
||||
waitForConfigRestartSettle: async (options) =>
|
||||
await waitForConfigRestartSettle(env, options?.restartDelayMs, options?.timeoutMs),
|
||||
await suiteRuntimeGateway.waitForConfigRestartSettle(
|
||||
env,
|
||||
options?.restartDelayMs,
|
||||
options?.timeoutMs,
|
||||
),
|
||||
});
|
||||
if (prepared) {
|
||||
Object.assign(vars, prepared);
|
||||
|
||||
Reference in New Issue
Block a user