diff --git a/extensions/qa-lab/src/scenario-catalog.openai-live.test.ts b/extensions/qa-lab/src/scenario-catalog.openai-live.test.ts index 612f5b748ec7..7c06b0b7caec 100644 --- a/extensions/qa-lab/src/scenario-catalog.openai-live.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.openai-live.test.ts @@ -60,6 +60,41 @@ describe("qa scenario catalog", () => { ]); }); + it("includes the live inbound voice talkback scenario", () => { + const scenario = readQaScenarioById("inbound-voice-talkback-live"); + const config = readQaScenarioExecutionConfig("inbound-voice-talkback-live") as + | { + requiredProviderMode?: string; + requiredProvider?: string; + expectedMarker?: string; + } + | undefined; + + expect(scenario.sourcePath).toBe("qa/scenarios/media/inbound-voice-talkback-live.yaml"); + expect(scenario.execution.channel).toBe("qa-channel"); + expect(scenario.gatewayConfigPatch?.tools).toEqual({ + media: { + models: [ + { + provider: "openai", + model: "gpt-4o-transcribe", + capabilities: ["audio"], + }, + ], + audio: { + enabled: true, + echoTranscript: false, + }, + }, + }); + expect(config?.requiredProviderMode).toBe("live-frontier"); + expect(config?.requiredProvider).toBe("openai"); + expect(config?.expectedMarker).toBe("MATRIX QA VOICE PREFLIGHT OK"); + expect(JSON.stringify(scenario.execution.flow)).toContain( + "./live-transports/matrix/scenarios/scenario-media-fixtures.js", + ); + }); + it("includes the Kitchen Sink live OpenAI plugin gauntlet", () => { const scenario = readQaScenarioById("kitchen-sink-live-openai"); const config = readQaScenarioExecutionConfig("kitchen-sink-live-openai") as diff --git a/extensions/qa-lab/src/scenario-flow-runner.test.ts b/extensions/qa-lab/src/scenario-flow-runner.test.ts index 36b5d7bdfc7f..0ff61885d5ac 100644 --- a/extensions/qa-lab/src/scenario-flow-runner.test.ts +++ b/extensions/qa-lab/src/scenario-flow-runner.test.ts @@ -404,6 +404,54 @@ const planningEvidenceFixtures = readQaScenarioPack() .map(createPlanningEvidenceFixture); describe("scenario-flow-runner", () => { + it("reuses the spoken WAV fixture for one live voice talkback reply", async () => { + const state = createQaBusState(); + const expectedReply = "Matrix QA voice pre-flight OK."; + + const result = await runLoadedScenarioFlow("inbound-voice-talkback-live", { + state, + api: { + env: { + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.4", + gateway: { + runtimeEnv: { + OPENAI_API_KEY: "test-openai-key", + }, + }, + }, + splitModelRef: (ref: string) => { + const slash = ref.indexOf("/"); + return slash > 0 ? { provider: ref.slice(0, slash), model: ref.slice(slash + 1) } : null; + }, + markGatewayLogCursor: () => 0, + readGatewayLogs: () => "", + resolveQaLiveTurnTimeoutMs: (_env: unknown, timeoutMs: number) => timeoutMs, + }, + onWaitForOutboundMessage: ({ state: currentState }) => { + currentState.addOutboundMessage({ + accountId: "qa-channel", + to: "dm:qa-live-voice-talkback", + text: expectedReply, + }); + }, + }); + + expect(result.status).toBe("pass"); + const inbound = state.getSnapshot().messages.find((message) => message.direction === "inbound"); + const audioBase64 = inbound?.attachments?.[0]?.contentBase64; + expect(audioBase64).toBeTruthy(); + const audio = Buffer.from(audioBase64 ?? "", "base64"); + expect(audio.subarray(0, 4).toString("ascii")).toBe("RIFF"); + expect(audio.length).toBeGreaterThan(44); + expect( + state + .getSnapshot() + .messages.filter((message) => message.direction === "outbound") + .map((message) => message.text), + ).toEqual([expectedReply]); + }); + it("keeps live goal followthrough inside the active-goal context limit", async () => { const state = createQaBusState(); const artifactFile = "goal-continuance-live-00000000.txt"; diff --git a/qa/scenarios/media/inbound-voice-talkback-live.yaml b/qa/scenarios/media/inbound-voice-talkback-live.yaml index c3afa6c68dc4..7f50ad20e8d6 100644 --- a/qa/scenarios/media/inbound-voice-talkback-live.yaml +++ b/qa/scenarios/media/inbound-voice-talkback-live.yaml @@ -38,7 +38,7 @@ scenario: suiteIsolation: isolated timeoutMs: 180000 retryCount: 0 - summary: Run with `OPENCLAW_LIVE_OPENAI_KEY="${OPENAI_API_KEY}" pnpm openclaw qa suite --provider-mode live-frontier --model openai/gpt-5.6-luna --alt-model openai/gpt-5.6-luna --fast --scenario inbound-voice-talkback-live --concurrency 1`. + summary: Run with `OPENCLAW_LIVE_OPENAI_KEY="${OPENAI_API_KEY}" pnpm openclaw qa suite --provider-mode live-frontier --model openai/gpt-5.4 --alt-model openai/gpt-5.4 --fast --scenario inbound-voice-talkback-live --concurrency 1`. config: requiredProviderMode: live-frontier requiredProvider: openai