test(qa): cover live voice talkback flow

This commit is contained in:
Vincent Koc
2026-08-02 18:31:25 +08:00
parent 1fca219c3c
commit 882a847670
3 changed files with 84 additions and 1 deletions
@@ -60,6 +60,41 @@ describe("qa scenario catalog", () => {
]);
});
it("includes the live inbound voice talkback scenario", () => {
const scenario = readQaScenarioById("inbound-voice-talkback-live");
const config = readQaScenarioExecutionConfig("inbound-voice-talkback-live") as
| {
requiredProviderMode?: string;
requiredProvider?: string;
expectedMarker?: string;
}
| undefined;
expect(scenario.sourcePath).toBe("qa/scenarios/media/inbound-voice-talkback-live.yaml");
expect(scenario.execution.channel).toBe("qa-channel");
expect(scenario.gatewayConfigPatch?.tools).toEqual({
media: {
models: [
{
provider: "openai",
model: "gpt-4o-transcribe",
capabilities: ["audio"],
},
],
audio: {
enabled: true,
echoTranscript: false,
},
},
});
expect(config?.requiredProviderMode).toBe("live-frontier");
expect(config?.requiredProvider).toBe("openai");
expect(config?.expectedMarker).toBe("MATRIX QA VOICE PREFLIGHT OK");
expect(JSON.stringify(scenario.execution.flow)).toContain(
"./live-transports/matrix/scenarios/scenario-media-fixtures.js",
);
});
it("includes the Kitchen Sink live OpenAI plugin gauntlet", () => {
const scenario = readQaScenarioById("kitchen-sink-live-openai");
const config = readQaScenarioExecutionConfig("kitchen-sink-live-openai") as
@@ -404,6 +404,54 @@ const planningEvidenceFixtures = readQaScenarioPack()
.map(createPlanningEvidenceFixture);
describe("scenario-flow-runner", () => {
it("reuses the spoken WAV fixture for one live voice talkback reply", async () => {
const state = createQaBusState();
const expectedReply = "Matrix QA voice pre-flight OK.";
const result = await runLoadedScenarioFlow("inbound-voice-talkback-live", {
state,
api: {
env: {
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.4",
gateway: {
runtimeEnv: {
OPENAI_API_KEY: "test-openai-key",
},
},
},
splitModelRef: (ref: string) => {
const slash = ref.indexOf("/");
return slash > 0 ? { provider: ref.slice(0, slash), model: ref.slice(slash + 1) } : null;
},
markGatewayLogCursor: () => 0,
readGatewayLogs: () => "",
resolveQaLiveTurnTimeoutMs: (_env: unknown, timeoutMs: number) => timeoutMs,
},
onWaitForOutboundMessage: ({ state: currentState }) => {
currentState.addOutboundMessage({
accountId: "qa-channel",
to: "dm:qa-live-voice-talkback",
text: expectedReply,
});
},
});
expect(result.status).toBe("pass");
const inbound = state.getSnapshot().messages.find((message) => message.direction === "inbound");
const audioBase64 = inbound?.attachments?.[0]?.contentBase64;
expect(audioBase64).toBeTruthy();
const audio = Buffer.from(audioBase64 ?? "", "base64");
expect(audio.subarray(0, 4).toString("ascii")).toBe("RIFF");
expect(audio.length).toBeGreaterThan(44);
expect(
state
.getSnapshot()
.messages.filter((message) => message.direction === "outbound")
.map((message) => message.text),
).toEqual([expectedReply]);
});
it("keeps live goal followthrough inside the active-goal context limit", async () => {
const state = createQaBusState();
const artifactFile = "goal-continuance-live-00000000.txt";
@@ -38,7 +38,7 @@ scenario:
suiteIsolation: isolated
timeoutMs: 180000
retryCount: 0
summary: Run with `OPENCLAW_LIVE_OPENAI_KEY="${OPENAI_API_KEY}" pnpm openclaw qa suite --provider-mode live-frontier --model openai/gpt-5.6-luna --alt-model openai/gpt-5.6-luna --fast --scenario inbound-voice-talkback-live --concurrency 1`.
summary: Run with `OPENCLAW_LIVE_OPENAI_KEY="${OPENAI_API_KEY}" pnpm openclaw qa suite --provider-mode live-frontier --model openai/gpt-5.4 --alt-model openai/gpt-5.4 --fast --scenario inbound-voice-talkback-live --concurrency 1`.
config:
requiredProviderMode: live-frontier
requiredProvider: openai