From e54de70ab4a66e7565f022c99e484575e219da12 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 10 Jul 2026 15:08:47 +0100 Subject: [PATCH] test(qa): wait for durable webchat transcript (#103718) --- .../qa-lab/src/scenario-flow-runner.test.ts | 106 +++++++++++++++++- .../webchat-direct-reply-routing.yaml | 15 ++- 2 files changed, 114 insertions(+), 7 deletions(-) diff --git a/extensions/qa-lab/src/scenario-flow-runner.test.ts b/extensions/qa-lab/src/scenario-flow-runner.test.ts index d2f2bfa1dd4c..1dbc5fdca2a8 100644 --- a/extensions/qa-lab/src/scenario-flow-runner.test.ts +++ b/extensions/qa-lab/src/scenario-flow-runner.test.ts @@ -1,7 +1,7 @@ // Qa Lab tests cover scenario flow runner plugin behavior. import { describe, expect, it } from "vitest"; import { createQaBusState } from "./bus-state.js"; -import { readQaScenarioById } from "./scenario-catalog.js"; +import { readQaScenarioById, type QaScenarioFlow } from "./scenario-catalog.js"; import { runScenarioFlow } from "./scenario-flow-runner.js"; type QaFlowStep = { @@ -19,6 +19,8 @@ function formatTestTranscript(state: ReturnType) { async function runLoadedScenarioFlow( scenarioId: string, params: { + flow?: QaScenarioFlow; + api?: Record; omitOutboundSequence?: boolean; onWaitForOutboundMessage?: (params: { waitCount: number; @@ -27,8 +29,8 @@ async function runLoadedScenarioFlow( } = {}, ) { const scenario = readQaScenarioById(scenarioId); - const flow = scenario.execution.flow; - if (!flow) { + const loadedFlow = scenario.execution.flow; + if (!loadedFlow) { throw new Error(`scenario has no flow: ${scenarioId}`); } @@ -137,12 +139,70 @@ async function runLoadedScenarioFlow( steps: stepResults, }; }, + ...params.api, }; return await runScenarioFlow({ api, scenarioTitle: scenario.title, - flow, + flow: params.flow ?? loadedFlow, + }); +} + +function readWebchatTranscriptWaitFlow() { + const scenario = readQaScenarioById("webchat-direct-reply-routing"); + const actions = scenario.execution.flow?.steps[0]?.actions; + if (!actions) { + throw new Error("webchat direct reply scenario has no actions"); + } + const waitIndex = actions.findIndex( + (action) => + typeof action === "object" && + action !== null && + "saveAs" in action && + action.saveAs === "transcriptSummary", + ); + if (waitIndex < 0) { + throw new Error("webchat direct reply scenario has no transcript wait"); + } + return { + steps: [ + { + name: "waits for the durable assistant transcript", + actions: [ + { set: "sessionKey", value: "agent:qa:test-session" }, + ...actions.slice(waitIndex, waitIndex + 3), + ], + }, + ], + } satisfies QaScenarioFlow; +} + +async function runWebchatTranscriptWait( + readSessionTranscriptSummary: () => Promise<{ + finalText: string; + hasDirectReplySelfMessage: boolean; + }>, +) { + return await runLoadedScenarioFlow("webchat-direct-reply-routing", { + flow: readWebchatTranscriptWaitFlow(), + api: { + readSessionTranscriptSummary, + waitForCondition: async (check: () => Promise) => { + for (let attempt = 0; attempt < 10; attempt += 1) { + const value = await check(); + if (value !== undefined) { + return value; + } + } + throw new Error("test condition was not met"); + }, + normalizeLowercaseStringOrEmpty: (value: unknown) => + typeof value === "string" ? value.trim().toLowerCase() : "", + formatErrorMessage: (error: unknown) => + error instanceof Error ? error.message : String(error), + liveTurnTimeoutMs: (_env: unknown, timeoutMs: number) => timeoutMs, + }, }); } @@ -449,4 +509,42 @@ describe("scenario-flow-runner", () => { }), ).rejects.toThrow("exactly one marked post-restart reply"); }); + + it("waits through transient transcript states until the webchat reply is durable", async () => { + let readCount = 0; + const missingFile = Object.assign(new Error("transcript not written yet"), { code: "ENOENT" }); + const summaries = [ + missingFile, + { finalText: "", hasDirectReplySelfMessage: false }, + { finalText: "WEBCHAT-DIRECT-REPLY-OK", hasDirectReplySelfMessage: false }, + ]; + + const result = await runWebchatTranscriptWait(async () => { + const summary = summaries[readCount]; + readCount += 1; + if (summary instanceof Error) { + throw summary; + } + if (!summary) { + throw new Error("unexpected transcript read"); + } + return summary; + }); + + expect(result.status).toBe("pass"); + expect(readCount).toBe(3); + }); + + it("fails the webchat transcript wait immediately on deterministic read errors", async () => { + let readCount = 0; + const permissionError = Object.assign(new Error("permission denied"), { code: "EACCES" }); + + await expect( + runWebchatTranscriptWait(async () => { + readCount += 1; + throw permissionError; + }), + ).rejects.toBe(permissionError); + expect(readCount).toBe(1); + }); }); diff --git a/qa/scenarios/channels/webchat-direct-reply-routing.yaml b/qa/scenarios/channels/webchat-direct-reply-routing.yaml index 859efa8e0bcc..b4f6137b68ec 100644 --- a/qa/scenarios/channels/webchat-direct-reply-routing.yaml +++ b/qa/scenarios/channels/webchat-direct-reply-routing.yaml @@ -96,11 +96,20 @@ flow: expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).map((request) => ({ plannedToolName: request.plannedToolName ?? null, plannedToolArgs: request.plannedToolArgs ?? null, allInputText: String(request.allInputText ?? '').slice(0, 400), finalText: String(request.finalText ?? '').slice(0, 200), toolOutput: request.toolOutput ? String(request.toolOutput).slice(0, 200) : null })) : []" - throw: expr: "`direct reply marker missing: ${directReplyError?.message ?? directReplyError}; transcript=${formatTransportTranscript(state, { conversationId })}; requests=${JSON.stringify(directReplyDebugRequests)}`" - - set: transcriptSummary - value: - expr: "await readSessionTranscriptSummary(env, sessionKey)" + - call: waitForCondition + saveAs: transcriptSummary + args: + - lambda: + async: true + expr: "readSessionTranscriptSummary(env, sessionKey).then((summary) => summary.hasDirectReplySelfMessage || normalizeLowercaseStringOrEmpty(summary.finalText).includes(normalizeLowercaseStringOrEmpty(config.expectedMarker)) ? summary : undefined).catch((error) => { const message = formatErrorMessage(error); return ((error && typeof error === 'object' && error.code === 'ENOENT') || message.includes('session transcript entry not found') || message.includes('session transcript is empty')) ? undefined : Promise.reject(error); })" + - expr: liveTurnTimeoutMs(env, 60000) + - 250 - assert: expr: "!transcriptSummary.hasDirectReplySelfMessage" message: expr: "`assistant self-sent direct reply through message(action=send); finalText=${transcriptSummary.finalText}`" + - assert: + expr: "normalizeLowercaseStringOrEmpty(transcriptSummary.finalText).includes(normalizeLowercaseStringOrEmpty(config.expectedMarker))" + message: + expr: "`persisted assistant transcript missing direct reply marker ${config.expectedMarker}; finalText=${transcriptSummary.finalText}`" detailsExpr: outbound.text