test(qa): wait for durable webchat transcript (#103718)

extensions/qa-lab/src/scenario-flow-runner.test.ts

@@ -1,7 +1,7 @@
 // Qa Lab tests cover scenario flow runner plugin behavior.
 import { describe, expect, it } from "vitest";
 import { createQaBusState } from "./bus-state.js";
-import { readQaScenarioById } from "./scenario-catalog.js";
+import { readQaScenarioById, type QaScenarioFlow } from "./scenario-catalog.js";
 import { runScenarioFlow } from "./scenario-flow-runner.js";
 
 type QaFlowStep = {
@@ -19,6 +19,8 @@ function formatTestTranscript(state: ReturnType<typeof createQaBusState>) {
 async function runLoadedScenarioFlow(
   scenarioId: string,
   params: {
+    flow?: QaScenarioFlow;
+    api?: Record<string, unknown>;
     omitOutboundSequence?: boolean;
     onWaitForOutboundMessage?: (params: {
       waitCount: number;
@@ -27,8 +29,8 @@ async function runLoadedScenarioFlow(
   } = {},
 ) {
   const scenario = readQaScenarioById(scenarioId);
-  const flow = scenario.execution.flow;
-  if (!flow) {
+  const loadedFlow = scenario.execution.flow;
+  if (!loadedFlow) {
     throw new Error(`scenario has no flow: ${scenarioId}`);
   }
 
@@ -137,12 +139,70 @@ async function runLoadedScenarioFlow(
         steps: stepResults,
       };
     },
+    ...params.api,
   };
 
   return await runScenarioFlow({
     api,
     scenarioTitle: scenario.title,
-    flow,
+    flow: params.flow ?? loadedFlow,
+  });
+}
+
+function readWebchatTranscriptWaitFlow() {
+  const scenario = readQaScenarioById("webchat-direct-reply-routing");
+  const actions = scenario.execution.flow?.steps[0]?.actions;
+  if (!actions) {
+    throw new Error("webchat direct reply scenario has no actions");
+  }
+  const waitIndex = actions.findIndex(
+    (action) =>
+      typeof action === "object" &&
+      action !== null &&
+      "saveAs" in action &&
+      action.saveAs === "transcriptSummary",
+  );
+  if (waitIndex < 0) {
+    throw new Error("webchat direct reply scenario has no transcript wait");
+  }
+  return {
+    steps: [
+      {
+        name: "waits for the durable assistant transcript",
+        actions: [
+          { set: "sessionKey", value: "agent:qa:test-session" },
+          ...actions.slice(waitIndex, waitIndex + 3),
+        ],
+      },
+    ],
+  } satisfies QaScenarioFlow;
+}
+
+async function runWebchatTranscriptWait(
+  readSessionTranscriptSummary: () => Promise<{
+    finalText: string;
+    hasDirectReplySelfMessage: boolean;
+  }>,
+) {
+  return await runLoadedScenarioFlow("webchat-direct-reply-routing", {
+    flow: readWebchatTranscriptWaitFlow(),
+    api: {
+      readSessionTranscriptSummary,
+      waitForCondition: async <T>(check: () => Promise<T | undefined>) => {
+        for (let attempt = 0; attempt < 10; attempt += 1) {
+          const value = await check();
+          if (value !== undefined) {
+            return value;
+          }
+        }
+        throw new Error("test condition was not met");
+      },
+      normalizeLowercaseStringOrEmpty: (value: unknown) =>
+        typeof value === "string" ? value.trim().toLowerCase() : "",
+      formatErrorMessage: (error: unknown) =>
+        error instanceof Error ? error.message : String(error),
+      liveTurnTimeoutMs: (_env: unknown, timeoutMs: number) => timeoutMs,
+    },
   });
 }
 
@@ -449,4 +509,42 @@ describe("scenario-flow-runner", () => {
       }),
     ).rejects.toThrow("exactly one marked post-restart reply");
   });
+
+  it("waits through transient transcript states until the webchat reply is durable", async () => {
+    let readCount = 0;
+    const missingFile = Object.assign(new Error("transcript not written yet"), { code: "ENOENT" });
+    const summaries = [
+      missingFile,
+      { finalText: "", hasDirectReplySelfMessage: false },
+      { finalText: "WEBCHAT-DIRECT-REPLY-OK", hasDirectReplySelfMessage: false },
+    ];
+
+    const result = await runWebchatTranscriptWait(async () => {
+      const summary = summaries[readCount];
+      readCount += 1;
+      if (summary instanceof Error) {
+        throw summary;
+      }
+      if (!summary) {
+        throw new Error("unexpected transcript read");
+      }
+      return summary;
+    });
+
+    expect(result.status).toBe("pass");
+    expect(readCount).toBe(3);
+  });
+
+  it("fails the webchat transcript wait immediately on deterministic read errors", async () => {
+    let readCount = 0;
+    const permissionError = Object.assign(new Error("permission denied"), { code: "EACCES" });
+
+    await expect(
+      runWebchatTranscriptWait(async () => {
+        readCount += 1;
+        throw permissionError;
+      }),
+    ).rejects.toBe(permissionError);
+    expect(readCount).toBe(1);
+  });
 });

qa/scenarios/channels/webchat-direct-reply-routing.yaml

@@ -96,11 +96,20 @@ flow:
                   expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).map((request) => ({ plannedToolName: request.plannedToolName ?? null, plannedToolArgs: request.plannedToolArgs ?? null, allInputText: String(request.allInputText ?? '').slice(0, 400), finalText: String(request.finalText ?? '').slice(0, 200), toolOutput: request.toolOutput ? String(request.toolOutput).slice(0, 200) : null })) : []"
               - throw:
                   expr: "`direct reply marker missing: ${directReplyError?.message ?? directReplyError}; transcript=${formatTransportTranscript(state, { conversationId })}; requests=${JSON.stringify(directReplyDebugRequests)}`"
-        - set: transcriptSummary
-          value:
-            expr: "await readSessionTranscriptSummary(env, sessionKey)"
+        - call: waitForCondition
+          saveAs: transcriptSummary
+          args:
+            - lambda:
+                async: true
+                expr: "readSessionTranscriptSummary(env, sessionKey).then((summary) => summary.hasDirectReplySelfMessage || normalizeLowercaseStringOrEmpty(summary.finalText).includes(normalizeLowercaseStringOrEmpty(config.expectedMarker)) ? summary : undefined).catch((error) => { const message = formatErrorMessage(error); return ((error && typeof error === 'object' && error.code === 'ENOENT') || message.includes('session transcript entry not found') || message.includes('session transcript is empty')) ? undefined : Promise.reject(error); })"
+            - expr: liveTurnTimeoutMs(env, 60000)
+            - 250
         - assert:
             expr: "!transcriptSummary.hasDirectReplySelfMessage"
             message:
               expr: "`assistant self-sent direct reply through message(action=send); finalText=${transcriptSummary.finalText}`"
+        - assert:
+            expr: "normalizeLowercaseStringOrEmpty(transcriptSummary.finalText).includes(normalizeLowercaseStringOrEmpty(config.expectedMarker))"
+            message:
+              expr: "`persisted assistant transcript missing direct reply marker ${config.expectedMarker}; finalText=${transcriptSummary.finalText}`"
       detailsExpr: outbound.text