fix(qa): stabilize Matrix release validation scenarios (#117376)

* fix(qa): stabilize legacy Matrix validation scenarios

* fix(qa): clean up Matrix fault proxy startup failure

* refactor(release): separate Matrix validation repairs

extensions/qa-lab/src/providers/mock-openai/server.test.ts

@@ -3989,6 +3989,29 @@ describe("qa mock openai server", () => {
     });
   });
 
+  it("serves the Matrix voice preflight transcription contract for its explicit fixture", async () => {
+    const server = await startQaMockOpenAiServer({
+      host: "127.0.0.1",
+      port: 0,
+    });
+    cleanups.push(async () => {
+      await server.stop();
+    });
+
+    const response = await fetch(`${server.baseUrl}/v1/audio/transcriptions`, {
+      method: "POST",
+      headers: {
+        "content-type": "multipart/form-data; boundary=qa",
+      },
+      body: '--qa\r\ncontent-disposition: form-data; name="file"; filename="matrix-qa-voice-preflight.wav"\r\n\r\nvoice\r\n--qa--\r\n',
+    });
+
+    expect(response.status).toBe(200);
+    await expect(response.json()).resolves.toEqual({
+      text: "MATRIX_QA_VOICE_PREFLIGHT_MENTION reply with only this exact marker: MATRIX_QA_VOICE_PREFLIGHT_OK",
+    });
+  });
+
   it("serves deterministic WhatsApp group audio transcription for large audio uploads", async () => {
     const server = await startQaMockOpenAiServer({
       host: "127.0.0.1",

extensions/qa-lab/src/providers/mock-openai/server.ts

@@ -191,6 +191,9 @@ const QA_AUDIO_TRANSCRIPTION_TEXT =
 const QA_GROUP_AUDIO_TRANSCRIPTION_TEXT =
   "openclawqa reply with only this exact marker after group audio preflight: WHATSAPP_QA_GROUP_AUDIO_TRANSCRIPT_OK";
 const QA_GROUP_AUDIO_MIN_MULTIPART_BODY_CHARS = 48_000;
+const QA_MATRIX_VOICE_PREFLIGHT_FILENAME = "matrix-qa-voice-preflight.wav";
+const QA_MATRIX_VOICE_PREFLIGHT_TRANSCRIPTION_TEXT =
+  "MATRIX_QA_VOICE_PREFLIGHT_MENTION reply with only this exact marker: MATRIX_QA_VOICE_PREFLIGHT_OK";
 const QA_MCP_CODE_MODE_API_FILE_PROMPT_RE = /mcp code mode api file qa check/i;
 
 type MockScenarioState = {
@@ -252,6 +255,9 @@ function writeOpenAiMalformedJsonError(res: ServerResponse, label: string) {
 }
 
 function transcriptionTextForAudioRequest(rawBody: string) {
+  if (rawBody.includes(QA_MATRIX_VOICE_PREFLIGHT_FILENAME)) {
+    return QA_MATRIX_VOICE_PREFLIGHT_TRANSCRIPTION_TEXT;
+  }
   if (rawBody.length >= QA_GROUP_AUDIO_MIN_MULTIPART_BODY_CHARS) {
     return QA_GROUP_AUDIO_TRANSCRIPTION_TEXT;
   }
@@ -2032,9 +2038,10 @@ async function buildResponsesPayload(
   const currentTurnIsToolProgress =
     QA_TOOL_PROGRESS_ERROR_PROMPT_RE.test(toolProgressPrompt) ||
     QA_TOOL_PROGRESS_PROMPT_RE.test(toolProgressPrompt);
-  const toolProgressToolOutput = currentTurnIsToolProgress && currentUserTurn
-    ? extractToolOutput(input.slice(currentUserTurn.index))
-    : "";
+  const toolProgressToolOutput =
+    currentTurnIsToolProgress && currentUserTurn
+      ? extractToolOutput(input.slice(currentUserTurn.index))
+      : "";
   const toolProgressToolJson = parseToolOutputJson(toolProgressToolOutput);
   const buildToolProgressReadEvents = () => {
     return buildToolCallEventsWithArgs("read", {
@@ -2813,11 +2820,7 @@ async function buildResponsesPayload(
       });
     }
   }
-  if (
-    QA_IMAGE_GENERATION_PROMPT_RE.test(allInputText) &&
-    !toolOutput &&
-    !completedImageMediaPath
-  ) {
+  if (QA_IMAGE_GENERATION_PROMPT_RE.test(allInputText) && !toolOutput && !completedImageMediaPath) {
     return buildToolCallEventsWithArgs("image_generate", {
       prompt: "A QA lighthouse on a dark sea with a tiny protocol droid silhouette.",
       filename: "qa-lighthouse.png",

extensions/qa-matrix/src/runners/contract/runtime.test.ts

@@ -4,11 +4,20 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
 import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
 import { renderQaMarkdownReport } from "openclaw/plugin-sdk/qa-runtime";
 import { afterEach, describe, expect, it, vi } from "vitest";
+const { startMatrixQaFaultProxy } = vi.hoisted(() => ({
+  startMatrixQaFaultProxy: vi.fn(),
+}));
+
+vi.mock("../../substrate/fault-proxy.js", () => ({
+  startMatrixQaFaultProxy,
+}));
+
 import { testing as liveTesting } from "./runtime.js";
 
 afterEach(() => {
   vi.restoreAllMocks();
   vi.useRealTimers();
+  startMatrixQaFaultProxy.mockReset();
 });
 
 type MatrixQaSummaryInput = Parameters<typeof liveTesting.buildMatrixQaSummary>[0];
@@ -78,6 +87,24 @@ function buildMatrixQaSummaryInput(
 }
 
 describe("matrix live qa runtime", () => {
+  it("stops the Matrix harness when fault proxy startup fails", async () => {
+    const proxyError = new Error("fault proxy bind failed");
+    const harnessStop = vi.fn().mockResolvedValue(undefined);
+    startMatrixQaFaultProxy.mockRejectedValue(proxyError);
+
+    await expect(
+      liveTesting.startMatrixQaFaultProxyForHarness({
+        harness: {
+          baseUrl: "http://127.0.0.1:28008/",
+          stop: harnessStop,
+          stopCommand: "docker compose down --volumes",
+        },
+      }),
+    ).rejects.toBe(proxyError);
+
+    expect(harnessStop).toHaveBeenCalledOnce();
+  });
+
   it("uses unique default artifact directories", () => {
     const repoRoot = "/repo";
     const firstOutputDir = liveTesting.resolveMatrixQaOutputDir({ repoRoot });
@@ -86,9 +113,9 @@ describe("matrix live qa runtime", () => {
     expect(path.dirname(firstOutputDir)).toBe(path.join(repoRoot, ".artifacts", "qa-e2e"));
     expect(path.basename(firstOutputDir)).toMatch(/^matrix-[a-z0-9]+-[a-f0-9]{8}$/u);
     expect(secondOutputDir).not.toBe(firstOutputDir);
-    expect(
-      liveTesting.resolveMatrixQaOutputDir({ outputDir: ".artifacts/custom", repoRoot }),
-    ).toBe(".artifacts/custom");
+    expect(liveTesting.resolveMatrixQaOutputDir({ outputDir: ".artifacts/custom", repoRoot })).toBe(
+      ".artifacts/custom",
+    );
   });
 
   it("prints Matrix QA progress by default for non-interactive runs", () => {

extensions/qa-matrix/src/runners/contract/runtime.ts

@@ -18,6 +18,7 @@ import {
   type QaReportCheck,
 } from "openclaw/plugin-sdk/qa-runtime";
 import { normalizeQaProviderMode, type QaProviderModeInput } from "../../run-config.js";
+import { createLiveTransportQaRunId } from "../../shared/live-transport-artifacts.js";
 import { buildMatrixQaObservedEventsArtifact } from "../../substrate/artifacts.js";
 import { provisionMatrixQaRoom, type MatrixQaProvisionResult } from "../../substrate/client.js";
 import {
@@ -28,8 +29,8 @@ import {
   type MatrixQaConfigSnapshot,
 } from "../../substrate/config.js";
 import type { MatrixQaObservedEvent } from "../../substrate/events.js";
+import { startMatrixQaFaultProxy } from "../../substrate/fault-proxy.js";
 import { startMatrixQaHarness } from "../../substrate/harness.runtime.js";
-import { createLiveTransportQaRunId } from "../../shared/live-transport-artifacts.js";
 import { resolveMatrixQaModels, type ResolvedMatrixQaModels } from "./model-selection.js";
 import type { MatrixQaSyncStreams } from "./scenario-runtime-shared.js";
 import {
@@ -324,6 +325,27 @@ async function cleanupMatrixQaResource(params: {
   }
 }
 
+async function startMatrixQaFaultProxyForHarness(params: {
+  harness: Pick<
+    Awaited<ReturnType<typeof startMatrixQaHarness>>,
+    "baseUrl" | "stop" | "stopCommand"
+  >;
+}) {
+  try {
+    return await startMatrixQaFaultProxy({
+      rules: [],
+      targetBaseUrl: params.harness.baseUrl,
+    });
+  } catch (error) {
+    await cleanupMatrixQaResource({
+      label: "Matrix homeserver cleanup after fault proxy startup failure",
+      action: () => params.harness.stop(),
+      recovery: params.harness.stopCommand,
+    }).catch(() => {});
+    throw error;
+  }
+}
+
 function countMatrixQaStatuses(entries: Array<{ status: "fail" | "pass" | "skip" }>) {
   return {
     failed: entries.filter((entry) => entry.status === "fail").length,
@@ -702,6 +724,9 @@ export async function runMatrixQaLive(params: {
   writeMatrixQaProgress(
     `harness ready ${formatMatrixQaDurationMs(harnessBootMs)} baseUrl=${harness.baseUrl}`,
   );
+  const faultProxy = await startMatrixQaFaultProxyForHarness({
+    harness,
+  });
   const { durationMs: provisioningMs, result: provisioning } = await (async () => {
     try {
       return await measureMatrixQaStep(() =>
@@ -723,6 +748,7 @@ export async function runMatrixQaLive(params: {
         action: () => harness.stop(),
         recovery: harness.stopCommand,
       }).catch(() => {});
+      await faultProxy.stop().catch(() => {});
       throw error;
     }
   })();
@@ -763,7 +789,7 @@ export async function runMatrixQaLive(params: {
   const gatewayConfigParams = {
     driverAccessToken: provisioning.driver.accessToken,
     driverUserId: provisioning.driver.userId,
-    homeserver: harness.baseUrl,
+    homeserver: faultProxy.baseUrl,
     observerAccessToken: provisioning.observer.accessToken,
     observerUserId: provisioning.observer.userId,
     sutAccessToken: provisioning.sut.accessToken,
@@ -940,6 +966,7 @@ export async function runMatrixQaLive(params: {
                 gatewayWorkspaceDir: scenarioGateway.harness.gateway.workspaceDir,
                 gatewayCall: async (method, paramsLocal, opts) =>
                   await scenarioGateway.harness.gateway.call(method, paramsLocal ?? {}, opts),
+                faultProxy,
                 outputDir,
                 registrationToken: harness.registrationToken,
                 restartGateway: async () => {
@@ -1099,6 +1126,14 @@ export async function runMatrixQaLive(params: {
         appendLiveLaneIssue(cleanupErrors, "live gateway cleanup", error);
       }
     }
+    try {
+      await cleanupMatrixQaResource({
+        label: "Matrix homeserver fault proxy cleanup",
+        action: () => faultProxy.stop(),
+      });
+    } catch (error) {
+      appendLiveLaneIssue(cleanupErrors, "Matrix fault proxy cleanup", error);
+    }
     try {
       await cleanupMatrixQaResource({
         label: "Matrix homeserver cleanup",
@@ -1286,6 +1321,7 @@ export const testing = {
   resolveMatrixQaCanaryTimeoutMs,
   resolveMatrixQaModels,
   shouldWriteMatrixQaProgress,
+  startMatrixQaFaultProxyForHarness,
   summarizeMatrixQaGatewayStderrLog,
   summarizeMatrixQaConfigSnapshot,
   waitForMatrixChannelReady,

extensions/qa-matrix/src/runners/contract/scenario-catalog.ts

@@ -13,6 +13,7 @@ import {
   type MatrixQaProvisionedTopology,
   type MatrixQaTopologySpec,
 } from "../../substrate/topology.js";
+import { MATRIX_QA_VOICE_PREFLIGHT_MENTION } from "./scenario-media-fixtures.js";
 
 type MatrixQaScenarioId =
   | "matrix-thread-follow-up"
@@ -514,10 +515,10 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [
       audio: {
         enabled: true,
       },
-      groupMentionPatterns: ["\\S"],
+      groupMentionPatterns: [MATRIX_QA_VOICE_PREFLIGHT_MENTION],
     },
-    providerMode: "live-frontier",
-    timeoutMs: 180_000,
+    providerMode: "mock-openai",
+    timeoutMs: 60_000,
     title: "Matrix voice notes can trigger mention gating through transcription",
     topology: MATRIX_QA_MEDIA_ROOM_TOPOLOGY,
   },

extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee.ts

@@ -1379,42 +1379,24 @@ export async function runMatrixQaE2eeStateAfterMissingEncryptionScenario(
   if (!context.restartGatewayAfterStateMutation) {
     throw new Error("Matrix E2EE state_after QA scenario requires hard gateway restart support");
   }
-  const accountId = context.sutAccountId ?? "sut";
-  const configPath = requireMatrixQaGatewayConfigPath(context);
-  const originalAccountConfig = await readMatrixQaGatewayMatrixAccount({
-    accountId,
-    configPath,
-  });
-  const proxy = await startMatrixQaFaultProxy({
-    targetBaseUrl: context.baseUrl,
-    rules: [buildSyncStateAfterMissingEncryptionFaultRule(context.sutAccessToken)],
-  });
-  let gatewayPatched = false;
-  try {
-    await context.restartGatewayAfterStateMutation(
-      async () => {
-        await patchMatrixQaGatewayMatrixAccount({
-          accountId,
-          accountPatch: {
-            homeserver: proxy.baseUrl,
-            network: {
-              dangerouslyAllowPrivateNetwork: true,
-            },
-          },
-          configPath,
-        });
-        gatewayPatched = true;
-      },
-      {
-        timeoutMs: context.timeoutMs,
-        waitAccountId: accountId,
-      },
+  if (!context.faultProxy) {
+    throw new Error(
+      "Matrix E2EE state_after QA scenario requires the stable homeserver fault proxy",
     );
+  }
+  const accountId = context.sutAccountId ?? "sut";
+  const faultRule = buildSyncStateAfterMissingEncryptionFaultRule(context.sutAccessToken);
+  context.faultProxy.installRule(faultRule);
+  try {
+    await context.restartGatewayAfterStateMutation(async () => undefined, {
+      timeoutMs: context.timeoutMs,
+      waitAccountId: accountId,
+    });
     const result = await runMatrixQaE2eeTopLevelScenario(context, {
       scenarioId: "matrix-e2ee-state-after-missing-encryption",
       tokenPrefix: "MATRIX_QA_E2EE_STATE_AFTER",
     });
-    const stateAfterHits = proxy
+    const stateAfterHits = context.faultProxy
       .hits()
       .filter((hit) => hit.ruleId === MATRIX_QA_SYNC_STATE_AFTER_FAULT_RULE_ID);
     if (stateAfterHits.length > 0) {
@@ -1425,7 +1407,6 @@ export async function runMatrixQaE2eeStateAfterMissingEncryptionScenario(
     return {
       artifacts: {
         driverEventId: result.driverEventId,
-        faultProxyBaseUrl: proxy.baseUrl,
         reply: result.reply,
         roomKey: result.roomKey,
         roomId: result.roomId,
@@ -1437,30 +1418,12 @@ export async function runMatrixQaE2eeStateAfterMissingEncryptionScenario(
         `encrypted room key: ${result.roomKey}`,
         `encrypted room id: ${result.roomId}`,
         `driver event: ${result.driverEventId}`,
-        `fault proxy: ${proxy.baseUrl}`,
         `state_after sync opt-in hits: ${stateAfterHits.length}`,
         ...buildMatrixReplyDetails("E2EE state_after reply", result.reply),
       ].join("\n"),
     };
   } finally {
-    if (gatewayPatched) {
-      await context
-        .restartGatewayAfterStateMutation(
-          async () => {
-            await replaceMatrixQaGatewayMatrixAccount({
-              accountConfig: originalAccountConfig,
-              accountId,
-              configPath,
-            });
-          },
-          {
-            timeoutMs: context.timeoutMs,
-            waitAccountId: accountId,
-          },
-        )
-        .catch(() => undefined);
-    }
-    await proxy.stop().catch(() => undefined);
+    context.faultProxy.removeRule(faultRule.id);
   }
 }
 

extensions/qa-matrix/src/runners/contract/scenario-runtime-media.ts

@@ -74,7 +74,9 @@ function normalizeMatrixQaVoiceReply(value: string | undefined) {
 }
 
 function hasMatrixQaVoicePreflightReply(body: string | undefined) {
-  return normalizeMatrixQaVoiceReply(body).includes(MATRIX_QA_VOICE_PREFLIGHT_REPLY_MARKER);
+  return normalizeMatrixQaVoiceReply(body).includes(
+    normalizeMatrixQaVoiceReply(MATRIX_QA_VOICE_PREFLIGHT_REPLY_MARKER),
+  );
 }
 
 export async function runImageUnderstandingAttachmentScenario(context: MatrixQaScenarioContext) {

extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts

@@ -2,6 +2,7 @@
 import { randomUUID } from "node:crypto";
 import { createMatrixQaClient, type MatrixQaRoomObserver } from "../../substrate/client.js";
 import type { MatrixQaObservedEvent } from "../../substrate/events.js";
+import type { MatrixQaFaultProxy } from "../../substrate/fault-proxy.js";
 import { createMatrixQaRoomObserver } from "../../substrate/sync.js";
 import type { MatrixQaProvisionedTopology } from "../../substrate/topology.js";
 import { resolveMatrixQaScenarioRoomId } from "./scenario-catalog.js";
@@ -36,6 +37,7 @@ export type MatrixQaScenarioContext = {
     params?: Record<string, unknown>,
     opts?: { expectFinal?: boolean; timeoutMs?: number },
   ) => Promise<unknown>;
+  faultProxy?: Pick<MatrixQaFaultProxy, "hits" | "installRule" | "removeRule">;
   outputDir?: string;
   registrationToken?: string;
   restartGateway?: () => Promise<void>;
@@ -94,9 +96,9 @@ export function buildMatrixPartialStreamingPrompt(sutUserId: string, text: strin
 }
 
 export const MATRIX_QA_TOOL_PROGRESS_TASK_FILENAME = "QA_KICKOFF_TASK.md";
-export const MATRIX_QA_TOOL_PROGRESS_MENTION_FILENAME =
-  "matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt";
 export const MATRIX_QA_TOOL_PROGRESS_COMMAND = "printf 'matrix-command-progress-start\\n'; sleep 2";
+export const MATRIX_QA_TOOL_PROGRESS_MENTION_COMMAND =
+  "printf '@room @alice:matrix-qa.test !room:matrix-qa.test\\n'; sleep 2";
 
 export function buildMatrixToolProgressTaskContent(text: string) {
   return [
@@ -133,10 +135,10 @@ export function buildMatrixToolProgressErrorPrompt(sutUserId: string, text: stri
 
 export function buildMatrixToolProgressMentionSafetyPrompt(sutUserId: string, text: string) {
   return [
-    `${sutUserId} Tool progress QA check: read the missing workspace file \`${MATRIX_QA_TOOL_PROGRESS_MENTION_FILENAME}\` before answering.`,
-    `The QA harness must observe that failed read in a Matrix tool-progress preview.`,
-    `Do not guess or send any marker before the tool result returns.`,
-    `After that read fails, reply exactly \`${text}\`.`,
+    `${sutUserId} Tool progress QA check: call the exec tool exactly once with this exact command before answering: \`${MATRIX_QA_TOOL_PROGRESS_MENTION_COMMAND}\`.`,
+    "The QA harness must observe that command in a Matrix tool-progress preview with all mention-looking text inert.",
+    "Do not guess or send any marker before the tool result returns.",
+    `After that command completes or fails, reply exactly \`${text}\`.`,
   ].join(" ");
 }
 

extensions/qa-matrix/src/runners/contract/scenarios.test.ts

@@ -59,6 +59,7 @@ import type { MatrixQaObservedEvent } from "../../substrate/events.js";
 import {
   MATRIX_QA_MEDIA_TYPE_COVERAGE_CASES,
   MATRIX_QA_VOICE_PREFLIGHT_FILENAME,
+  MATRIX_QA_VOICE_PREFLIGHT_MENTION,
   MATRIX_QA_VOICE_PREFLIGHT_REPLY_MARKER,
 } from "./scenario-media-fixtures.js";
 import {
@@ -1152,13 +1153,9 @@ describe("matrix live qa scenarios", () => {
           },
         },
       });
-      const proxyStop = vi.fn().mockResolvedValue(undefined);
-      const proxyHits = vi.fn().mockReturnValue([]);
-      startMatrixQaFaultProxy.mockResolvedValue({
-        baseUrl: "http://127.0.0.1:39879",
-        hits: proxyHits,
-        stop: proxyStop,
-      });
+      const faultProxyHits = vi.fn().mockReturnValue([]);
+      const installFaultRule = vi.fn();
+      const removeFaultRule = vi.fn();
       let replyToken = "";
       const driverStop = vi.fn().mockResolvedValue(undefined);
       const driverClient = {
@@ -1195,6 +1192,11 @@ describe("matrix live qa scenarios", () => {
           OPENCLAW_CONFIG_PATH: gatewayConfigPath,
           PATH: process.env.PATH,
         },
+        faultProxy: {
+          hits: faultProxyHits,
+          installRule: installFaultRule,
+          removeRule: removeFaultRule,
+        },
         outputDir,
         restartGatewayAfterStateMutation,
         sutAccountId: "sut",
@@ -1249,11 +1251,13 @@ describe("matrix live qa scenarios", () => {
         "http://127.0.0.1:28008/",
       );
       expect(restoredConfig.channels.matrix.accounts.sut.network).toEqual({ existing: true });
-      expect(restartGatewayAfterStateMutation).toHaveBeenCalledTimes(2);
-      expect(proxyStop).toHaveBeenCalledTimes(1);
+      expect(restartGatewayAfterStateMutation).toHaveBeenCalledTimes(1);
+      expect(startMatrixQaFaultProxy).not.toHaveBeenCalled();
+      expect(installFaultRule).toHaveBeenCalledTimes(1);
+      expect(removeFaultRule).toHaveBeenCalledWith("sync-state-after-missing-encryption");
 
-      const proxyArgs = mockObjectArg(startMatrixQaFaultProxy, "startMatrixQaFaultProxy") as {
-        rules: Array<{
+      const [faultRule] = installFaultRule.mock.calls[0] as [
+        {
           match: (params: {
             bearerToken?: string;
             headers: Record<string, string>;
@@ -1279,14 +1283,11 @@ describe("matrix live qa scenarios", () => {
                 headers: Headers;
                 status: number;
               }>;
-        }>;
-        targetBaseUrl?: unknown;
-      };
-      const [faultRule] = proxyArgs.rules;
+        },
+      ];
       if (!faultRule) {
         throw new Error("expected Matrix QA fault proxy rule");
       }
-      expect(proxyArgs.targetBaseUrl).toBe("http://127.0.0.1:28008/");
       expect(
         faultRule.match({
           bearerToken: "sut-token",
@@ -4106,9 +4107,9 @@ describe("matrix live qa scenarios", () => {
           event: matrixQaMessageEvent({
             kind: "message",
             eventId: "$tool-progress-mention-edit",
-            body: "Working...\n- `read matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt failed`",
+            body: "Working...\n- `exec printf '@room @alice:matrix-qa.test !room:matrix-qa.test\\n'; sleep 2`",
             formattedBody:
-              "Working...<br><ul><li><code>read matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt failed</code></li></ul>",
+              "Working...<br><ul><li><code>exec printf '@room @alice:matrix-qa.test !room:matrix-qa.test\\n'; sleep 2</code></li></ul>",
             mentions: {},
             relatesTo: {
               relType: "m.replace",
@@ -4151,9 +4152,12 @@ describe("matrix live qa scenarios", () => {
     expect(artifacts.reply?.eventId).toBe("$tool-progress-mention-final");
     const prompt = mockMessageBody(sendTextMessage, "sendTextMessage");
     expect(prompt).toContain(
-      "read the missing workspace file `matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt` before answering",
+      "call the exec tool exactly once with this exact command before answering",
     );
-    expect(prompt).toContain("The QA harness must observe that failed read");
+    expect(prompt).toContain(
+      "printf '@room @alice:matrix-qa.test !room:matrix-qa.test\\n'; sleep 2",
+    );
+    expect(prompt).toContain("mention-looking text inert");
   });
 
   it("rejects active Matrix mentions in final-first tool-progress previews", async () => {
@@ -4176,9 +4180,9 @@ describe("matrix live qa scenarios", () => {
           event: matrixQaMessageEvent({
             kind: "message",
             eventId: "$tool-progress-mention-final-first-progress",
-            body: "Working...\n- `read matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt failed`",
+            body: "Working...\n- `exec printf '@room @alice:matrix-qa.test !room:matrix-qa.test\\n'; sleep 2`",
             formattedBody:
-              "Working...<br><ul><li><code>read matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt failed</code></li></ul>",
+              "Working...<br><ul><li><code>exec printf '@room @alice:matrix-qa.test !room:matrix-qa.test\\n'; sleep 2</code></li></ul>",
             mentions: {
               room: true,
               userIds: ["@alice:matrix-qa.test"],
@@ -4195,7 +4199,7 @@ describe("matrix live qa scenarios", () => {
       /active mentions/,
     );
     expect(mockMessageBody(sendTextMessage, "sendTextMessage")).toContain(
-      "read the missing workspace file",
+      "call the exec tool exactly once",
     );
   });
 
@@ -4219,9 +4223,9 @@ describe("matrix live qa scenarios", () => {
           event: matrixQaMessageEvent({
             kind: "message",
             eventId: "$tool-progress-mention-top-level-progress",
-            body: "⚠️ 🛠️ `show matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt (workspace)` failed",
+            body: "⚠️ 🛠️ `exec printf '@room @alice:matrix-qa.test !room:matrix-qa.test\\n'; sleep 2`",
             formattedBody:
-              "<p>⚠️ 🛠️ <code>show matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt (workspace)</code> failed</p>",
+              "<p>⚠️ 🛠️ <code>exec printf '@room @alice:matrix-qa.test !room:matrix-qa.test\\n'; sleep 2</code></p>",
             mentions: {},
           }),
           since: "driver-sync-progress",
@@ -4239,7 +4243,7 @@ describe("matrix live qa scenarios", () => {
       reply?: { eventId?: unknown };
     };
     expect(artifacts.previewEventId).toBe("$tool-progress-mention-top-level-progress");
-    expect(artifacts.previewFormattedBodyPreview).toContain("<code>show matrix-progress-@room");
+    expect(artifacts.previewFormattedBodyPreview).toContain("<code>exec printf '@room");
     expect(artifacts.previewMentions).toEqual({});
     expect(artifacts.reply?.eventId).toBe("$tool-progress-mention-top-level-final");
   });
@@ -4251,9 +4255,9 @@ describe("matrix live qa scenarios", () => {
     const previewEvent = matrixQaMessageEvent({
       kind: "message",
       eventId: "$tool-progress-mention-stale-preview",
-      body: "Working...\n- `read matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt failed`",
+      body: "Working...\n- `exec printf '@room @alice:matrix-qa.test !room:matrix-qa.test\\n'; sleep 2`",
       formattedBody:
-        "Working...<br><ul><li><code>read matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt failed</code></li></ul>",
+        "Working...<br><ul><li><code>exec printf '@room @alice:matrix-qa.test !room:matrix-qa.test\\n'; sleep 2</code></li></ul>",
       mentions: {},
     });
     const waitForRoomEvent = vi
@@ -4770,7 +4774,10 @@ describe("matrix live qa scenarios", () => {
 
     const scenario = requireMatrixQaScenario("matrix-voice-preflight-mention");
     expect(scenario.configOverrides?.audio?.enabled).toBe(true);
-    expect(scenario.configOverrides?.groupMentionPatterns).toEqual(["\\S"]);
+    expect(scenario.configOverrides?.groupMentionPatterns).toEqual([
+      MATRIX_QA_VOICE_PREFLIGHT_MENTION,
+    ]);
+    expect(scenario.providerMode).toBe("mock-openai");
 
     const result = await runMatrixQaScenario(scenario, {
       baseUrl: "http://127.0.0.1:28008/",

extensions/qa-matrix/src/substrate/fault-proxy.test.ts

@@ -173,6 +173,37 @@ describe("Matrix QA fault proxy", () => {
     ]);
   });
 
+  it("installs and removes a fault rule without changing the proxy endpoint", async () => {
+    const target = await startTargetServer();
+    proxy = await startMatrixQaFaultProxy({
+      targetBaseUrl: target.baseUrl,
+      rules: [],
+    });
+    const baseUrl = proxy.baseUrl;
+    proxy.installRule({
+      id: "temporary-sync-fault",
+      match: (request) => request.method === "GET" && request.path === "/_matrix/client/v3/sync",
+      response: () => ({ body: { faulted: true }, status: 418 }),
+    });
+
+    const faulted = await fetch(`${baseUrl}/_matrix/client/v3/sync`);
+    expect(faulted.status).toBe(418);
+    await expect(faulted.json()).resolves.toEqual({ faulted: true });
+
+    proxy.removeRule("temporary-sync-fault");
+    expect(proxy.baseUrl).toBe(baseUrl);
+    const forwarded = await fetch(`${baseUrl}/_matrix/client/v3/sync`);
+    expect(forwarded.status).toBe(200);
+    await expect(forwarded.json()).resolves.toEqual({ forwarded: true });
+    expect(proxy.hits()).toEqual([
+      {
+        method: "GET",
+        path: "/_matrix/client/v3/sync",
+        ruleId: "temporary-sync-fault",
+      },
+    ]);
+  });
+
   it("rejects oversized forwarded request bodies before contacting the target", async () => {
     const target = await startTargetServer();
     proxy = await startMatrixQaFaultProxy({

extensions/qa-matrix/src/substrate/fault-proxy.ts

@@ -73,6 +73,8 @@ export type MatrixQaFaultProxyHit = {
 export type MatrixQaFaultProxy = {
   baseUrl: string;
   hits(): MatrixQaFaultProxyHit[];
+  installRule(rule: MatrixQaFaultProxyRule): void;
+  removeRule(ruleId: string): void;
   stop(): Promise<void>;
 };
 
@@ -300,6 +302,7 @@ export async function startMatrixQaFaultProxy(params: {
   const maxRequestBytes = params.maxRequestBytes ?? DEFAULT_FAULT_PROXY_REQUEST_MAX_BYTES;
   const maxResponseBytes = params.maxResponseBytes ?? DEFAULT_FAULT_PROXY_RESPONSE_MAX_BYTES;
   const hits: MatrixQaFaultProxyHit[] = [];
+  const rules = new Map(params.rules.map((rule) => [rule.id, rule]));
   const server = createServer((req, res) => {
     void (async () => {
       try {
@@ -314,7 +317,7 @@ export async function startMatrixQaFaultProxy(params: {
           search: requestUrl.search,
         };
         const body = await readRequestBody(req, maxRequestBytes);
-        const rule = params.rules.find((candidate) => candidate.match(request));
+        const rule = [...rules.values()].find((candidate) => candidate.match(request));
         if (rule) {
           hits.push({
             method: request.method,
@@ -380,6 +383,12 @@ export async function startMatrixQaFaultProxy(params: {
   return {
     baseUrl: `http://127.0.0.1:${address.port}`,
     hits: () => [...hits],
+    installRule: (rule) => {
+      rules.set(rule.id, rule);
+    },
+    removeRule: (ruleId) => {
+      rules.delete(ruleId);
+    },
     stop: async () => {
       await new Promise<void>((resolve, reject) => {
         server.close((error) => {