From bf5fef857aadc499ff98927f8b6ccc3b6eb0c321 Mon Sep 17 00:00:00 2001
From: Vincent Koc <vincentkoc@ieee.org>
Date: Wed, 27 May 2026 15:59:30 +0200
Subject: [PATCH] fix(qa): use read failure for Matrix mention progress

---
 .../src/runners/contract/scenario-runtime-room.ts   |  1 +
 .../src/runners/contract/scenario-runtime-shared.ts |  7 +++----
 .../src/runners/contract/scenarios.test.ts          | 13 +++++--------
 3 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts
index 0812e30e34e8..ffe34f1f1669 100644
--- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts
+++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts
@@ -1227,6 +1227,7 @@ export async function runToolProgressMentionSafetyScenario(context: MatrixQaScen
     expectedPreviewKind: "message",
     finalText: buildMatrixQaToken("MATRIX_QA_TOOL_PROGRESS_MENTION_SAFE"),
     label: "tool progress mention safety",
+    allowTopLevelFinalWithProgress: true,
     mentionSafety: true,
     progressPattern: /@room|@alice:matrix-qa\.test|!room:matrix-qa\.test/i,
     triggerBodyBuilder: buildMatrixToolProgressMentionSafetyPrompt,
diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts
index f4b7ed25c09b..b91d350beb2a 100644
--- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts
+++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts
@@ -121,12 +121,11 @@ export function buildMatrixToolProgressErrorPrompt(sutUserId: string, text: stri
 }
 
 export function buildMatrixToolProgressMentionSafetyPrompt(sutUserId: string, text: string) {
-  const command = `rg -n '${MATRIX_QA_TOOL_PROGRESS_MENTION_FILENAME}' . ; sleep 2`;
   return [
-    `${sutUserId} Tool progress QA check: call the exec tool exactly once with this exact command before answering: \`${command}\`.`,
-    `The QA harness must observe that exec tool call in a Matrix tool-progress preview.`,
+    `${sutUserId} Tool progress QA check: read the missing workspace file \`${MATRIX_QA_TOOL_PROGRESS_MENTION_FILENAME}\` before answering.`,
+    `The QA harness must observe that failed read in a Matrix tool-progress preview.`,
     `Do not guess or send any marker before the tool result returns.`,
-    `After that exec command completes or fails, reply exactly \`${text}\`.`,
+    `After that read fails, reply exactly \`${text}\`.`,
   ].join(" ");
 }
 
diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts
index a5b370cb11d9..5184343b32ec 100644
--- a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts
+++ b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts
@@ -3879,7 +3879,7 @@ describe("matrix live qa scenarios", () => {
           event: matrixQaMessageEvent({
             kind: "message",
             eventId: previewEventId,
-            body: "Working...\n- `tool: exec`",
+            body: "Working...\n- `tool: read`",
           }),
           since: "driver-sync-preview",
         },
@@ -3888,9 +3888,9 @@ describe("matrix live qa scenarios", () => {
             kind: "message",
             eventId: "$tool-progress-mention-edit",
             body:
-              'Working...\n- `search "matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt" in . -> run sleep 2`',
+              "Working...\n- `read matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt failed`",
             formattedBody:
-              'Working...<br><ul><li><code>search "matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt" in . -&gt; run sleep 2</code></li></ul>',
+              "Working...<br><ul><li><code>read matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt failed</code></li></ul>",
             mentions: {},
             relatesTo: {
               relType: "m.replace",
@@ -3933,12 +3933,9 @@ describe("matrix live qa scenarios", () => {
     expect(artifacts.reply?.eventId).toBe("$tool-progress-mention-final");
     const prompt = mockMessageBody(sendTextMessage, "sendTextMessage");
     expect(prompt).toContain(
-      "call the exec tool exactly once with this exact command before answering",
+      "read the missing workspace file `matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt` before answering",
     );
-    expect(prompt).toContain(
-      "`rg -n 'matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt' . ; sleep 2`",
-    );
-    expect(prompt).toContain("The QA harness must observe that exec tool call");
+    expect(prompt).toContain("The QA harness must observe that failed read");
   });
 
   it("preserves separate finalized block events when Matrix block streaming is enabled", async () => {