fix(qa): align Slack progress scenarios with fresh finals (#116445)

* fix(qa): verify Slack progress drafts against fresh finals

* fix(qa): narrow Slack progress message identity

extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts

@@ -478,22 +478,26 @@ describe("Slack live QA runtime helpers", () => {
     const cases = [
       {
         id: "slack-progress-commentary-true",
-        commentaryTs: "2.000000",
+        commentaryTs: "1.500000",
+        commentaryStyle: "lane",
         toolProgress: "absent",
       },
       {
         id: "slack-progress-commentary-false",
-        commentaryTs: undefined,
+        commentaryTs: "1.500000",
+        commentaryStyle: "headline",
         toolProgress: "absent",
       },
       {
         id: "slack-progress-commentary-omitted",
-        commentaryTs: "2.000000",
+        commentaryTs: "1.500000",
+        commentaryStyle: "headline",
         toolProgress: "draft",
       },
       {
         id: "slack-progress-commentary-verbose-dedupe",
         commentaryTs: "1.500000",
+        commentaryStyle: "standalone",
         toolProgress: "standalone",
       },
     ] as const;
@@ -519,7 +523,8 @@ describe("Slack live QA runtime helpers", () => {
           ? [
               {
                 channelId: "C123456789",
-                text: `💬 ${commentaryMarker}`,
+                text:
+                  testCase.commentaryStyle === "lane" ? `💬 ${commentaryMarker}` : commentaryMarker,
                 ts: testCase.commentaryTs,
               },
             ]
@@ -530,7 +535,7 @@ describe("Slack live QA runtime helpers", () => {
               {
                 channelId: "C123456789",
                 text: `🛠️ Exec ${toolMarker}`,
-                ts: testCase.toolProgress === "draft" ? "2.000000" : "1.750000",
+                ts: testCase.toolProgress === "draft" ? "1.500000" : "1.750000",
               },
             ]),
       ];
@@ -576,28 +581,31 @@ describe("Slack live QA runtime helpers", () => {
           messages: mutate(completeMarkers).map((text) => ({
             channelId: "C123456789",
             text,
-            ts: "2.000000",
+            ts: text.includes(completeMarkers[2]) ? "2.000000" : "1.500000",
           })),
         });
     };
 
     expect(
-      verify("slack-progress-commentary-false", ([commentary, , final]) => [commentary, final]),
-    ).toThrow("commentary to stay out");
+      verify("slack-progress-commentary-false", ([commentary, , final]) => [
+        `💬 ${commentary}`,
+        final,
+      ]),
+    ).toThrow("status headline");
     expect(
       verify("slack-progress-commentary-true", ([commentary, tool, final]) => [
-        commentary,
+        `💬 ${commentary}`,
         tool,
         final,
       ]),
     ).toThrow("tool progress to stay out");
     expect(
       verify("slack-progress-commentary-omitted", ([commentary, , final]) => [commentary, final]),
-    ).toThrow("tool progress on the progress draft");
+    ).toThrow("tool progress on the draft");
     expect(
       verify(
         "slack-progress-commentary-true",
-        ([commentary, , final]) => [`${commentary} ${final}`],
+        ([commentary, , final]) => [`💬 ${commentary} ${final}`],
         "echo",
       ),
     ).toThrow("only the final marker");

extensions/qa-lab/src/live-transports/slack/slack-live.scenario-fixtures.ts

@@ -73,7 +73,7 @@ export function renderSlackTableAccessibleText(summaryText: string) {
 }
 
 type SlackProgressCommentaryExpectation = {
-  commentary: "absent" | "draft" | "standalone";
+  commentary: "headline" | "lane" | "standalone";
   toolProgress: "absent" | "draft" | "standalone";
 };
 
@@ -109,20 +109,32 @@ export function buildSlackProgressCommentaryRun(
         message.text.includes(commentaryMarker),
       );
       const commentaryTimestamps = new Set(commentaryMessages.map((message) => message.ts));
-      if (expectation.commentary === "absent" && commentaryTimestamps.size !== 0) {
-        throw new Error("expected commentary to stay out of Slack progress messages");
-      }
-      if (expectation.commentary !== "absent" && commentaryTimestamps.size !== 1) {
+      const [commentaryTs] = commentaryTimestamps;
+      if (commentaryTimestamps.size !== 1 || commentaryTs === undefined) {
         throw new Error(
           `expected exactly one Slack message identity containing commentary; got ${commentaryTimestamps.size}`,
         );
       }
-      const commentaryTs = [...commentaryTimestamps][0];
-      if (expectation.commentary === "draft" && commentaryTs !== finalMessage.ts) {
-        throw new Error("expected commentary on the progress draft finalized as the answer");
+      if (commentaryTs === finalMessage.ts) {
+        throw new Error("expected Slack progress commentary to stay separate from the fresh final");
       }
-      if (expectation.commentary === "standalone" && commentaryTs === finalMessage.ts) {
-        throw new Error("expected commentary only in the standalone verbose message");
+      const commentaryLaneTimestamps = new Set(
+        commentaryMessages
+          .filter((message) => message.text.includes(`💬 ${commentaryMarker}`))
+          .map((message) => message.ts),
+      );
+      if (
+        expectation.commentary === "lane" &&
+        (commentaryLaneTimestamps.size !== 1 || !commentaryLaneTimestamps.has(commentaryTs))
+      ) {
+        throw new Error("expected commentary in the Slack progress commentary lane");
+      }
+      if (expectation.commentary !== "lane" && commentaryLaneTimestamps.size !== 0) {
+        throw new Error(
+          expectation.commentary === "headline"
+            ? "expected the preamble as the Slack progress status headline"
+            : "expected commentary only in the standalone verbose message",
+        );
       }
       const toolTimestamps = new Set(
         progressMessages
@@ -130,8 +142,11 @@ export function buildSlackProgressCommentaryRun(
           .map((message) => message.ts),
       );
       if (expectation.toolProgress === "draft") {
-        if (toolTimestamps.size !== 1 || !toolTimestamps.has(finalMessage.ts)) {
-          throw new Error("expected tool progress on the progress draft finalized as the answer");
+        if (toolTimestamps.size !== 1 || toolTimestamps.has(finalMessage.ts)) {
+          throw new Error("expected tool progress on the draft separate from the fresh final");
+        }
+        if (expectation.commentary !== "standalone" && !toolTimestamps.has(commentaryTs)) {
+          throw new Error("expected commentary and tool progress on one Slack draft identity");
         }
       } else if (expectation.toolProgress === "standalone") {
         if (toolTimestamps.size === 0 || toolTimestamps.has(finalMessage.ts)) {
@@ -151,11 +166,11 @@ export function buildSlackProgressCommentaryRun(
         );
       }
       const commentaryDetails =
-        expectation.commentary === "draft"
-          ? "commentary on progress/final identity"
+        expectation.commentary === "lane"
+          ? "commentary in the progress lane"
           : expectation.commentary === "standalone"
             ? "one standalone commentary identity"
-            : "commentary absent from Slack progress";
+            : "preamble in the progress status headline";
       return `verified ${commentaryDetails}; tool progress ${expectation.toolProgress}; final identity unique`;
     },
   };

extensions/qa-lab/src/live-transports/slack/slack-live.scenario-implementations.ts

@@ -212,7 +212,7 @@ export const slackQaProgressCommentaryTrueScenario: SlackQaScenarioImplementatio
   },
   buildRun: (sutUserId) =>
     buildSlackProgressCommentaryRun(sutUserId, {
-      commentary: "draft",
+      commentary: "lane",
       toolProgress: "absent",
     }),
 };
@@ -223,7 +223,7 @@ export const slackQaProgressCommentaryFalseScenario: SlackQaScenarioImplementati
   },
   buildRun: (sutUserId) =>
     buildSlackProgressCommentaryRun(sutUserId, {
-      commentary: "absent",
+      commentary: "headline",
       toolProgress: "absent",
     }),
 };
@@ -234,7 +234,7 @@ export const slackQaProgressCommentaryOmittedScenario: SlackQaScenarioImplementa
   },
   buildRun: (sutUserId) =>
     buildSlackProgressCommentaryRun(sutUserId, {
-      commentary: "draft",
+      commentary: "headline",
       toolProgress: "draft",
     }),
 };

qa/scenarios/channels/slack-progress-commentary-false.yaml

@@ -1,4 +1,4 @@
-title: Slack progress commentary false stays out of the progress draft
+title: Slack progress commentary false uses the status headline instead of the commentary lane
 scenario:
   id: slack-progress-commentary-false
   surface: channels