diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts index b99a85f9d9ee..3d8c067f9c87 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts @@ -478,22 +478,26 @@ describe("Slack live QA runtime helpers", () => { const cases = [ { id: "slack-progress-commentary-true", - commentaryTs: "2.000000", + commentaryTs: "1.500000", + commentaryStyle: "lane", toolProgress: "absent", }, { id: "slack-progress-commentary-false", - commentaryTs: undefined, + commentaryTs: "1.500000", + commentaryStyle: "headline", toolProgress: "absent", }, { id: "slack-progress-commentary-omitted", - commentaryTs: "2.000000", + commentaryTs: "1.500000", + commentaryStyle: "headline", toolProgress: "draft", }, { id: "slack-progress-commentary-verbose-dedupe", commentaryTs: "1.500000", + commentaryStyle: "standalone", toolProgress: "standalone", }, ] as const; @@ -519,7 +523,8 @@ describe("Slack live QA runtime helpers", () => { ? [ { channelId: "C123456789", - text: `💬 ${commentaryMarker}`, + text: + testCase.commentaryStyle === "lane" ? `💬 ${commentaryMarker}` : commentaryMarker, ts: testCase.commentaryTs, }, ] @@ -530,7 +535,7 @@ describe("Slack live QA runtime helpers", () => { { channelId: "C123456789", text: `🛠️ Exec ${toolMarker}`, - ts: testCase.toolProgress === "draft" ? "2.000000" : "1.750000", + ts: testCase.toolProgress === "draft" ? "1.500000" : "1.750000", }, ]), ]; @@ -576,28 +581,31 @@ describe("Slack live QA runtime helpers", () => { messages: mutate(completeMarkers).map((text) => ({ channelId: "C123456789", text, - ts: "2.000000", + ts: text.includes(completeMarkers[2]) ? "2.000000" : "1.500000", })), }); }; expect( - verify("slack-progress-commentary-false", ([commentary, , final]) => [commentary, final]), - ).toThrow("commentary to stay out"); + verify("slack-progress-commentary-false", ([commentary, , final]) => [ + `💬 ${commentary}`, + final, + ]), + ).toThrow("status headline"); expect( verify("slack-progress-commentary-true", ([commentary, tool, final]) => [ - commentary, + `💬 ${commentary}`, tool, final, ]), ).toThrow("tool progress to stay out"); expect( verify("slack-progress-commentary-omitted", ([commentary, , final]) => [commentary, final]), - ).toThrow("tool progress on the progress draft"); + ).toThrow("tool progress on the draft"); expect( verify( "slack-progress-commentary-true", - ([commentary, , final]) => [`${commentary} ${final}`], + ([commentary, , final]) => [`💬 ${commentary} ${final}`], "echo", ), ).toThrow("only the final marker"); diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-fixtures.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-fixtures.ts index ab272fc853bc..3dc93b5b9deb 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-fixtures.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-fixtures.ts @@ -73,7 +73,7 @@ export function renderSlackTableAccessibleText(summaryText: string) { } type SlackProgressCommentaryExpectation = { - commentary: "absent" | "draft" | "standalone"; + commentary: "headline" | "lane" | "standalone"; toolProgress: "absent" | "draft" | "standalone"; }; @@ -109,20 +109,32 @@ export function buildSlackProgressCommentaryRun( message.text.includes(commentaryMarker), ); const commentaryTimestamps = new Set(commentaryMessages.map((message) => message.ts)); - if (expectation.commentary === "absent" && commentaryTimestamps.size !== 0) { - throw new Error("expected commentary to stay out of Slack progress messages"); - } - if (expectation.commentary !== "absent" && commentaryTimestamps.size !== 1) { + const [commentaryTs] = commentaryTimestamps; + if (commentaryTimestamps.size !== 1 || commentaryTs === undefined) { throw new Error( `expected exactly one Slack message identity containing commentary; got ${commentaryTimestamps.size}`, ); } - const commentaryTs = [...commentaryTimestamps][0]; - if (expectation.commentary === "draft" && commentaryTs !== finalMessage.ts) { - throw new Error("expected commentary on the progress draft finalized as the answer"); + if (commentaryTs === finalMessage.ts) { + throw new Error("expected Slack progress commentary to stay separate from the fresh final"); } - if (expectation.commentary === "standalone" && commentaryTs === finalMessage.ts) { - throw new Error("expected commentary only in the standalone verbose message"); + const commentaryLaneTimestamps = new Set( + commentaryMessages + .filter((message) => message.text.includes(`💬 ${commentaryMarker}`)) + .map((message) => message.ts), + ); + if ( + expectation.commentary === "lane" && + (commentaryLaneTimestamps.size !== 1 || !commentaryLaneTimestamps.has(commentaryTs)) + ) { + throw new Error("expected commentary in the Slack progress commentary lane"); + } + if (expectation.commentary !== "lane" && commentaryLaneTimestamps.size !== 0) { + throw new Error( + expectation.commentary === "headline" + ? "expected the preamble as the Slack progress status headline" + : "expected commentary only in the standalone verbose message", + ); } const toolTimestamps = new Set( progressMessages @@ -130,8 +142,11 @@ export function buildSlackProgressCommentaryRun( .map((message) => message.ts), ); if (expectation.toolProgress === "draft") { - if (toolTimestamps.size !== 1 || !toolTimestamps.has(finalMessage.ts)) { - throw new Error("expected tool progress on the progress draft finalized as the answer"); + if (toolTimestamps.size !== 1 || toolTimestamps.has(finalMessage.ts)) { + throw new Error("expected tool progress on the draft separate from the fresh final"); + } + if (expectation.commentary !== "standalone" && !toolTimestamps.has(commentaryTs)) { + throw new Error("expected commentary and tool progress on one Slack draft identity"); } } else if (expectation.toolProgress === "standalone") { if (toolTimestamps.size === 0 || toolTimestamps.has(finalMessage.ts)) { @@ -151,11 +166,11 @@ export function buildSlackProgressCommentaryRun( ); } const commentaryDetails = - expectation.commentary === "draft" - ? "commentary on progress/final identity" + expectation.commentary === "lane" + ? "commentary in the progress lane" : expectation.commentary === "standalone" ? "one standalone commentary identity" - : "commentary absent from Slack progress"; + : "preamble in the progress status headline"; return `verified ${commentaryDetails}; tool progress ${expectation.toolProgress}; final identity unique`; }, }; diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-implementations.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-implementations.ts index 4c9a1db711ff..ed048546576f 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-implementations.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-implementations.ts @@ -212,7 +212,7 @@ export const slackQaProgressCommentaryTrueScenario: SlackQaScenarioImplementatio }, buildRun: (sutUserId) => buildSlackProgressCommentaryRun(sutUserId, { - commentary: "draft", + commentary: "lane", toolProgress: "absent", }), }; @@ -223,7 +223,7 @@ export const slackQaProgressCommentaryFalseScenario: SlackQaScenarioImplementati }, buildRun: (sutUserId) => buildSlackProgressCommentaryRun(sutUserId, { - commentary: "absent", + commentary: "headline", toolProgress: "absent", }), }; @@ -234,7 +234,7 @@ export const slackQaProgressCommentaryOmittedScenario: SlackQaScenarioImplementa }, buildRun: (sutUserId) => buildSlackProgressCommentaryRun(sutUserId, { - commentary: "draft", + commentary: "headline", toolProgress: "draft", }), }; diff --git a/qa/scenarios/channels/slack-progress-commentary-false.yaml b/qa/scenarios/channels/slack-progress-commentary-false.yaml index 9c51196069ee..7ef82dafcddd 100644 --- a/qa/scenarios/channels/slack-progress-commentary-false.yaml +++ b/qa/scenarios/channels/slack-progress-commentary-false.yaml @@ -1,4 +1,4 @@ -title: Slack progress commentary false stays out of the progress draft +title: Slack progress commentary false uses the status headline instead of the commentary lane scenario: id: slack-progress-commentary-false surface: channels