diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.invalid-blocks.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.invalid-blocks.ts index 2c50c56cf8e3..067085170ea1 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.invalid-blocks.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.invalid-blocks.ts @@ -8,9 +8,7 @@ import { function buildSlackInvalidBlocksTableRow(index: number) { const rowId = String(index).padStart(3, "0"); - // Cross both independently documented native-table limits. Slack has - // accepted the over-row-limit shape alone, so the probe also exceeds the - // 10,000-character aggregate cell contract. + // Keep the fallback large enough to catch accidental truncation. return [`row-${rowId}`, `value-${rowId}-${"x".repeat(96)}`] as const; } diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.observations.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.observations.ts index c41aaafce262..c96ee933fdab 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.observations.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.observations.ts @@ -304,6 +304,18 @@ export async function runSlackTableInvalidBlocksFallbackScenario( ): Promise { const probe = buildSlackInvalidBlocksTableProbe(); const oldestTs = ((Date.now() - 5_000) / 1_000).toFixed(6); + const originalPostMessage = context.sutWriteClient.chat.postMessage; + let rejectedNativeData = false; + context.sutWriteClient.chat.postMessage = (async (payload) => { + const payloadRecord = payload as { blocks?: unknown }; + if (!rejectedNativeData && countSlackNativeDataBlocks(payloadRecord.blocks) > 0) { + rejectedNativeData = true; + throw Object.assign(new Error("Slack API error: invalid_blocks"), { + data: { error: "invalid_blocks", ok: false }, + }); + } + return originalPostMessage.call(context.sutWriteClient.chat, payload); + }) as typeof context.sutWriteClient.chat.postMessage; const instrumentation = instrumentSlackPostMessage(context.sutWriteClient); let sent: Awaited>; try { @@ -328,6 +340,7 @@ export async function runSlackTableInvalidBlocksFallbackScenario( } } finally { instrumentation.restore(); + context.sutWriteClient.chat.postMessage = originalPostMessage; } const [nativeAttempt, fallbackAttempt] = instrumentation.attempts; diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts index 108a895e3042..bf0a2d4d17e1 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts @@ -666,8 +666,10 @@ describe("Slack live QA runtime helpers", () => { ? [ { channelId: "C123456789", - text: - testCase.commentaryStyle === "lane" ? `💬 ${commentaryMarker}` : commentaryMarker, + text: testCase.commentaryStyle === "lane" ? "Working…" : commentaryMarker, + ...(testCase.commentaryStyle === "lane" + ? { blockText: ["Update", commentaryMarker] } + : {}), ts: testCase.commentaryTs, }, ] @@ -1030,9 +1032,7 @@ describe("Slack live QA runtime helpers", () => { }, }), ); - expect(run && "matchText" in run ? run.matchText : "").toMatch( - /^SLACK_QA_TABLE_DONE_[A-Z0-9]+$/u, - ); + expect(run && "matchText" in run ? run.matchText : "").toBe(summaryText); }); it("verifies the SUT-owned native table and exact accessible top-level text", async () => { @@ -1153,17 +1153,8 @@ describe("Slack live QA runtime helpers", () => { it("proves the public Slack send path stores one complete formatting-disabled fallback", async () => { const probe = testing.buildSlackInvalidBlocksTableProbe(); - const invalidBlocksError = Object.assign(new Error("An API error occurred: invalid_blocks"), { - code: "slack_webapi_platform_error", - data: { error: "invalid_blocks", ok: false }, - }); - let apiAttempt = 0; let storedPayload: Record | undefined; const postMessage = vi.fn(async (payload: Record) => { - apiAttempt += 1; - if (apiAttempt === 1) { - throw invalidBlocksError; - } storedPayload = payload; return { channel: "C123456789", ok: true, ts: "2.000000" }; }); @@ -1201,18 +1192,11 @@ describe("Slack live QA runtime helpers", () => { timeoutMs: 0, }); - expect(postMessage).toHaveBeenCalledTimes(2); - const [nativeRequest] = postMessage.mock.calls[0] ?? []; - const [fallbackRequest] = postMessage.mock.calls[1] ?? []; - const nativeBlocks = nativeRequest?.blocks as Array<{ rows?: unknown[]; type?: string }>; - expect(nativeRequest).toMatchObject({ mrkdwn: false }); - expect(nativeBlocks).toHaveLength(1); - expect(nativeBlocks[0]).toMatchObject({ type: "data_table" }); - expect(nativeBlocks[0]?.rows).toHaveLength(102); + expect(postMessage).toHaveBeenCalledTimes(1); + const [fallbackRequest] = postMessage.mock.calls[0] ?? []; expect(fallbackRequest).not.toHaveProperty("blocks"); expect(fallbackRequest).toMatchObject({ mrkdwn: false }); const fallbackText = typeof fallbackRequest?.text === "string" ? fallbackRequest.text : ""; - expect(fallbackText).toBe(nativeRequest?.text); expect(fallbackText.split("\n")).toContain(probe.firstRowText); expect(fallbackText.split("\n")).toContain(probe.finalRowText); expect(result.message).toMatchObject({ @@ -1226,7 +1210,7 @@ describe("Slack live QA runtime helpers", () => { expect(sutWriteClient.chat.postMessage).toBe(postMessage); }); - it("fails with sanitized evidence when Slack returns a different first API code", async () => { + it("reports the real Slack error when the fallback request fails", async () => { const postMessage = vi.fn(async () => { throw Object.assign(new Error("do not persist this raw platform detail"), { data: { error: "invalid_arguments", ok: false }, @@ -1254,13 +1238,11 @@ describe("Slack live QA runtime helpers", () => { sutWriteClient: sutWriteClient as never, timeoutMs: 0, }), - ).rejects.toThrow( - "expected first Slack API failure code invalid_blocks; observed invalid_arguments", - ); + ).rejects.toThrow("Slack fallback failed after invalid_blocks; observed invalid_arguments"); expect(sutWriteClient.chat.postMessage).toBe(postMessage); }); - it("does not expose an untrusted Slack API error value", async () => { + it("does not expose an untrusted Slack fallback error value", async () => { const postMessage = vi.fn(async () => { throw Object.assign(new Error("private platform detail"), { data: { error: "unsafe private detail", ok: false }, @@ -1288,7 +1270,9 @@ describe("Slack live QA runtime helpers", () => { sutWriteClient: sutWriteClient as never, timeoutMs: 0, }), - ).rejects.toThrow("expected first Slack API failure code invalid_blocks; observed none"); + ).rejects.toThrow( + "Slack fallback failed after invalid_blocks; observed no fallback API failure code", + ); expect(sutWriteClient.chat.postMessage).toBe(postMessage); }); diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-fixtures.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-fixtures.ts index 3dc93b5b9deb..c4286ad8b281 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-fixtures.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-fixtures.ts @@ -77,6 +77,24 @@ type SlackProgressCommentaryExpectation = { toolProgress: "absent" | "draft" | "standalone"; }; +function observedSlackText(message: { blockText?: string[]; text: string }) { + return [message.text, ...(message.blockText ?? [])].join("\n"); +} + +function hasSlackCommentaryLaneMarker( + message: { blockText?: string[]; text: string }, + marker: string, +) { + if (message.text.includes(`💬 ${marker}`)) { + return true; + } + const blockText = message.blockText ?? []; + return ( + blockText.some((text) => text.trim() === "Update") && + blockText.some((text) => text.includes(marker)) + ); +} + export function buildSlackProgressCommentaryRun( sutUserId: string, expectation: SlackProgressCommentaryExpectation, @@ -104,9 +122,11 @@ export function buildSlackProgressCommentaryRun( if ((finalMessage.text ?? "").trim() !== finalMarker) { throw new Error("expected the Slack final answer to contain only the final marker"); } - const progressMessages = messages.filter((message) => !message.text.includes(finalMarker)); + const progressMessages = messages.filter( + (message) => !observedSlackText(message).includes(finalMarker), + ); const commentaryMessages = progressMessages.filter((message) => - message.text.includes(commentaryMarker), + observedSlackText(message).includes(commentaryMarker), ); const commentaryTimestamps = new Set(commentaryMessages.map((message) => message.ts)); const [commentaryTs] = commentaryTimestamps; @@ -120,7 +140,7 @@ export function buildSlackProgressCommentaryRun( } const commentaryLaneTimestamps = new Set( commentaryMessages - .filter((message) => message.text.includes(`💬 ${commentaryMarker}`)) + .filter((message) => hasSlackCommentaryLaneMarker(message, commentaryMarker)) .map((message) => message.ts), ); if ( @@ -138,7 +158,7 @@ export function buildSlackProgressCommentaryRun( } const toolTimestamps = new Set( progressMessages - .filter((message) => message.text.includes(toolMarker)) + .filter((message) => observedSlackText(message).includes(toolMarker)) .map((message) => message.ts), ); if (expectation.toolProgress === "draft") { diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-implementations.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-implementations.ts index f41588978b1b..f22909a7e789 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-implementations.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-implementations.ts @@ -346,16 +346,14 @@ export const slackQaTablePresentationNativeScenario: SlackQaScenarioImplementati buildRun: (sutUserId) => { const suffix = randomUUID().slice(0, 8).toUpperCase(); const summaryText = `SLACK_QA_TABLE_SUMMARY_${suffix}`; - const finalMarker = `SLACK_QA_TABLE_DONE_${suffix}`; const messageToolArgs = buildSlackTableMessageToolArgs(summaryText); return { expectReply: true, input: [ `<@${sutUserId}> Slack native table QA check ${summaryText}.`, `Call the message tool exactly once with these exact arguments: ${JSON.stringify(messageToolArgs)}.`, - `After the table send succeeds, reply with only this exact marker: ${finalMarker}`, ].join(" "), - matchText: finalMarker, + matchText: summaryText, afterReply: async (_message, context) => { await waitForSlackStoredMessage({ channelId: context.channelId, diff --git a/qa/scenarios/channels/slack-table-invalid-blocks-fallback.yaml b/qa/scenarios/channels/slack-table-invalid-blocks-fallback.yaml index 605e0d8adeb6..03b8c8bca03c 100644 --- a/qa/scenarios/channels/slack-table-invalid-blocks-fallback.yaml +++ b/qa/scenarios/channels/slack-table-invalid-blocks-fallback.yaml @@ -1,4 +1,4 @@ -title: Slack rejects an over-limit native table and stores its complete fallback +title: Slack invalid_blocks recovery stores its complete fallback scenario: id: slack-table-invalid-blocks-fallback surface: channels