From edf434b1a301b166d4954e057a367b8449efce46 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Sun, 2 Aug 2026 18:22:44 +0800 Subject: [PATCH] fix(qa): exercise blocked Slack lifecycle on selected account (#117805) * fix(qa): exercise blocked Slack lifecycle on selected account * test(qa): split Slack lifecycle catalog coverage * qa(slack): wait for blocked lifecycle observation --- ...io-catalog.slack-blocked-lifecycle.test.ts | 26 +++++++++++++++++++ .../suite-run-standard.parity-retry.test.ts | 23 ++++++++++++++-- extensions/qa-lab/src/suite-run-standard.ts | 24 ++++++++++------- .../slack-blocked-lifecycle-no-restart.yaml | 24 ++++++++--------- 4 files changed, 72 insertions(+), 25 deletions(-) create mode 100644 extensions/qa-lab/src/scenario-catalog.slack-blocked-lifecycle.test.ts diff --git a/extensions/qa-lab/src/scenario-catalog.slack-blocked-lifecycle.test.ts b/extensions/qa-lab/src/scenario-catalog.slack-blocked-lifecycle.test.ts new file mode 100644 index 000000000000..9f307b61607f --- /dev/null +++ b/extensions/qa-lab/src/scenario-catalog.slack-blocked-lifecycle.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { readQaScenarioById } from "./scenario-catalog.js"; +import { flowContainsCall, requireFlowScenario } from "./scenario-catalog.test-utils.js"; + +describe("Slack blocked lifecycle scenario", () => { + it("invalidates and observes the selected account without waiting for readiness", () => { + const scenario = requireFlowScenario(readQaScenarioById("slack-blocked-lifecycle-no-restart")); + + expect(scenario.gatewayConfigPatch).toMatchObject({ + channels: { + slack: { + accounts: { + $selectedAccount: { botToken: "xoxb-intentionally-invalid-lifecycle" }, + }, + }, + }, + }); + const flow = JSON.stringify(scenario.execution.flow); + expect(flow).toContain("transport.accountId"); + expect(flow).toContain("await env.gateway.call('channels.status'"); + expect(flow).toContain("account?.lifecycle === 'blocked'"); + expect(flow).not.toContain("account.accountId === 'default'"); + expect(flowContainsCall(scenario.execution.flow, "waitForCondition")).toBe(true); + expect(flowContainsCall(scenario.execution.flow, "waitForTransportReady")).toBe(false); + }); +}); diff --git a/extensions/qa-lab/src/suite-run-standard.parity-retry.test.ts b/extensions/qa-lab/src/suite-run-standard.parity-retry.test.ts index 4ad19587768a..c96408dee5a3 100644 --- a/extensions/qa-lab/src/suite-run-standard.parity-retry.test.ts +++ b/extensions/qa-lab/src/suite-run-standard.parity-retry.test.ts @@ -42,6 +42,8 @@ const mocks = vi.hoisted(() => ({ reportPath: "/qa-output/qa-suite-report.md", summaryPath: "/qa-output/qa-suite-summary.json", })), + waitForGatewayHealthy: vi.fn(async () => {}), + waitForTransportReady: vi.fn(async () => {}), })); vi.mock("openclaw/plugin-sdk/agent-harness", () => ({ @@ -60,8 +62,8 @@ vi.mock("./suite-artifacts.js", () => ({ writeQaSuiteArtifacts: mocks.writeQaSuiteArtifacts, })); vi.mock("./suite-runtime-gateway.js", () => ({ - waitForGatewayHealthy: vi.fn(async () => {}), - waitForTransportReady: vi.fn(async () => {}), + waitForGatewayHealthy: mocks.waitForGatewayHealthy, + waitForTransportReady: mocks.waitForTransportReady, })); vi.mock("./suite.js", () => ({ buildQaSuiteRuntimeMetrics: vi.fn(() => ({ wallMs: 1 })), @@ -129,6 +131,23 @@ beforeEach(() => { }); describe("QA runtime parity scenario retry isolation", () => { + it("skips connected-transport readiness for intentionally unhealthy startup", async () => { + const context = makeRetryTestContext(); + context.gatewayRuntimeOptions = { allowUnhealthyStartup: true }; + const runScenario = vi + .fn() + .mockResolvedValue(makeRetryTestResult("pass")); + + await runQaFlowSuiteStandard({ lab: makeRetryTestLab() }, context, runScenario); + + expect(mocks.startQaGatewayChild).toHaveBeenCalledWith( + expect.objectContaining({ allowUnhealthyStartup: true }), + ); + expect(mocks.waitForGatewayHealthy).not.toHaveBeenCalled(); + expect(mocks.waitForTransportReady).not.toHaveBeenCalled(); + expect(runScenario).toHaveBeenCalledOnce(); + }); + it("captures one failed parity attempt without replaying its transcript or usage", async () => { const runScenario = vi .fn() diff --git a/extensions/qa-lab/src/suite-run-standard.ts b/extensions/qa-lab/src/suite-run-standard.ts index 41812c5cc78c..e1afad5595e5 100644 --- a/extensions/qa-lab/src/suite-run-standard.ts +++ b/extensions/qa-lab/src/suite-run-standard.ts @@ -165,16 +165,20 @@ export async function runQaFlowSuiteStandard( }; env = activeEnv; - const transportReadyTimeoutMs = resolveQaSuiteTransportReadyTimeoutMs( - params?.transportReadyTimeoutMs, - ); - // The gateway child already waits for /readyz before returning, but the - // selected transport can still be finishing account startup. Pay that - // readiness cost once here so the first scenario does not race bootstrap. - await waitForTransportReady(activeEnv, transportReadyTimeoutMs).catch(async () => { - await waitForGatewayHealthy(activeEnv, transportReadyTimeoutMs); - await waitForTransportReady(activeEnv, transportReadyTimeoutMs); - }); + // Lifecycle scenarios deliberately start a blocked channel. Waiting for + // connected-channel readiness here would prevent those scenarios from running. + if (!gatewayRuntimeOptions?.allowUnhealthyStartup) { + const transportReadyTimeoutMs = resolveQaSuiteTransportReadyTimeoutMs( + params?.transportReadyTimeoutMs, + ); + // The gateway child already waits for /readyz before returning, but the + // selected transport can still be finishing account startup. Pay that + // readiness cost once here so the first scenario does not race bootstrap. + await waitForTransportReady(activeEnv, transportReadyTimeoutMs).catch(async () => { + await waitForGatewayHealthy(activeEnv, transportReadyTimeoutMs); + await waitForTransportReady(activeEnv, transportReadyTimeoutMs); + }); + } const scenarios: QaSuiteScenarioResult[] = []; let runtimeParityCellTiming: QaRuntimeParityCellTiming | undefined; const liveScenarioOutcomes: QaLabScenarioOutcome[] = selectedScenarios.map((scenario) => ({ diff --git a/qa/scenarios/channels/slack-blocked-lifecycle-no-restart.yaml b/qa/scenarios/channels/slack-blocked-lifecycle-no-restart.yaml index c1a06a442529..4826e5414744 100644 --- a/qa/scenarios/channels/slack-blocked-lifecycle-no-restart.yaml +++ b/qa/scenarios/channels/slack-blocked-lifecycle-no-restart.yaml @@ -12,7 +12,9 @@ scenario: gatewayConfigPatch: channels: slack: - botToken: xoxb-intentionally-invalid-lifecycle + accounts: + $selectedAccount: + botToken: xoxb-intentionally-invalid-lifecycle healthMonitor: enabled: true gatewayRuntime: @@ -38,18 +40,14 @@ flow: steps: - name: reports blocked without restarting actions: - - call: waitForTransportReady - args: [{ ref: env }, 60000] - - call: env.gateway.call - saveAs: initialStatus + - call: waitForCondition + saveAs: initialAccount args: - - channels.status - - probe: false - timeoutMs: 10000 - - timeoutMs: 15000 - - set: initialAccount - value: - expr: "(initialStatus.channelAccounts?.slack ?? []).find((account) => account.accountId === 'default')" + - lambda: + async: true + expr: "((status) => { const account = (status.channelAccounts?.slack ?? []).find((candidate) => candidate.accountId === transport.accountId); return account?.lifecycle === 'blocked' && account?.healthState === 'blocked' ? account : undefined; })(await env.gateway.call('channels.status', { probe: false, timeoutMs: 10000 }, { timeoutMs: 15000 }))" + - 30000 + - 250 - assert: expr: "initialAccount?.lifecycle === 'blocked' && initialAccount?.healthState === 'blocked' && initialAccount?.running === true && initialAccount?.restartPending !== true && typeof initialAccount?.lastStartAt === 'number'" message: @@ -66,7 +64,7 @@ flow: - timeoutMs: 15000 - set: finalAccount value: - expr: "(finalStatus.channelAccounts?.slack ?? []).find((account) => account.accountId === 'default')" + expr: "(finalStatus.channelAccounts?.slack ?? []).find((account) => account.accountId === transport.accountId)" - assert: expr: "finalAccount?.lifecycle === 'blocked' && finalAccount?.healthState === 'blocked' && finalAccount?.running === true && finalAccount?.restartPending !== true && finalAccount?.lastStartAt === initialAccount.lastStartAt" message: