fix(qa): exercise blocked Slack lifecycle on selected account (#117805)

* fix(qa): exercise blocked Slack lifecycle on selected account

* test(qa): split Slack lifecycle catalog coverage

* qa(slack): wait for blocked lifecycle observation
This commit is contained in:
Dallin Romney
2026-08-02 18:22:44 +08:00
committed by GitHub
parent 45b74bf312
commit edf434b1a3
4 changed files with 72 additions and 25 deletions
@@ -0,0 +1,26 @@
import { describe, expect, it } from "vitest";
import { readQaScenarioById } from "./scenario-catalog.js";
import { flowContainsCall, requireFlowScenario } from "./scenario-catalog.test-utils.js";
describe("Slack blocked lifecycle scenario", () => {
it("invalidates and observes the selected account without waiting for readiness", () => {
const scenario = requireFlowScenario(readQaScenarioById("slack-blocked-lifecycle-no-restart"));
expect(scenario.gatewayConfigPatch).toMatchObject({
channels: {
slack: {
accounts: {
$selectedAccount: { botToken: "xoxb-intentionally-invalid-lifecycle" },
},
},
},
});
const flow = JSON.stringify(scenario.execution.flow);
expect(flow).toContain("transport.accountId");
expect(flow).toContain("await env.gateway.call('channels.status'");
expect(flow).toContain("account?.lifecycle === 'blocked'");
expect(flow).not.toContain("account.accountId === 'default'");
expect(flowContainsCall(scenario.execution.flow, "waitForCondition")).toBe(true);
expect(flowContainsCall(scenario.execution.flow, "waitForTransportReady")).toBe(false);
});
});
@@ -42,6 +42,8 @@ const mocks = vi.hoisted(() => ({
reportPath: "/qa-output/qa-suite-report.md",
summaryPath: "/qa-output/qa-suite-summary.json",
})),
waitForGatewayHealthy: vi.fn(async () => {}),
waitForTransportReady: vi.fn(async () => {}),
}));
vi.mock("openclaw/plugin-sdk/agent-harness", () => ({
@@ -60,8 +62,8 @@ vi.mock("./suite-artifacts.js", () => ({
writeQaSuiteArtifacts: mocks.writeQaSuiteArtifacts,
}));
vi.mock("./suite-runtime-gateway.js", () => ({
waitForGatewayHealthy: vi.fn(async () => {}),
waitForTransportReady: vi.fn(async () => {}),
waitForGatewayHealthy: mocks.waitForGatewayHealthy,
waitForTransportReady: mocks.waitForTransportReady,
}));
vi.mock("./suite.js", () => ({
buildQaSuiteRuntimeMetrics: vi.fn(() => ({ wallMs: 1 })),
@@ -129,6 +131,23 @@ beforeEach(() => {
});
describe("QA runtime parity scenario retry isolation", () => {
it("skips connected-transport readiness for intentionally unhealthy startup", async () => {
const context = makeRetryTestContext();
context.gatewayRuntimeOptions = { allowUnhealthyStartup: true };
const runScenario = vi
.fn<QaSuiteScenarioRunner>()
.mockResolvedValue(makeRetryTestResult("pass"));
await runQaFlowSuiteStandard({ lab: makeRetryTestLab() }, context, runScenario);
expect(mocks.startQaGatewayChild).toHaveBeenCalledWith(
expect.objectContaining({ allowUnhealthyStartup: true }),
);
expect(mocks.waitForGatewayHealthy).not.toHaveBeenCalled();
expect(mocks.waitForTransportReady).not.toHaveBeenCalled();
expect(runScenario).toHaveBeenCalledOnce();
});
it("captures one failed parity attempt without replaying its transcript or usage", async () => {
const runScenario = vi
.fn<QaSuiteScenarioRunner>()
+14 -10
View File
@@ -165,16 +165,20 @@ export async function runQaFlowSuiteStandard(
};
env = activeEnv;
const transportReadyTimeoutMs = resolveQaSuiteTransportReadyTimeoutMs(
params?.transportReadyTimeoutMs,
);
// The gateway child already waits for /readyz before returning, but the
// selected transport can still be finishing account startup. Pay that
// readiness cost once here so the first scenario does not race bootstrap.
await waitForTransportReady(activeEnv, transportReadyTimeoutMs).catch(async () => {
await waitForGatewayHealthy(activeEnv, transportReadyTimeoutMs);
await waitForTransportReady(activeEnv, transportReadyTimeoutMs);
});
// Lifecycle scenarios deliberately start a blocked channel. Waiting for
// connected-channel readiness here would prevent those scenarios from running.
if (!gatewayRuntimeOptions?.allowUnhealthyStartup) {
const transportReadyTimeoutMs = resolveQaSuiteTransportReadyTimeoutMs(
params?.transportReadyTimeoutMs,
);
// The gateway child already waits for /readyz before returning, but the
// selected transport can still be finishing account startup. Pay that
// readiness cost once here so the first scenario does not race bootstrap.
await waitForTransportReady(activeEnv, transportReadyTimeoutMs).catch(async () => {
await waitForGatewayHealthy(activeEnv, transportReadyTimeoutMs);
await waitForTransportReady(activeEnv, transportReadyTimeoutMs);
});
}
const scenarios: QaSuiteScenarioResult[] = [];
let runtimeParityCellTiming: QaRuntimeParityCellTiming | undefined;
const liveScenarioOutcomes: QaLabScenarioOutcome[] = selectedScenarios.map((scenario) => ({
@@ -12,7 +12,9 @@ scenario:
gatewayConfigPatch:
channels:
slack:
botToken: xoxb-intentionally-invalid-lifecycle
accounts:
$selectedAccount:
botToken: xoxb-intentionally-invalid-lifecycle
healthMonitor:
enabled: true
gatewayRuntime:
@@ -38,18 +40,14 @@ flow:
steps:
- name: reports blocked without restarting
actions:
- call: waitForTransportReady
args: [{ ref: env }, 60000]
- call: env.gateway.call
saveAs: initialStatus
- call: waitForCondition
saveAs: initialAccount
args:
- channels.status
- probe: false
timeoutMs: 10000
- timeoutMs: 15000
- set: initialAccount
value:
expr: "(initialStatus.channelAccounts?.slack ?? []).find((account) => account.accountId === 'default')"
- lambda:
async: true
expr: "((status) => { const account = (status.channelAccounts?.slack ?? []).find((candidate) => candidate.accountId === transport.accountId); return account?.lifecycle === 'blocked' && account?.healthState === 'blocked' ? account : undefined; })(await env.gateway.call('channels.status', { probe: false, timeoutMs: 10000 }, { timeoutMs: 15000 }))"
- 30000
- 250
- assert:
expr: "initialAccount?.lifecycle === 'blocked' && initialAccount?.healthState === 'blocked' && initialAccount?.running === true && initialAccount?.restartPending !== true && typeof initialAccount?.lastStartAt === 'number'"
message:
@@ -66,7 +64,7 @@ flow:
- timeoutMs: 15000
- set: finalAccount
value:
expr: "(finalStatus.channelAccounts?.slack ?? []).find((account) => account.accountId === 'default')"
expr: "(finalStatus.channelAccounts?.slack ?? []).find((account) => account.accountId === transport.accountId)"
- assert:
expr: "finalAccount?.lifecycle === 'blocked' && finalAccount?.healthState === 'blocked' && finalAccount?.running === true && finalAccount?.restartPending !== true && finalAccount?.lastStartAt === initialAccount.lastStartAt"
message: