diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index 204c509cb0c4..1c41a686293c 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -1281,6 +1281,18 @@ jobs: pnpm openclaw qa suite "${runtime_pair_args[@]}" + if [[ "$RUNTIME_PAIR_LANE" == "core" ]]; then + # Restart recovery owns an OpenClaw Gateway lifecycle; it is release-critical + # but cannot be compared against the Codex app-server runtime. + pnpm openclaw qa suite \ + --provider-mode mock-openai \ + --concurrency "${QA_PARITY_CONCURRENCY}" \ + --model "${OPENCLAW_CI_OPENAI_MODEL}" \ + --alt-model "openai/gpt-5.6-luna-alt" \ + --scenario gateway-restart-inflight-run \ + --output-dir ".artifacts/qa-e2e/openclaw-core-restart" + fi + - name: Generate runtime-pair lane report id: generate_runtime_parity_report if: always() diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index 8a3b6ae225bd..6db70c841417 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -1739,7 +1739,6 @@ describe("qa cli runtime", () => { "channel-chat-baseline", "runtime-tool-bash", "approval-turn-tool-followthrough", - "gateway-restart-inflight-run", "runtime-first-hour-20-turn", "runtime-tool-apply-patch", "source-docs-discovery-report", @@ -1793,10 +1792,6 @@ describe("qa cli runtime", () => { stderrWrite, "excluded incompatible non-flow scenario(s): codex-plugin-cold-install (script)", ); - expectWriteContains( - stderrWrite, - "excluded lane-incompatible scenario(s): gateway-restart-inflight-run", - ); }); it("rejects explicit runtime-pair scenarios with no compatible flow execution", async () => { diff --git a/extensions/qa-lab/src/scenario-catalog-channels.test.ts b/extensions/qa-lab/src/scenario-catalog-channels.test.ts index 7dc3115f36bd..e818c2eaad20 100644 --- a/extensions/qa-lab/src/scenario-catalog-channels.test.ts +++ b/extensions/qa-lab/src/scenario-catalog-channels.test.ts @@ -67,6 +67,8 @@ describe("qa scenario catalog channel contracts", () => { ); const storeReads = [...flow.matchAll(/readRawQaSessionStore/gu)].map((match) => match.index); + expect(flow).toContain("readSessionTranscriptSummary(env, sessionKey)"); + expect(flow).not.toContain("waitForAgentHistoryReply"); expect(completionWaits).toHaveLength(2); expect(storeReads).toHaveLength(2); expect(completionWaits.every((wait, index) => wait < (storeReads[index] ?? -1))).toBe(true); diff --git a/extensions/qa-lab/src/scenario-catalog.runtime-pair.test.ts b/extensions/qa-lab/src/scenario-catalog.runtime-pair.test.ts index e636bfd41b87..2709939b6b98 100644 --- a/extensions/qa-lab/src/scenario-catalog.runtime-pair.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.runtime-pair.test.ts @@ -19,7 +19,7 @@ describe("QA runtime-pair scenario catalog", () => { .length, ]), ); - expect(laneCounts).toEqual({ core: 39, extended: 8, soak: 2 }); + expect(laneCounts).toEqual({ core: 38, extended: 8, soak: 2 }); }); it("declares every release agentic scenario in the core lane", () => { @@ -51,7 +51,12 @@ describe("QA runtime-pair scenario catalog", () => { }); } - expect(readQaScenarioById("gateway-restart-inflight-run").runtimePairLane).toBe("core"); + expect(readQaScenarioById("gateway-restart-inflight-run").runtimePairLane).toBeUndefined(); + expect(readQaScenarioById("gateway-restart-inflight-run").execution).toMatchObject({ + kind: "flow", + runtime: "openclaw", + timeoutMs: 420_000, + }); expect(readQaScenarioExecutionConfig("gateway-restart-inflight-run")).toMatchObject({ requiredProviderMode: "mock-openai", }); diff --git a/extensions/qa-lab/src/scenario-catalog.test.ts b/extensions/qa-lab/src/scenario-catalog.test.ts index ca95a5e78b5a..045c6066087e 100644 --- a/extensions/qa-lab/src/scenario-catalog.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.test.ts @@ -467,14 +467,26 @@ describe("qa scenario catalog", () => { ).toContain('"alsoAllow":["qa_restart_wait","qa_restart_unsafe_probe"]'); expect(gatewayRestartContract).toContain("plannedToolName === 'wait'"); expect(gatewayRestartContract).toContain("lastAssistantToolNames?.includes('wait')"); - expect(gatewayRestartContract).toContain('"taskTracking":false'); + expect(gatewayRestartContract).toContain("restartRecoveryDeliveryContext"); + expect(gatewayRestartContract).toContain("sendInbound"); + expect(gatewayRestartContract).not.toContain("startAgentRun"); expect(gatewayRestartContract).toContain('"restartGatewayWithConfigPatch"'); expect(gatewayRestartContract).toContain("interruptedMatches.length === 1"); expect(gatewayRestartContract).toContain("restartNotices.length === 0"); expect(gatewayRestartContract).toContain("dispatching restart-safe recovery"); expect(gatewayRestartContract).toContain("[OpenClaw heartbeat poll]"); expect(gatewayRestartContract).toContain("liveTurnTimeoutMs(env, 180000)"); - expect(gatewayRestartContract).toContain("dmScope: 'per-channel-peer'"); + expect(gatewayRestartContract).toContain("id: `dm:${conversationId}`"); + expect(gatewayRestartContract).toContain("dmScope: env.cfg.session?.dmScope"); + expect(readQaScenarioById("gateway-restart-inflight-run").gatewayConfigPatch).toMatchObject({ + plugins: { + slots: { memory: "none" }, + entries: { + acpx: { enabled: false }, + "memory-core": { enabled: false }, + }, + }, + }); const liveMultiRestart = readQaScenarioById("gateway-restart-multi-live"); const liveMultiRestartContract = JSON.stringify(liveMultiRestart.execution.flow); expect(JSON.stringify(liveMultiRestart.gatewayConfigPatch)).toContain( diff --git a/qa/scenarios/agents/subagent-fanout-synthesis.yaml b/qa/scenarios/agents/subagent-fanout-synthesis.yaml index a80047eb92cb..b8952cf18c3a 100644 --- a/qa/scenarios/agents/subagent-fanout-synthesis.yaml +++ b/qa/scenarios/agents/subagent-fanout-synthesis.yaml @@ -102,14 +102,12 @@ flow: ref: prompt timeoutMs: expr: liveTurnTimeoutMs(env, 90000) - - call: waitForAgentHistoryReply - saveAs: outbound + - call: waitForCondition + saveAs: parentTranscript args: - - ref: env - - ref: sessionKey - lambda: - params: [text] - expr: "config.expectedReplyGroups.every((group) => group.some((needle) => normalizeLowercaseStringOrEmpty(text).includes(needle)))" + async: true + expr: "readSessionTranscriptSummary(env, sessionKey).then((summary) => config.expectedReplyGroups.every((group) => group.some((needle) => normalizeLowercaseStringOrEmpty(summary.finalText).includes(needle))) ? summary : undefined).catch(() => undefined)" - expr: "30000" - expr: "env.providerMode === 'mock-openai' ? 100 : 250" - if: @@ -173,7 +171,7 @@ flow: expr: "`expected at least two sessions_spawn tool calls during subagent fanout scenario, saw ${fanoutSpawnRequests.length}`" - set: details value: - expr: "outbound.text" + expr: "parentTranscript.finalText" - set: lastError value: __done__ catchAs: attemptError diff --git a/qa/scenarios/runtime/gateway-restart-inflight-run.yaml b/qa/scenarios/runtime/gateway-restart-inflight-run.yaml index 9e6d13ba2cc9..2d43fa80938e 100644 --- a/qa/scenarios/runtime/gateway-restart-inflight-run.yaml +++ b/qa/scenarios/runtime/gateway-restart-inflight-run.yaml @@ -3,7 +3,6 @@ title: Gateway restart Code Mode wait recovery scenario: id: gateway-restart-inflight-run surface: runtime - runtimePairLane: core coverage: primary: - session-memory.recovery-restart-recovery @@ -11,6 +10,16 @@ scenario: - gateway.restart-and-stop-gateway-restart - session-memory.recovery-delivery gatewayConfigPatch: + # Channel restart recovery deliberately fails closed when reply hooks are active. + # This scenario isolates the replay-safe Code Mode contract from those hooks. + plugins: + slots: + memory: none + entries: + acpx: + enabled: false + memory-core: + enabled: false tools: alsoAllow: - qa_restart_wait @@ -33,6 +42,8 @@ scenario: - src/gateway/server-restart-sentinel.ts execution: kind: flow + runtime: openclaw + timeoutMs: 420000 summary: Restart while replay-safe Code Mode wait is executing, then verify automatic reconstruction and delivery. config: requiredProviderMode: mock-openai @@ -64,26 +75,26 @@ flow: expr: "`restart-inflight-${randomUUID().slice(0, 8)}`" - set: sessionKey value: - expr: "buildAgentSessionKey({ agentId: 'qa', channel: 'qa-channel', accountId: transport.accountId, peer: { kind: 'direct', id: conversationId }, dmScope: 'per-channel-peer', identityLinks: env.cfg.session?.identityLinks })" + expr: "buildAgentSessionKey({ agentId: 'qa', channel: 'qa-channel', accountId: transport.accountId, peer: { kind: 'direct', id: `dm:${conversationId}` }, dmScope: env.cfg.session?.dmScope, identityLinks: env.cfg.session?.identityLinks })" - set: restartPatch value: expr: "({ gateway: { controlUi: { allowedOrigins: [`http://127.0.0.1:${64000 + Math.floor(Math.random() * 999)}`] } }, tools: { codeMode: { enabled: false } } })" - set: requestCursorBefore value: expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - - call: startAgentRun - saveAs: started - args: - - ref: env - - sessionKey: - ref: sessionKey - message: - expr: config.prompt - to: - expr: "`dm:${conversationId}`" - taskTracking: false - timeoutMs: - expr: liveTurnTimeoutMs(env, 180000) + # Real channel ingress owns the durable delivery context needed by recovery. + # Synthetic chat/agent RPC starts cannot prove recovered channel delivery. + - sendInbound: + accountId: default + conversation: + id: + ref: conversationId + kind: direct + senderId: + ref: conversationId + senderName: QA Restart Operator + text: + expr: config.prompt - call: waitForCondition saveAs: plannedWait args: @@ -100,6 +111,17 @@ flow: expr: "readSessionTranscriptSummary(env, sessionKey).then((summary) => summary.lastMessageRole === 'assistant' && summary.lastAssistantToolNames?.includes('wait') ? summary : undefined)" - expr: liveTurnTimeoutMs(env, 120000) - 25 + - call: readRawQaSessionStore + saveAs: preRestartStore + args: + - ref: env + - set: preRestartEntry + value: + expr: preRestartStore[sessionKey] + - assert: + expr: "preRestartEntry?.restartRecoveryDeliveryContext?.channel === 'qa-channel' && preRestartEntry.restartRecoveryDeliveryContext.to === `dm:${conversationId}`" + message: + expr: "`restart recovery delivery claim missing before restart: ${JSON.stringify(preRestartEntry ?? null)}`" - call: restartGatewayWithConfigPatch args: - env: @@ -166,4 +188,4 @@ flow: expr: "recoveryLogs.includes('dispatching restart-safe recovery') && recoveryLogs.includes('restart-safe recovery tool policy retained')" message: expr: "`restart-safe host policy was not observed; logs=${recoveryLogs}`" - detailsExpr: "`runId=${started.runId} session=${sessionKey} plannedTool=${plannedWait.plannedToolName} persistedTail=${interruptedTranscript.lastMessageRole}/${interruptedTranscript.lastAssistantStopReason}/${interruptedTranscript.lastAssistantToolNames?.join(',')} recoveredTail=${postRestartTranscript.lastMessageRole}/${postRestartTranscript.lastAssistantStopReason}/${postRestartTranscript.lastAssistantContentTypes?.join(',')} recoveredMarkers=${interruptedMatches.length} resendNotices=${restartNotices.length} restartSafePolicy=true heartbeatPoll=false\\n${outbound.text}`" + detailsExpr: "`session=${sessionKey} plannedTool=${plannedWait.plannedToolName} persistedTail=${interruptedTranscript.lastMessageRole}/${interruptedTranscript.lastAssistantStopReason}/${interruptedTranscript.lastAssistantToolNames?.join(',')} recoveredTail=${postRestartTranscript.lastMessageRole}/${postRestartTranscript.lastAssistantStopReason}/${postRestartTranscript.lastAssistantContentTypes?.join(',')} recoveredMarkers=${interruptedMatches.length} resendNotices=${restartNotices.length} restartSafePolicy=true heartbeatPoll=false\\n${outbound.text}`" diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 2ece73918e97..8bf4e7d0b3f2 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -2771,6 +2771,8 @@ describe("package artifact reuse", () => { expect(runtimePairRun).toContain("--runtime-parity-tier standard,live-only"); expect(runtimePairRun).toContain("--runtime-parity-tier soak"); expect(runtimePairRun).toContain("Frozen candidate cannot select runtime-pair lane"); + expect(runtimePairRun).toContain("--scenario gateway-restart-inflight-run"); + expect(runtimePairRun).toContain('--output-dir ".artifacts/qa-e2e/openclaw-core-restart"'); expect(workflowStep(laneJob, "Upload runtime-pair lane artifacts").with?.name).toContain( "${{ matrix.lane }}", );