diff --git a/extensions/qa-lab/src/scenario-catalog.config-restart.test.ts b/extensions/qa-lab/src/scenario-catalog.config-restart.test.ts index 38c85d62fad1..6b06dea00827 100644 --- a/extensions/qa-lab/src/scenario-catalog.config-restart.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.config-restart.test.ts @@ -7,28 +7,28 @@ describe("QA config-restart scenario catalog", () => { const flow = JSON.stringify(readQaScenarioById("config-restart-capability-flip")); const wakeStartIndex = flow.indexOf('"set":"wakeStartIndex"'); const nextConfigIndex = flow.indexOf('"set":"nextConfig"'); + const restartRequestedIndex = flow.indexOf('"set":"restartRequestedAtMs"'); const restartApplyIndex = flow.indexOf('"call":"applyConfig"'); const applyResultIndex = flow.indexOf('"saveAs":"applyResult"', restartApplyIndex); const sentinelAssertIndex = flow.indexOf("applyResult.sentinel?.persisted"); const wakeWaitIndex = flow.indexOf("candidate.text.includes(wakeMarker)"); const wakeSinceIndex = flow.indexOf('"sinceIndex":{"ref":"wakeStartIndex"}', wakeWaitIndex); - const settledSessionIndex = flow.indexOf("sessions.list", wakeWaitIndex); - const idleSessionIndex = flow.indexOf("hasActiveRun", settledSessionIndex); + const lifecycleWaitIndex = flow.indexOf('"call":"waitForSessionRunAfter"', wakeWaitIndex); const capabilityPollIndex = flow.indexOf('"saveAs":"afterTools"'); const promptIndex = flow.indexOf('"call":"runAgentPrompt"'); const cleanupApplyIndex = flow.lastIndexOf('"call":"applyConfig"'); expect(wakeStartIndex).toBeGreaterThanOrEqual(0); expect(nextConfigIndex).toBeGreaterThan(wakeStartIndex); - expect(restartApplyIndex).toBeGreaterThan(nextConfigIndex); + expect(restartRequestedIndex).toBeGreaterThan(nextConfigIndex); + expect(restartApplyIndex).toBeGreaterThan(restartRequestedIndex); expect(applyResultIndex).toBeGreaterThan(restartApplyIndex); expect(sentinelAssertIndex).toBeGreaterThan(applyResultIndex); expect(flow).toContain("payload?.stats?.requiresRestart === true"); expect(wakeWaitIndex).toBeGreaterThan(sentinelAssertIndex); expect(wakeSinceIndex).toBeGreaterThan(wakeWaitIndex); - expect(settledSessionIndex).toBeGreaterThan(wakeSinceIndex); - expect(idleSessionIndex).toBeGreaterThan(settledSessionIndex); - expect(capabilityPollIndex).toBeGreaterThan(idleSessionIndex); + expect(lifecycleWaitIndex).toBeGreaterThan(wakeSinceIndex); + expect(capabilityPollIndex).toBeGreaterThan(lifecycleWaitIndex); expect(promptIndex).toBeGreaterThan(capabilityPollIndex); expect(cleanupApplyIndex).toBeGreaterThan(promptIndex); expect(flow.match(/"call":"applyConfig"/g)).toHaveLength(2); diff --git a/extensions/qa-lab/src/scenario-runtime-api.ts b/extensions/qa-lab/src/scenario-runtime-api.ts index 8ec8891d39dd..afb097cdde93 100644 --- a/extensions/qa-lab/src/scenario-runtime-api.ts +++ b/extensions/qa-lab/src/scenario-runtime-api.ts @@ -72,6 +72,7 @@ export type QaScenarioRuntimeDeps = { resolveGeneratedImagePath: QaScenarioRuntimeFunction; startAgentRun: QaScenarioRuntimeFunction; waitForAgentRun: QaScenarioRuntimeFunction; + waitForSessionRunAfter: QaScenarioRuntimeFunction; waitForAgentHistoryReply: QaScenarioRuntimeFunction; listCronJobs: QaScenarioRuntimeFunction; findManagedDreamingCronJob: QaScenarioRuntimeFunction; diff --git a/extensions/qa-lab/src/suite-runtime-agent-process.test.ts b/extensions/qa-lab/src/suite-runtime-agent-process.test.ts index 97e6ca93adf6..04d8b5f80d76 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-process.test.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-process.test.ts @@ -43,6 +43,7 @@ import { startAgentRun, waitForAgentRun, waitForAgentHistoryReply, + waitForSessionRunAfter, } from "./suite-runtime-agent-process.js"; type MockEmitter = { @@ -900,6 +901,195 @@ describe("qa suite runtime agent process helpers", () => { ); }); + it("surfaces agent.wait errors after a correlated active run appears", async () => { + vi.useFakeTimers(); + try { + let listCalls = 0; + const gatewayCall = vi.fn(async (method: string) => { + if (method === "agent.wait") { + return { status: "error", error: "restart wake failed" }; + } + listCalls += 1; + return { + sessions: [ + listCalls === 1 + ? { + key: "agent:qa:capability-flip", + agentId: "qa", + status: "done", + hasActiveRun: false, + startedAt: 50, + endedAt: 60, + } + : { + key: "agent:qa:capability-flip", + agentId: "qa", + status: "running", + hasActiveRun: true, + activeRunIds: ["run-wake"], + startedAt: 110, + }, + ], + }; + }); + + const pending = waitForSessionRunAfter( + { gateway: { call: gatewayCall } } as never, + "agent:qa:capability-flip", + "qa", + 100, + 1_000, + ); + await vi.advanceTimersByTimeAsync(100); + + await expect(pending).rejects.toThrow("agent.wait returned error: restart wake failed"); + } finally { + vi.useRealTimers(); + } + }); + + it("surfaces the redacted row error when an unidentified active run fails", async () => { + vi.useFakeTimers(); + try { + let listCalls = 0; + const gatewayCall = vi.fn(async () => { + listCalls += 1; + const row = + listCalls === 1 + ? { + status: "done", + hasActiveRun: false, + startedAt: 50, + endedAt: 60, + } + : listCalls === 2 + ? { + status: "running", + hasActiveRun: true, + startedAt: 110, + } + : { + status: "failed", + hasActiveRun: false, + startedAt: 110, + endedAt: 150, + lastRunError: "provider rejected restart wake", + }; + return { + sessions: [{ key: "agent:qa:capability-flip", agentId: "qa", ...row }], + }; + }); + + const pending = waitForSessionRunAfter( + { gateway: { call: gatewayCall } } as never, + "agent:qa:capability-flip", + "qa", + 100, + 1_000, + ); + await vi.advanceTimersByTimeAsync(200); + + await expect(pending).rejects.toThrow("session run failed: provider rejected restart wake"); + } finally { + vi.useRealTimers(); + } + }); + + it("accepts a correlated terminal row after missing the active polling window", async () => { + const gatewayCall = vi.fn(async () => ({ + sessions: [ + { + key: "agent:qa:capability-flip", + agentId: "qa", + status: "done", + hasActiveRun: false, + startedAt: 110, + endedAt: 150, + }, + ], + })); + + await expect( + waitForSessionRunAfter( + { gateway: { call: gatewayCall } } as never, + "agent:qa:capability-flip", + "qa", + 100, + 1_000, + ), + ).resolves.toEqual({ status: "done" }); + }); + + it("times out while the session remains permanently idle", async () => { + vi.useFakeTimers(); + try { + const gatewayCall = vi.fn(async () => ({ + sessions: [ + { + key: "agent:qa:capability-flip", + agentId: "qa", + status: "done", + hasActiveRun: false, + startedAt: 50, + endedAt: 60, + }, + ], + })); + + const pending = waitForSessionRunAfter( + { gateway: { call: gatewayCall } } as never, + "agent:qa:capability-flip", + "qa", + 100, + 250, + ); + const errorPromise = pending.catch((error: unknown) => error); + await vi.advanceTimersByTimeAsync(250); + const error = await errorPromise; + + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain("timed out after 250ms"); + expect((error as Error).message).toContain('"hasActiveRun":false'); + } finally { + vi.useRealTimers(); + } + }); + + it("omits unrelated secret-shaped session fields from timeout diagnostics", async () => { + vi.useFakeTimers(); + try { + const gatewayCall = vi.fn(async () => ({ + sessions: [ + { + key: "agent:qa:capability-flip", + agentId: "qa", + status: "running", + hasActiveRun: false, + startedAt: 50, + privateApiToken: "do-not-leak-this-token", + }, + ], + })); + + const pending = waitForSessionRunAfter( + { gateway: { call: gatewayCall } } as never, + "agent:qa:capability-flip", + "qa", + 100, + 100, + ); + const errorPromise = pending.catch((error: unknown) => error); + await vi.advanceTimersByTimeAsync(100); + const error = await errorPromise; + + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).not.toContain("privateApiToken"); + expect((error as Error).message).not.toContain("do-not-leak-this-token"); + } finally { + vi.useRealTimers(); + } + }); + it.each(["restart", "aborted"])( "preserves the %s stop reason from agent.wait", async (stopReason) => { diff --git a/extensions/qa-lab/src/suite-runtime-agent-process.ts b/extensions/qa-lab/src/suite-runtime-agent-process.ts index e751cac08a82..9311dbd7db06 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-process.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-process.ts @@ -48,6 +48,17 @@ type QaAgentWaitResult = { terminalReply?: QaAgentTerminalReply; }; +type QaSessionRunRow = { + key?: string; + agentId?: string; + status?: "queued" | "running" | "done" | "failed" | "killed" | "timeout"; + lastRunError?: string; + hasActiveRun?: boolean; + activeRunIds?: string[]; + startedAt?: number; + endedAt?: number; +}; + const MANAGED_DREAMING_CRON_MARKER = "[managed-by=memory-core.short-term-promotion]"; const MANAGED_DREAMING_CRON_NAME = "Memory Dreaming Promotion"; const MANAGED_DREAMING_PROMPT = "__openclaw_memory_core_short_term_promotion_dream__"; @@ -159,6 +170,123 @@ function isSuccessfulAgentWaitResult(waited: QaAgentWaitResult) { return waited.status === "error" && waited.error?.trim().toLowerCase() === "completed"; } +function assertSuccessfulAgentWaitResult(waited: QaAgentWaitResult) { + if (isSuccessfulAgentWaitResult(waited)) { + return; + } + throw new Error( + `agent.wait returned ${waited.status ?? "unknown"}: ${waited.error ?? "no error"}`, + ); +} + +function isTerminalSessionRunStatus(status: QaSessionRunRow["status"]) { + return status === "done" || status === "failed" || status === "killed" || status === "timeout"; +} + +function parseSessionRunRow(value: unknown): QaSessionRunRow | undefined { + if (!isRecord(value)) { + return undefined; + } + const status = + value.status === "queued" || + value.status === "running" || + value.status === "done" || + value.status === "failed" || + value.status === "killed" || + value.status === "timeout" + ? value.status + : undefined; + const activeRunIds = Array.isArray(value.activeRunIds) + ? value.activeRunIds.filter((runId): runId is string => typeof runId === "string") + : undefined; + return { + key: typeof value.key === "string" ? value.key : undefined, + agentId: typeof value.agentId === "string" ? value.agentId : undefined, + status, + lastRunError: typeof value.lastRunError === "string" ? value.lastRunError : undefined, + hasActiveRun: typeof value.hasActiveRun === "boolean" ? value.hasActiveRun : undefined, + activeRunIds, + startedAt: typeof value.startedAt === "number" ? value.startedAt : undefined, + endedAt: typeof value.endedAt === "number" ? value.endedAt : undefined, + }; +} + +function formatSessionRunDiagnostics(row: QaSessionRunRow | undefined) { + return JSON.stringify({ + status: row?.status ?? null, + startedAt: row?.startedAt ?? null, + endedAt: row?.endedAt ?? null, + hasActiveRun: row?.hasActiveRun ?? null, + lastRunError: row?.lastRunError ?? null, + }); +} + +async function waitForSessionRunAfter( + env: Pick, + sessionKey: string, + agentId: string, + startedAfterMs: number, + timeoutMs = 30_000, +) { + const waitTimeoutMs = resolveTimerTimeoutMs(timeoutMs, 30_000); + const deadlineMs = Date.now() + waitTimeoutMs; + let lastRow: QaSessionRunRow | undefined; + + while (Date.now() < deadlineMs) { + const remainingMs = Math.max(1, deadlineMs - Date.now()); + const result = await env.gateway.call( + "sessions.list", + {}, + { timeoutMs: Math.min(30_000, remainingMs) }, + ); + const sessions = + isRecord(result) && Array.isArray(result.sessions) + ? result.sessions.map(parseSessionRunRow).filter((row) => row !== undefined) + : []; + const row = sessions.find( + (candidate) => candidate.key === sessionKey && candidate.agentId === agentId, + ); + lastRow = row; + + if (row && typeof row.startedAt === "number" && row.startedAt >= startedAfterMs) { + const correlatedStartedAt = row.startedAt; + const activeRunId = + row.hasActiveRun === true && row.activeRunIds?.length === 1 + ? row.activeRunIds[0] + : undefined; + if (activeRunId) { + const waited = await waitForAgentRun(env, activeRunId, remainingMs); + assertSuccessfulAgentWaitResult(waited); + return { runId: activeRunId, status: waited.status }; + } + + if ( + isTerminalSessionRunStatus(row.status) && + typeof row.endedAt === "number" && + row.endedAt >= correlatedStartedAt + ) { + if (row.status === "done") { + return { status: row.status }; + } + throw new Error( + row.lastRunError + ? `session run ${row.status}: ${row.lastRunError}` + : `session run ${row.status}`, + ); + } + } + + const delayMs = Math.min(100, deadlineMs - Date.now()); + if (delayMs > 0) { + await sleep(delayMs); + } + } + + throw new Error( + `timed out after ${waitTimeoutMs}ms waiting for correlated session run: ${formatSessionRunDiagnostics(lastRow)}`, + ); +} + function readLatestAssistantTextFromHistory(history: QaChatHistoryResponse | undefined) { for (const message of (history?.messages ?? []).toReversed()) { if (!isRecord(message) || message.role !== "assistant") { @@ -403,11 +531,7 @@ async function runAgentPrompt( ) { const started = await startAgentRun(env, params); const waited = await waitForAgentRun(env, started.runId!, params.timeoutMs ?? 30_000); - if (!isSuccessfulAgentWaitResult(waited)) { - throw new Error( - `agent.wait returned ${waited.status ?? "unknown"}: ${waited.error ?? "no error"}`, - ); - } + assertSuccessfulAgentWaitResult(waited); if (params.transcriptToolName) { await waitForPersistedTranscriptToolEvidence(env, { sessionKey: params.sessionKey, @@ -430,4 +554,5 @@ export { startAgentRun, waitForAgentHistoryReply, waitForAgentRun, + waitForSessionRunAfter, }; diff --git a/extensions/qa-lab/src/suite-runtime-agent.ts b/extensions/qa-lab/src/suite-runtime-agent.ts index 1ed898ddb0ba..6387b85be9ff 100644 --- a/extensions/qa-lab/src/suite-runtime-agent.ts +++ b/extensions/qa-lab/src/suite-runtime-agent.ts @@ -16,6 +16,7 @@ export { startAgentRun, waitForAgentHistoryReply, waitForAgentRun, + waitForSessionRunAfter, } from "./suite-runtime-agent-process.js"; export { runQaCli } from "./qa-cli-process.js"; export { inspectQaExecutionIdentityStorage } from "./execution-identity-storage-inspection.js"; diff --git a/qa/scenarios/config/config-restart-capability-flip.yaml b/qa/scenarios/config/config-restart-capability-flip.yaml index aa719b2dda0a..bc951ad4720b 100644 --- a/qa/scenarios/config/config-restart-capability-flip.yaml +++ b/qa/scenarios/config/config-restart-capability-flip.yaml @@ -101,6 +101,9 @@ flow: - set: nextConfig value: expr: "(() => { const nextConfig = structuredClone(original.config); const gatewayConfig = (nextConfig.gateway ??= {}); const controlUi = (gatewayConfig.controlUi ??= {}); const allowedOrigins = Array.isArray(controlUi.allowedOrigins) ? [...controlUi.allowedOrigins] : []; controlUi.allowedOrigins = [...allowedOrigins, restartOrigin]; return nextConfig; })()" + - set: restartRequestedAtMs + value: + expr: "Date.now()" - call: applyConfig saveAs: applyResult args: @@ -136,13 +139,13 @@ flow: - expr: liveTurnTimeoutMs(env, config.imageTurnTimeoutMs) - sinceIndex: ref: wakeStartIndex - - call: waitForCondition + - call: waitForSessionRunAfter args: - - lambda: - async: true - expr: "env.gateway.call('sessions.list', {}, { timeoutMs: 30000 }).then((result) => result.sessions?.find((session) => session.key === sessionKey && session.hasActiveRun === false))" + - ref: env + - ref: sessionKey + - qa + - ref: restartRequestedAtMs - expr: liveTurnTimeoutMs(env, config.imageTurnTimeoutMs) - - 100 - call: waitForCondition saveAs: afterTools args: