diff --git a/extensions/qa-lab/src/scenario-catalog-causality.test.ts b/extensions/qa-lab/src/scenario-catalog-causality.test.ts index cb23e619badb..61fe2cb2b9fc 100644 --- a/extensions/qa-lab/src/scenario-catalog-causality.test.ts +++ b/extensions/qa-lab/src/scenario-catalog-causality.test.ts @@ -112,7 +112,9 @@ describe("qa scenario catalog causality", () => { expect(liveMultiRestartContract).toContain("dmScope: env.cfg.session?.dmScope"); expect(liveMultiRestartContract).toContain('"saveAs":"inbound"'); expect(liveMultiRestartContract).toContain("probeText: config.finalMarker"); - expect(liveMultiRestartContract).toContain("completedToolCallCounts.wait ?? 0) < checkpoint"); + expect(liveMultiRestartContract).toContain( + "assistantToolCallCounts.wait ?? 0) > (summary.completedToolCallCounts.wait ?? 0)", + ); expect(checkpointTranscriptIndex).toBeGreaterThanOrEqual(0); expect(checkpointStoreIndex).toBeGreaterThan(checkpointTranscriptIndex); expect(checkpointPersistenceAssertIndex).toBeGreaterThan(checkpointStoreIndex); @@ -199,7 +201,12 @@ describe("qa scenario catalog causality", () => { expect(contract.match(/"sendInbound"/gu)).toHaveLength(1); expect(contract).not.toContain("startAgentRun"); expect(contract).not.toContain("chat.send"); - expect(contract).toContain("completedToolCallCounts.wait ?? 0) < checkpoint"); + expect(contract).toContain( + "assistantToolCallCounts.wait ?? 0) > (summary.completedToolCallCounts.wait ?? 0)", + ); + expect(contract).toContain( + "checkpointTranscript.assistantToolCallCounts.wait ?? 0) > (checkpointTranscript.completedToolCallCounts.wait ?? 0)", + ); expect(contract).toContain("probeText: config.promptMarker"); expect(pendingWaitIndex).toBeGreaterThanOrEqual(0); expect(checkpointStoreIndex).toBeGreaterThan(pendingWaitIndex); diff --git a/qa/scenarios/runtime/gateway-restart-inflight-run.yaml b/qa/scenarios/runtime/gateway-restart-inflight-run.yaml index 52fe63a1a6e6..2430d224ea6c 100644 --- a/qa/scenarios/runtime/gateway-restart-inflight-run.yaml +++ b/qa/scenarios/runtime/gateway-restart-inflight-run.yaml @@ -123,11 +123,11 @@ flow: args: - lambda: async: true - expr: "readSessionTranscriptSummary(env, sessionKey, { probeText: config.promptMarker }).then((summary) => (summary.assistantToolCallCounts.exec ?? 0) >= checkpoint && (summary.assistantToolCallCounts.wait ?? 0) >= checkpoint ? summary : undefined).catch(() => undefined)" + expr: "readSessionTranscriptSummary(env, sessionKey, { probeText: config.promptMarker }).then((summary) => (summary.assistantToolCallCounts.exec ?? 0) >= checkpoint && (summary.assistantToolCallCounts.wait ?? 0) >= checkpoint && (summary.assistantToolCallCounts.wait ?? 0) > (summary.completedToolCallCounts.wait ?? 0) ? summary : undefined).catch(() => undefined)" - expr: liveTurnTimeoutMs(env, 120000) - 25 - assert: - expr: "(checkpointTranscript.completedToolCallCounts.wait ?? 0) < checkpoint" + expr: "(checkpointTranscript.assistantToolCallCounts.wait ?? 0) > (checkpointTranscript.completedToolCallCounts.wait ?? 0)" message: expr: "`checkpoint ${checkpoint} wait was not pending before restart: ${JSON.stringify(checkpointTranscript)}`" - set: checkpointRequests diff --git a/qa/scenarios/runtime/gateway-restart-multi-live.yaml b/qa/scenarios/runtime/gateway-restart-multi-live.yaml index cbd0ea236867..ff6dcb89795d 100644 --- a/qa/scenarios/runtime/gateway-restart-multi-live.yaml +++ b/qa/scenarios/runtime/gateway-restart-multi-live.yaml @@ -175,7 +175,7 @@ flow: args: - lambda: async: true - expr: "readSessionTranscriptSummary(env, sessionKey, { probeText: config.finalMarker }).then((summary) => (summary.assistantToolCallCounts.exec ?? 0) >= checkpoint && (summary.assistantToolCallCounts.wait ?? 0) >= checkpoint && (summary.completedToolCallCounts.wait ?? 0) < checkpoint ? summary : undefined).catch(() => undefined)" + expr: "readSessionTranscriptSummary(env, sessionKey, { probeText: config.finalMarker }).then((summary) => (summary.assistantToolCallCounts.exec ?? 0) >= checkpoint && (summary.assistantToolCallCounts.wait ?? 0) >= checkpoint && (summary.assistantToolCallCounts.wait ?? 0) > (summary.completedToolCallCounts.wait ?? 0) ? summary : undefined).catch(() => undefined)" - expr: liveTurnTimeoutMs(env, 180000) - 50 - call: readRawQaSessionStore