diff --git a/qa/scenarios/runtime/agent-progress-evidence.yaml b/qa/scenarios/runtime/agent-progress-evidence.yaml new file mode 100644 index 000000000000..bdaba2aea150 --- /dev/null +++ b/qa/scenarios/runtime/agent-progress-evidence.yaml @@ -0,0 +1,222 @@ +title: Agent progress evidence + +scenario: + id: agent-progress-evidence + surface: agent-runtime + coverage: + primary: + - agent-runtime.progress-visibility-evidence + objective: Verify a completion reply is backed by an exact read, read, write result chain and an artifact that predates the reply. + successCriteria: + - Two real reads and one real write execute in exact order with matched call and result identities. + - The written artifact contains the requested evidence and the write result reports success. + - Artifact modification time is no later than the final completion timestamp. + - Exactly one durable completion reply cites the artifact and marker, with none before the write result. + docsRefs: + - docs/concepts/qa-e2e-automation.md + - docs/channels/qa-channel.md + codeRefs: + - extensions/qa-channel/src/inbound.ts + - extensions/qa-lab/src/providers/mock-openai/server.ts + - src/agents/embedded-agent-runner/run/attempt.ts + execution: + kind: flow + channel: qa-channel + providerMode: mock-openai + retryCount: 0 + summary: Match an exact read, read, write chain to one artifact-backed completion reply. + config: + requiredProviderMode: mock-openai + conversationId: agent-progress-evidence + senderId: qa-progress-evidence-operator + promptSnippet: "Personal task followthrough check" + artifactFile: personal-task-status.txt + completionMarker: "Done:" + completionText: "Done: local evidence captured in personal-task-status.txt" + workspaceFiles: + PERSONAL_TASK_LEDGER.md: |- + # Personal task ledger + + Required status contract: + 1. Read PERSONAL_TASK_LEDGER.md. + 2. Read FOLLOWTHROUGH_NOTE.md. + 3. Write ./personal-task-status.txt. + 4. Include these exact status fields: + Personal task followthrough + Pending: maintainer feedback before publishing. + Blocked: publishing needs explicit user approval. + Done: local evidence captured in personal-task-status.txt. + 5. Reply with three labeled lines exactly once: Pending, Blocked, Done. + + Do not mark the task done until the status artifact has been written. + FOLLOWTHROUGH_NOTE.md: |- + Task: prepare a local OpenClaw PR readiness note. + Pending: maintainer feedback before publishing. + Blocked: publishing needs explicit user approval. + Done: local evidence captured in personal-task-status.txt. + prompt: |- + Personal task followthrough check. Read PERSONAL_TASK_LEDGER.md and FOLLOWTHROUGH_NOTE.md first. + Then write ./personal-task-status.txt with the exact status fields from the files. + Report the final status only after the write succeeds. + expectedArtifactAll: + - personal task followthrough + - "pending:" + - maintainer feedback + - "blocked:" + - explicit user approval + - "done:" + - local evidence captured in personal-task-status.txt + +flow: + steps: + - name: gates one durable completion on exact artifact evidence + actions: + - assert: + expr: "env.providerMode === config.requiredProviderMode" + message: + expr: "`expected provider mode ${config.requiredProviderMode}, got ${env.providerMode}`" + - call: waitForGatewayHealthy + args: + - ref: env + - 60000 + - call: waitForQaChannelReady + args: + - ref: env + - 60000 + - call: reset + - forEach: + items: + expr: "Object.entries(config.workspaceFiles ?? {})" + item: workspaceFile + actions: + - call: fs.writeFile + args: + - expr: "path.join(env.gateway.workspaceDir, String(workspaceFile[0]))" + - expr: "`${String(workspaceFile[1] ?? '').trimEnd()}\\n`" + - utf8 + - set: artifactPath + value: + expr: "path.join(env.gateway.workspaceDir, config.artifactFile)" + - call: fs.rm + args: + - ref: artifactPath + - force: true + - set: requestCursorBefore + value: + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor" + - set: busCursorBefore + value: + expr: "state.getSnapshot().cursor" + - set: outboundStartIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: + id: + ref: config.conversationId + kind: direct + senderId: + ref: config.senderId + senderName: QA Progress Evidence Operator + text: + ref: config.prompt + saveAs: inbound + - waitForOutbound: + conversation: + id: + ref: config.conversationId + kind: direct + sinceIndex: + ref: outboundStartIndex + textIncludes: + ref: config.completionText + timeoutMs: + expr: liveTurnTimeoutMs(env, 60000) + saveAs: outbound + - call: fs.readFile + saveAs: artifact + args: + - ref: artifactPath + - utf8 + - call: fs.stat + saveAs: artifactStat + args: + - ref: artifactPath + - set: normalizedArtifact + value: + expr: "normalizeLowercaseStringOrEmpty(artifact)" + - assert: + expr: "config.expectedArtifactAll.every((needle) => normalizedArtifact.includes(normalizeLowercaseStringOrEmpty(needle)))" + message: + expr: "`artifact omitted required evidence: ${artifact}`" + - set: scenarioRequests + value: + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet))" + - assert: + expr: "scenarioRequests.length === 4" + message: + expr: "`expected exactly four provider requests, got ${JSON.stringify(scenarioRequests)}`" + - set: firstReadRequest + value: + expr: "scenarioRequests[0]" + - set: secondReadRequest + value: + expr: "scenarioRequests[1]" + - set: writeRequest + value: + expr: "scenarioRequests[2]" + - set: writeResultRequest + value: + expr: "scenarioRequests[3]" + - assert: + expr: "firstReadRequest.plannedToolName === 'read' && firstReadRequest.plannedToolArgs?.path === 'PERSONAL_TASK_LEDGER.md' && typeof firstReadRequest.plannedToolCallId === 'string' && firstReadRequest.plannedToolCallId.length > 0 && !firstReadRequest.toolOutputCallId" + message: + expr: "`request 1 did not exclusively plan the ledger read: ${JSON.stringify(firstReadRequest)}`" + - assert: + expr: "secondReadRequest.toolOutputCallId === firstReadRequest.plannedToolCallId && secondReadRequest.toolOutputStructuredError !== true && String(secondReadRequest.toolOutput ?? '').includes('# Personal task ledger') && secondReadRequest.plannedToolName === 'read' && secondReadRequest.plannedToolArgs?.path === 'FOLLOWTHROUGH_NOTE.md' && typeof secondReadRequest.plannedToolCallId === 'string' && secondReadRequest.plannedToolCallId.length > 0 && secondReadRequest.plannedToolCallId !== firstReadRequest.plannedToolCallId" + message: + expr: "`request 2 did not consume read 1 and plan a distinct read 2: ${JSON.stringify(secondReadRequest)}`" + - assert: + expr: "writeRequest.toolOutputCallId === secondReadRequest.plannedToolCallId && writeRequest.toolOutputStructuredError !== true && String(writeRequest.toolOutput ?? '').includes('Task: prepare a local OpenClaw PR readiness note.') && writeRequest.plannedToolName === 'write' && writeRequest.plannedToolArgs?.path === config.artifactFile && typeof writeRequest.plannedToolCallId === 'string' && writeRequest.plannedToolCallId.length > 0 && writeRequest.plannedToolCallId !== secondReadRequest.plannedToolCallId" + message: + expr: "`request 3 did not consume read 2 and plan the artifact write: ${JSON.stringify(writeRequest)}`" + - assert: + expr: "writeResultRequest.toolOutputCallId === writeRequest.plannedToolCallId && writeResultRequest.toolOutputStructuredError !== true && /successfully (?:wrote|created|updated|replaced)/i.test(String(writeResultRequest.toolOutput ?? '')) && !writeResultRequest.plannedToolName" + message: + expr: "`request 4 did not consume the successful write result before terminal generation: ${JSON.stringify(writeResultRequest)}`" + - set: allBusEvents + value: + expr: "state.getSnapshot().events.filter((event) => event.cursor > busCursorBefore && 'message' in event && event.message.direction === 'outbound' && event.message.conversation.id === config.conversationId)" + - set: busEvents + value: + expr: "allBusEvents.filter((event) => event.kind === 'outbound-message' || event.kind === 'message-edited')" + - set: completionEvents + value: + expr: "busEvents.filter((event) => event.message.text.includes(config.completionText))" + - set: finalCompletionEvent + value: + expr: "completionEvents.findLast((event) => event.message.id === outbound.id)" + - set: completionMessages + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound' && message.conversation.id === config.conversationId && message.text.includes(config.completionText) && !message.deleted)" + - assert: + expr: "completionMessages.length === 1 && completionMessages[0].id === outbound.id" + message: + expr: "`expected exactly one durable completion reply: ${JSON.stringify(completionMessages)}`" + - assert: + expr: "completionEvents.length >= 1 && finalCompletionEvent && completionEvents.every((event) => artifactStat.mtimeMs <= (event.message.editedAt ?? event.message.timestamp))" + message: + expr: "`completion appeared before the write-backed terminal request: artifactMtime=${artifactStat.mtimeMs} events=${JSON.stringify(completionEvents)}`" + - assert: + expr: "completionEvents.filter((event) => event.message.id !== outbound.id).every((event) => allBusEvents.some((candidate) => candidate.kind === 'message-deleted' && candidate.cursor > event.cursor && candidate.message.id === event.message.id))" + message: + expr: "`a transient completion preview remained durable: ${JSON.stringify(allBusEvents)}`" + - assert: + expr: "outbound.replyToId === inbound.id && outbound.text.includes(config.artifactFile) && outbound.text.includes(config.completionMarker)" + message: + expr: "`durable completion did not cite the artifact and marker: ${JSON.stringify(outbound)}`" + - assert: + expr: "artifactStat.mtimeMs <= (finalCompletionEvent.message.editedAt ?? finalCompletionEvent.message.timestamp)" + message: + expr: "`artifact postdated the durable completion reply: artifactMtime=${artifactStat.mtimeMs} replyTimestamp=${finalCompletionEvent.message.editedAt ?? finalCompletionEvent.message.timestamp}`" + detailsExpr: "`read:${firstReadRequest.plannedToolCallId} -> read:${secondReadRequest.plannedToolCallId} -> write:${writeRequest.plannedToolCallId} -> result:${writeResultRequest.toolOutputCallId}; artifactBeforeReply=${artifactStat.mtimeMs <= (finalCompletionEvent.message.editedAt ?? finalCompletionEvent.message.timestamp)}; durableCompletions=${completionMessages.length}`" diff --git a/qa/scenarios/runtime/agent-progress-task-followthrough.yaml b/qa/scenarios/runtime/agent-progress-task-followthrough.yaml new file mode 100644 index 000000000000..a17282e6c5da --- /dev/null +++ b/qa/scenarios/runtime/agent-progress-task-followthrough.yaml @@ -0,0 +1,145 @@ +title: Agent progress task followthrough + +scenario: + id: agent-progress-task-followthrough + surface: agent-runtime + coverage: + primary: + - agent-runtime.progress-visibility-task-followthrough + objective: Verify visible task progress precedes a real read and the final response follows consumption of that read result. + successCriteria: + - A visible progress marker reaches the QA bus before the read tool starts. + - The read call has a nonempty identity matched by the result consumed in request 2. + - The terminal marker is delivered only after the matched read result. + - QA bus progress delivery is ordered before final delivery. + docsRefs: + - docs/concepts/qa-e2e-automation.md + - docs/channels/qa-channel.md + codeRefs: + - extensions/qa-channel/src/inbound.ts + - extensions/qa-lab/src/providers/mock-openai/server.ts + - src/agents/embedded-agent-runner/run/attempt.ts + execution: + kind: flow + channel: qa-channel + providerMode: mock-openai + retryCount: 0 + summary: Deliver visible progress, execute one real read, consume its matched result, then deliver the final marker. + config: + requiredProviderMode: mock-openai + conversationId: agent-progress-task-followthrough + senderId: qa-progress-operator + fixtureFile: TASK_FOLLOWTHROUGH_EVIDENCE.txt + fixtureContent: "Task followthrough evidence: the read completed successfully.\n" + promptSnippet: "Block streaming QA check" + progressMarker: TASK-FOLLOWTHROUGH-READING + finalMarker: TASK-FOLLOWTHROUGH-OK + prompt: |- + Block streaming QA check: complete this whole sequence in one turn. + Step 1: send an assistant text block containing only this exact marker: `TASK-FOLLOWTHROUGH-READING`. + That first marker block must be emitted before any tool call. + Step 2: after the first marker block, use the read tool exactly once on `TASK_FOLLOWTHROUGH_EVIDENCE.txt`. + Step 3: after that read completes, send a final assistant text block containing only this exact marker: `TASK-FOLLOWTHROUGH-OK`. + Never put both markers in the same assistant text block. + +flow: + steps: + - name: exposes progress before consuming the read result + actions: + - assert: + expr: "env.providerMode === config.requiredProviderMode" + message: + expr: "`expected provider mode ${config.requiredProviderMode}, got ${env.providerMode}`" + - call: waitForGatewayHealthy + args: + - ref: env + - 60000 + - call: waitForQaChannelReady + args: + - ref: env + - 60000 + - call: reset + - call: fs.writeFile + args: + - expr: "path.join(env.gateway.workspaceDir, config.fixtureFile)" + - expr: config.fixtureContent + - utf8 + - set: requestCursorBefore + value: + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor" + - set: busCursorBefore + value: + expr: "state.getSnapshot().cursor" + - set: outboundStartIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: + id: + ref: config.conversationId + kind: direct + senderId: + ref: config.senderId + senderName: QA Progress Operator + text: + ref: config.prompt + saveAs: inbound + - waitForOutbound: + conversation: + id: + ref: config.conversationId + kind: direct + sinceIndex: + ref: outboundStartIndex + textIncludes: + ref: config.finalMarker + timeoutMs: + expr: liveTurnTimeoutMs(env, 60000) + saveAs: outbound + - set: scenarioRequests + value: + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet))" + - assert: + expr: "scenarioRequests.length === 2" + message: + expr: "`expected exactly two provider requests, got ${JSON.stringify(scenarioRequests)}`" + - set: readPlanRequest + value: + expr: "scenarioRequests[0]" + - set: readResultRequest + value: + expr: "scenarioRequests[1]" + - assert: + expr: "readPlanRequest.plannedToolName === 'read' && readPlanRequest.plannedToolArgs?.path === config.fixtureFile && typeof readPlanRequest.plannedToolCallId === 'string' && readPlanRequest.plannedToolCallId.length > 0 && !readPlanRequest.toolOutputCallId" + message: + expr: "`request 1 did not plan the real read with a nonempty call ID: ${JSON.stringify(readPlanRequest)}`" + - assert: + expr: "readResultRequest.toolOutputCallId === readPlanRequest.plannedToolCallId && readResultRequest.toolOutputStructuredError !== true && String(readResultRequest.toolOutput ?? '').includes(config.fixtureContent.trim()) && !readResultRequest.plannedToolName" + message: + expr: "`request 2 did not consume the matched successful read result before terminal generation: ${JSON.stringify(readResultRequest)}`" + - set: busEvents + value: + expr: "state.getSnapshot().events.filter((event) => event.cursor > busCursorBefore && (event.kind === 'outbound-message' || event.kind === 'message-edited') && event.message.direction === 'outbound' && event.message.conversation.id === config.conversationId)" + - set: progressEvent + value: + expr: "busEvents.find((event) => event.message.text.includes(config.progressMarker))" + - set: finalEvent + value: + expr: "busEvents.findLast((event) => event.message.text.includes(config.finalMarker))" + - assert: + expr: "progressEvent && finalEvent && progressEvent.cursor < finalEvent.cursor" + message: + expr: "`QA bus did not deliver progress before final: ${JSON.stringify(busEvents)}`" + - assert: + expr: "!progressEvent.message.toolCalls || progressEvent.message.toolCalls.length === 0" + message: + expr: "`progress marker was not visible before the read started: ${JSON.stringify(progressEvent)}`" + - assert: + expr: "JSON.stringify(finalEvent.message.toolCalls?.map((toolCall) => toolCall.name) ?? []) === JSON.stringify(['read'])" + message: + expr: "`final delivery did not carry the completed read trace: ${JSON.stringify(finalEvent)}`" + - assert: + expr: "outbound.replyToId === inbound.id && outbound.text.trim() === config.finalMarker" + message: + expr: "`terminal marker was not the final reply to the real inbound turn: ${JSON.stringify(outbound)}`" + detailsExpr: "`${progressEvent.kind}@${progressEvent.cursor} -> read:${readPlanRequest.plannedToolCallId} -> result:${readResultRequest.toolOutputCallId} -> ${finalEvent.kind}@${finalEvent.cursor}`"