diff --git a/qa/scenarios/runtime/telemetry-qa-lab-bus.yaml b/qa/scenarios/runtime/telemetry-qa-lab-bus.yaml new file mode 100644 index 000000000000..46944ea0055a --- /dev/null +++ b/qa/scenarios/runtime/telemetry-qa-lab-bus.yaml @@ -0,0 +1,97 @@ +title: Telemetry QA Lab bus runtime evidence + +scenario: + id: telemetry-qa-lab-bus + surface: harness + coverage: + primary: + - observability.telemetry-qa-lab + - observability.telemetry-qa-bus + objective: Verify a real QA-channel agent turn exposes searchable, sanitized tool evidence through the QA Lab bus. + successCriteria: + - A real inbound QA-channel message reaches the mock OpenAI-backed agent runtime. + - The agent reads the seeded workspace file before replying. + - The durable outbound QA-bus message carries the observed read tool call. + - Tool arguments are redacted while the tool name remains searchable. + docsRefs: + - docs/concepts/qa-e2e-automation.md + - docs/channels/qa-channel.md + codeRefs: + - extensions/qa-channel/src/inbound.ts + - extensions/qa-lab/src/bus-state.ts + - extensions/qa-lab/src/providers/mock-openai/server.ts + execution: + kind: flow + summary: Send a real QA-channel turn and inspect its sanitized outbound tool trace on the QA Lab bus. + channel: qa-channel + config: + conversationId: telemetry-qa-lab-bus + senderId: qa-telemetry-operator + probeFile: QA_TELEMETRY_PROBE.md + replyMarker: TELEMETRY-QA-BUS-OK + prompt: Read QA_TELEMETRY_PROBE.md, then reply exactly TELEMETRY-QA-BUS-OK. + +flow: + steps: + - name: records searchable sanitized tool evidence from a real channel turn + actions: + - call: waitForGatewayHealthy + args: + - ref: env + - 60000 + - call: waitForQaChannelReady + args: + - ref: env + - 60000 + - call: reset + - call: fs.writeFile + args: + - expr: "path.join(env.gateway.workspaceDir, config.probeFile)" + - "Telemetry QA bus probe.\n" + - utf8 + - set: outboundStartIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: + id: + ref: config.conversationId + kind: direct + senderId: + ref: config.senderId + senderName: QA Telemetry Operator + text: + ref: config.prompt + saveAs: inbound + - waitForOutbound: + conversation: + id: + ref: config.conversationId + kind: direct + sinceIndex: + ref: outboundStartIndex + textIncludes: + ref: config.replyMarker + timeoutMs: + expr: liveTurnTimeoutMs(env, 60000) + saveAs: outbound + - assert: + expr: "outbound.replyToId === inbound.id" + message: + expr: "`outbound telemetry reply was not attached to the real inbound turn: inbound=${inbound.id} replyTo=${String(outbound.replyToId ?? '')}`" + - assert: + expr: "outbound.toolCalls?.map((toolCall) => toolCall.name).includes('read') === true" + message: + expr: "`durable QA-bus reply did not include the runtime read trace: ${JSON.stringify(outbound.toolCalls ?? null)}`" + - assert: + expr: "JSON.stringify(outbound.toolCalls ?? []).includes('[redacted]') && !JSON.stringify(outbound.toolCalls ?? []).includes(config.probeFile)" + message: + expr: "`QA-bus tool arguments were not safely redacted: ${JSON.stringify(outbound.toolCalls ?? null)}`" + - set: searchMatches + value: + expr: "state.searchMessages({ query: 'read' })" + - assert: + expr: "searchMatches.some((message) => message.id === outbound.id)" + message: + expr: "`QA-bus search could not find the real outbound tool trace: ${JSON.stringify(searchMatches.map((message) => message.id))}`" + detailsExpr: "`reply=${outbound.text}; tool=${outbound.toolCalls?.[0]?.name ?? 'missing'}; searchable=${searchMatches.some((message) => message.id === outbound.id)}`" diff --git a/qa/scenarios/runtime/telemetry-task-evidence-followthrough.yaml b/qa/scenarios/runtime/telemetry-task-evidence-followthrough.yaml new file mode 100644 index 000000000000..0f0e62e50e2f --- /dev/null +++ b/qa/scenarios/runtime/telemetry-task-evidence-followthrough.yaml @@ -0,0 +1,179 @@ +title: Telemetry task evidence followthrough + +scenario: + id: telemetry-task-evidence-followthrough + surface: harness + coverage: + primary: + - observability.telemetry-evidence + - observability.telemetry-no-fake-progress + - observability.telemetry-task-followthrough + objective: Verify a real QA-channel task reads its evidence, writes a status artifact, and reports only proof-backed Pending, Blocked, and Done claims. + successCriteria: + - The agent reads both seeded task files before writing the status artifact. + - The durable outbound tool trace records the ordered read, read, write sequence. + - The status artifact exists before the final channel reply. + - Pending, Blocked, and Done claims match the artifact without overstating external progress. + docsRefs: + - docs/concepts/qa-e2e-automation.md + - docs/channels/qa-channel.md + codeRefs: + - extensions/qa-channel/src/inbound.ts + - extensions/qa-lab/src/providers/mock-openai/server.ts + - src/agents/embedded-agent-runner/run/incomplete-turn.ts + execution: + kind: flow + summary: Complete a proof-backed QA-channel task and verify artifact-before-reply ordering plus honest terminal status. + channel: qa-channel + config: + conversationId: telemetry-task-evidence-followthrough + senderId: qa-telemetry-operator + artifactFile: personal-task-status.txt + workspaceFiles: + PERSONAL_TASK_LEDGER.md: |- + # Personal task ledger + + Required status contract: + 1. Read PERSONAL_TASK_LEDGER.md. + 2. Read FOLLOWTHROUGH_NOTE.md. + 3. Write ./personal-task-status.txt. + 4. Include these exact status fields: + Personal task followthrough + Pending: maintainer feedback before publishing. + Blocked: publishing needs explicit user approval. + Done: local evidence captured in personal-task-status.txt. + 5. Reply with three labeled lines exactly once: Pending, Blocked, Done. + + Do not mark the task done until the status artifact has been written. + FOLLOWTHROUGH_NOTE.md: |- + Task: prepare a local OpenClaw PR readiness note. + Pending: wait for maintainer feedback before publishing. + Blocked: publishing needs explicit user approval. + Done: local evidence captured in personal-task-status.txt. + prompt: |- + Personal task followthrough check. Read PERSONAL_TASK_LEDGER.md and FOLLOWTHROUGH_NOTE.md first. + Then write ./personal-task-status.txt with the exact status fields from PERSONAL_TASK_LEDGER.md and reply with three labeled lines: Pending, Blocked, Done. + Do not claim the task is done until the status file exists. + expectedReplyAll: + - "pending:" + - maintainer feedback + - "blocked:" + - explicit user approval + - "done:" + - local evidence captured + expectedArtifactAll: + - personal task followthrough + - "pending:" + - maintainer feedback + - "blocked:" + - explicit user approval + - "done:" + - local evidence captured + forbiddenNeedles: + - fully complete + - published successfully + - nothing is blocked + - no blockers + - ready to publish + +flow: + steps: + - name: reports only artifact-backed task progress + actions: + - call: waitForGatewayHealthy + args: + - ref: env + - 60000 + - call: waitForQaChannelReady + args: + - ref: env + - 60000 + - call: reset + - forEach: + items: + expr: "Object.entries(config.workspaceFiles ?? {})" + item: workspaceFile + actions: + - call: fs.writeFile + args: + - expr: "path.join(env.gateway.workspaceDir, String(workspaceFile[0]))" + - expr: "`${String(workspaceFile[1] ?? '').trimEnd()}\\n`" + - utf8 + - set: artifactPath + value: + expr: "path.join(env.gateway.workspaceDir, config.artifactFile)" + - call: fs.rm + args: + - ref: artifactPath + - force: true + - set: outboundStartIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: + id: + ref: config.conversationId + kind: direct + senderId: + ref: config.senderId + senderName: QA Telemetry Operator + text: + ref: config.prompt + saveAs: inbound + - waitForOutbound: + conversation: + id: + ref: config.conversationId + kind: direct + sinceIndex: + ref: outboundStartIndex + textIncludes: "Done: local evidence captured" + timeoutMs: + expr: liveTurnTimeoutMs(env, 60000) + saveAs: outbound + - call: waitForCondition + saveAs: artifact + args: + - lambda: + async: true + expr: "fs.readFile(artifactPath, 'utf8').then((value) => config.expectedArtifactAll.every((needle) => normalizeLowercaseStringOrEmpty(value).includes(normalizeLowercaseStringOrEmpty(needle))) ? value : undefined).catch(() => undefined)" + - expr: liveTurnTimeoutMs(env, 30000) + - expr: "env.providerMode === 'mock-openai' ? 100 : 250" + - set: artifactStat + value: + expr: "await fs.stat(artifactPath)" + - set: normalizedReply + value: + expr: "normalizeLowercaseStringOrEmpty(outbound.text)" + - set: normalizedArtifact + value: + expr: "normalizeLowercaseStringOrEmpty(artifact)" + - assert: + expr: "outbound.replyToId === inbound.id" + message: + expr: "`task telemetry reply was not attached to the real inbound turn: inbound=${inbound.id} replyTo=${String(outbound.replyToId ?? '')}`" + - assert: + expr: "config.expectedReplyAll.every((needle) => normalizedReply.includes(normalizeLowercaseStringOrEmpty(needle)))" + message: + expr: "`terminal task status omitted required claims: ${outbound.text}`" + - assert: + expr: "config.expectedArtifactAll.every((needle) => normalizedArtifact.includes(normalizeLowercaseStringOrEmpty(needle)))" + message: + expr: "`task evidence artifact omitted required claims: ${artifact}`" + - assert: + expr: "!config.forbiddenNeedles.some((needle) => normalizedReply.includes(normalizeLowercaseStringOrEmpty(needle)))" + message: + expr: "`terminal task status overclaimed external progress: ${outbound.text}`" + - assert: + expr: "JSON.stringify(outbound.toolCalls?.map((toolCall) => toolCall.name) ?? []) === JSON.stringify(['read', 'read', 'write'])" + message: + expr: "`expected real ordered read/read/write telemetry, got ${JSON.stringify(outbound.toolCalls ?? null)}`" + - assert: + expr: "JSON.stringify(outbound.toolCalls ?? []).includes('[redacted]') && !JSON.stringify(outbound.toolCalls ?? []).includes('PERSONAL_TASK_LEDGER.md') && !JSON.stringify(outbound.toolCalls ?? []).includes('FOLLOWTHROUGH_NOTE.md') && !JSON.stringify(outbound.toolCalls ?? []).includes(config.artifactFile)" + message: + expr: "`task tool telemetry leaked raw arguments: ${JSON.stringify(outbound.toolCalls ?? null)}`" + - assert: + expr: "artifactStat.mtimeMs <= outbound.timestamp" + message: + expr: "`final Done claim preceded its status artifact: artifactMtime=${artifactStat.mtimeMs} outboundTimestamp=${outbound.timestamp}`" + detailsExpr: "`${outbound.text}; artifactBeforeReply=${artifactStat.mtimeMs <= outbound.timestamp}; tools=${outbound.toolCalls?.map((toolCall) => toolCall.name).join(',') ?? 'missing'}`" diff --git a/test/e2e/qa-lab/runtime/telemetry-runtime-boundaries.e2e.test.ts b/test/e2e/qa-lab/runtime/telemetry-runtime-boundaries.e2e.test.ts new file mode 100644 index 000000000000..27523a1d58f2 --- /dev/null +++ b/test/e2e/qa-lab/runtime/telemetry-runtime-boundaries.e2e.test.ts @@ -0,0 +1,121 @@ +// Telemetry runtime boundary tests cover real QA-channel evidence and honest task followthrough. +import fs from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + runQaSuite, + startQaLabServer, + validateQaEvidenceSummaryJson, +} from "../../../../extensions/qa-lab/api.js"; +import { useAutoCleanupTempDirTracker } from "../../../helpers/temp-dir.js"; + +const SCENARIO_IDS = ["telemetry-qa-lab-bus", "telemetry-task-evidence-followthrough"] as const; +const PRIMARY_COVERAGE_IDS = [ + "observability.telemetry-evidence", + "observability.telemetry-no-fake-progress", + "observability.telemetry-qa-bus", + "observability.telemetry-qa-lab", + "observability.telemetry-task-followthrough", +] as const; +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +describe("telemetry runtime boundaries", () => { + it("writes exact QA Lab evidence for real bus telemetry and task followthrough", async () => { + const repoRoot = process.cwd(); + const artifactRoot = path.join(repoRoot, ".artifacts", "qa-e2e"); + await fs.mkdir(artifactRoot, { recursive: true }); + const outputDir = tempDirs.make("openclaw-telemetry-runtime-", artifactRoot); + const runtime = await runQaSuite({ + alternateModel: "mock-openai/gpt-5.6-luna", + concurrency: 1, + outputDir, + primaryModel: "mock-openai/gpt-5.6-luna", + providerMode: "mock-openai", + repoRoot, + scenarioIds: [...SCENARIO_IDS], + startLab: startQaLabServer, + }); + + expect(runtime.executionKind).toBe("flow"); + expect(runtime.result.scenarios).toEqual( + SCENARIO_IDS.map(() => + expect.objectContaining({ + name: expect.any(String), + status: "pass", + steps: [expect.objectContaining({ status: "pass" })], + }), + ), + ); + + const summary = JSON.parse(await fs.readFile(runtime.result.summaryPath, "utf8")) as { + counts?: { failed?: number; passed?: number; skipped?: number; total?: number }; + run?: { + providerMode?: string; + primaryModel?: string; + scenarioIds?: string[]; + }; + scenarios?: Array<{ name?: string; status?: string }>; + }; + expect(summary.counts).toEqual({ total: 2, passed: 2, failed: 0, skipped: 0 }); + expect(summary.run).toMatchObject({ + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.6-luna", + scenarioIds: [...SCENARIO_IDS], + }); + expect(summary.scenarios?.map((scenario) => scenario.status)).toEqual(["pass", "pass"]); + + const evidence = validateQaEvidenceSummaryJson( + JSON.parse(await fs.readFile(runtime.result.evidencePath, "utf8")), + ); + expect(evidence.entries.map((entry) => entry.test.id)).toEqual([...SCENARIO_IDS]); + expect( + evidence.entries + .flatMap((entry) => entry.coverage) + .filter((coverage) => coverage.role === "primary") + .map((coverage) => coverage.id) + .toSorted(), + ).toEqual([...PRIMARY_COVERAGE_IDS].toSorted()); + + for (const entry of evidence.entries) { + expect(entry.result.status).toBe("pass"); + expect(entry.execution).toMatchObject({ + runner: "host", + provider: { + id: "openai", + live: false, + fixture: "mock-openai", + model: { + name: "gpt-5.6-luna", + ref: "mock-openai/gpt-5.6-luna", + }, + }, + channel: { + id: "qa-channel", + live: false, + }, + packageSource: { kind: "source-checkout" }, + }); + expect(entry.execution?.environment).toEqual({ + ref: expect.any(String), + os: expect.any(String), + nodeVersion: expect.stringMatching(/^v\d+/u), + }); + expect(entry.execution?.artifacts).toEqual([ + { kind: "summary", path: "qa-suite-summary.json", source: "qa-suite" }, + { kind: "report", path: "qa-suite-report.md", source: "qa-suite" }, + ]); + } + + const artifactText = await Promise.all( + [runtime.result.summaryPath, runtime.result.evidencePath, runtime.result.reportPath].map( + async (artifactPath) => await fs.readFile(artifactPath, "utf8"), + ), + ); + const serializedArtifacts = artifactText.join("\n"); + expect(serializedArtifacts).not.toContain(repoRoot); + expect(serializedArtifacts).not.toContain("PERSONAL_TASK_LEDGER.md"); + expect(serializedArtifacts).not.toContain("FOLLOWTHROUGH_NOTE.md"); + expect(serializedArtifacts).not.toContain("QA_TELEMETRY_PROBE.md"); + expect(path.basename(runtime.result.outputDir)).toMatch(/^openclaw-telemetry-runtime-/u); + }, 180_000); +});