test(qa): cover runtime telemetry evidence (#118950)

* test(qa): add telemetry runtime evidence * test(qa): fix telemetry evidence typecheck
2026-08-12 21:53:00 -06:00 · 2026-08-04 06:11:07 +08:00
parent d3ea24c669
commit 0483ca712b
3 changed files with 397 additions and 0 deletions
@@ -0,0 +1,97 @@
+title: Telemetry QA Lab bus runtime evidence
+
+scenario:
+  id: telemetry-qa-lab-bus
+  surface: harness
+  coverage:
+    primary:
+      - observability.telemetry-qa-lab
+      - observability.telemetry-qa-bus
+  objective: Verify a real QA-channel agent turn exposes searchable, sanitized tool evidence through the QA Lab bus.
+  successCriteria:
+    - A real inbound QA-channel message reaches the mock OpenAI-backed agent runtime.
+    - The agent reads the seeded workspace file before replying.
+    - The durable outbound QA-bus message carries the observed read tool call.
+    - Tool arguments are redacted while the tool name remains searchable.
+  docsRefs:
+    - docs/concepts/qa-e2e-automation.md
+    - docs/channels/qa-channel.md
+  codeRefs:
+    - extensions/qa-channel/src/inbound.ts
+    - extensions/qa-lab/src/bus-state.ts
+    - extensions/qa-lab/src/providers/mock-openai/server.ts
+  execution:
+    kind: flow
+    summary: Send a real QA-channel turn and inspect its sanitized outbound tool trace on the QA Lab bus.
+    channel: qa-channel
+    config:
+      conversationId: telemetry-qa-lab-bus
+      senderId: qa-telemetry-operator
+      probeFile: QA_TELEMETRY_PROBE.md
+      replyMarker: TELEMETRY-QA-BUS-OK
+      prompt: Read QA_TELEMETRY_PROBE.md, then reply exactly TELEMETRY-QA-BUS-OK.
+
+flow:
+  steps:
+    - name: records searchable sanitized tool evidence from a real channel turn
+      actions:
+        - call: waitForGatewayHealthy
+          args:
+            - ref: env
+            - 60000
+        - call: waitForQaChannelReady
+          args:
+            - ref: env
+            - 60000
+        - call: reset
+        - call: fs.writeFile
+          args:
+            - expr: "path.join(env.gateway.workspaceDir, config.probeFile)"
+            - "Telemetry QA bus probe.\n"
+            - utf8
+        - set: outboundStartIndex
+          value:
+            expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
+        - sendInbound:
+            conversation:
+              id:
+                ref: config.conversationId
+              kind: direct
+            senderId:
+              ref: config.senderId
+            senderName: QA Telemetry Operator
+            text:
+              ref: config.prompt
+          saveAs: inbound
+        - waitForOutbound:
+            conversation:
+              id:
+                ref: config.conversationId
+              kind: direct
+            sinceIndex:
+              ref: outboundStartIndex
+            textIncludes:
+              ref: config.replyMarker
+            timeoutMs:
+              expr: liveTurnTimeoutMs(env, 60000)
+          saveAs: outbound
+        - assert:
+            expr: "outbound.replyToId === inbound.id"
+            message:
+              expr: "`outbound telemetry reply was not attached to the real inbound turn: inbound=${inbound.id} replyTo=${String(outbound.replyToId ?? '')}`"
+        - assert:
+            expr: "outbound.toolCalls?.map((toolCall) => toolCall.name).includes('read') === true"
+            message:
+              expr: "`durable QA-bus reply did not include the runtime read trace: ${JSON.stringify(outbound.toolCalls ?? null)}`"
+        - assert:
+            expr: "JSON.stringify(outbound.toolCalls ?? []).includes('[redacted]') && !JSON.stringify(outbound.toolCalls ?? []).includes(config.probeFile)"
+            message:
+              expr: "`QA-bus tool arguments were not safely redacted: ${JSON.stringify(outbound.toolCalls ?? null)}`"
+        - set: searchMatches
+          value:
+            expr: "state.searchMessages({ query: 'read' })"
+        - assert:
+            expr: "searchMatches.some((message) => message.id === outbound.id)"
+            message:
+              expr: "`QA-bus search could not find the real outbound tool trace: ${JSON.stringify(searchMatches.map((message) => message.id))}`"
+      detailsExpr: "`reply=${outbound.text}; tool=${outbound.toolCalls?.[0]?.name ?? 'missing'}; searchable=${searchMatches.some((message) => message.id === outbound.id)}`"
@@ -0,0 +1,179 @@
+title: Telemetry task evidence followthrough
+
+scenario:
+  id: telemetry-task-evidence-followthrough
+  surface: harness
+  coverage:
+    primary:
+      - observability.telemetry-evidence
+      - observability.telemetry-no-fake-progress
+      - observability.telemetry-task-followthrough
+  objective: Verify a real QA-channel task reads its evidence, writes a status artifact, and reports only proof-backed Pending, Blocked, and Done claims.
+  successCriteria:
+    - The agent reads both seeded task files before writing the status artifact.
+    - The durable outbound tool trace records the ordered read, read, write sequence.
+    - The status artifact exists before the final channel reply.
+    - Pending, Blocked, and Done claims match the artifact without overstating external progress.
+  docsRefs:
+    - docs/concepts/qa-e2e-automation.md
+    - docs/channels/qa-channel.md
+  codeRefs:
+    - extensions/qa-channel/src/inbound.ts
+    - extensions/qa-lab/src/providers/mock-openai/server.ts
+    - src/agents/embedded-agent-runner/run/incomplete-turn.ts
+  execution:
+    kind: flow
+    summary: Complete a proof-backed QA-channel task and verify artifact-before-reply ordering plus honest terminal status.
+    channel: qa-channel
+    config:
+      conversationId: telemetry-task-evidence-followthrough
+      senderId: qa-telemetry-operator
+      artifactFile: personal-task-status.txt
+      workspaceFiles:
+        PERSONAL_TASK_LEDGER.md: |-
+          # Personal task ledger
+
+          Required status contract:
+          1. Read PERSONAL_TASK_LEDGER.md.
+          2. Read FOLLOWTHROUGH_NOTE.md.
+          3. Write ./personal-task-status.txt.
+          4. Include these exact status fields:
+             Personal task followthrough
+             Pending: maintainer feedback before publishing.
+             Blocked: publishing needs explicit user approval.
+             Done: local evidence captured in personal-task-status.txt.
+          5. Reply with three labeled lines exactly once: Pending, Blocked, Done.
+
+          Do not mark the task done until the status artifact has been written.
+        FOLLOWTHROUGH_NOTE.md: |-
+          Task: prepare a local OpenClaw PR readiness note.
+          Pending: wait for maintainer feedback before publishing.
+          Blocked: publishing needs explicit user approval.
+          Done: local evidence captured in personal-task-status.txt.
+      prompt: |-
+        Personal task followthrough check. Read PERSONAL_TASK_LEDGER.md and FOLLOWTHROUGH_NOTE.md first.
+        Then write ./personal-task-status.txt with the exact status fields from PERSONAL_TASK_LEDGER.md and reply with three labeled lines: Pending, Blocked, Done.
+        Do not claim the task is done until the status file exists.
+      expectedReplyAll:
+        - "pending:"
+        - maintainer feedback
+        - "blocked:"
+        - explicit user approval
+        - "done:"
+        - local evidence captured
+      expectedArtifactAll:
+        - personal task followthrough
+        - "pending:"
+        - maintainer feedback
+        - "blocked:"
+        - explicit user approval
+        - "done:"
+        - local evidence captured
+      forbiddenNeedles:
+        - fully complete
+        - published successfully
+        - nothing is blocked
+        - no blockers
+        - ready to publish
+
+flow:
+  steps:
+    - name: reports only artifact-backed task progress
+      actions:
+        - call: waitForGatewayHealthy
+          args:
+            - ref: env
+            - 60000
+        - call: waitForQaChannelReady
+          args:
+            - ref: env
+            - 60000
+        - call: reset
+        - forEach:
+            items:
+              expr: "Object.entries(config.workspaceFiles ?? {})"
+            item: workspaceFile
+            actions:
+              - call: fs.writeFile
+                args:
+                  - expr: "path.join(env.gateway.workspaceDir, String(workspaceFile[0]))"
+                  - expr: "`${String(workspaceFile[1] ?? '').trimEnd()}\\n`"
+                  - utf8
+        - set: artifactPath
+          value:
+            expr: "path.join(env.gateway.workspaceDir, config.artifactFile)"
+        - call: fs.rm
+          args:
+            - ref: artifactPath
+            - force: true
+        - set: outboundStartIndex
+          value:
+            expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
+        - sendInbound:
+            conversation:
+              id:
+                ref: config.conversationId
+              kind: direct
+            senderId:
+              ref: config.senderId
+            senderName: QA Telemetry Operator
+            text:
+              ref: config.prompt
+          saveAs: inbound
+        - waitForOutbound:
+            conversation:
+              id:
+                ref: config.conversationId
+              kind: direct
+            sinceIndex:
+              ref: outboundStartIndex
+            textIncludes: "Done: local evidence captured"
+            timeoutMs:
+              expr: liveTurnTimeoutMs(env, 60000)
+          saveAs: outbound
+        - call: waitForCondition
+          saveAs: artifact
+          args:
+            - lambda:
+                async: true
+                expr: "fs.readFile(artifactPath, 'utf8').then((value) => config.expectedArtifactAll.every((needle) => normalizeLowercaseStringOrEmpty(value).includes(normalizeLowercaseStringOrEmpty(needle))) ? value : undefined).catch(() => undefined)"
+            - expr: liveTurnTimeoutMs(env, 30000)
+            - expr: "env.providerMode === 'mock-openai' ? 100 : 250"
+        - set: artifactStat
+          value:
+            expr: "await fs.stat(artifactPath)"
+        - set: normalizedReply
+          value:
+            expr: "normalizeLowercaseStringOrEmpty(outbound.text)"
+        - set: normalizedArtifact
+          value:
+            expr: "normalizeLowercaseStringOrEmpty(artifact)"
+        - assert:
+            expr: "outbound.replyToId === inbound.id"
+            message:
+              expr: "`task telemetry reply was not attached to the real inbound turn: inbound=${inbound.id} replyTo=${String(outbound.replyToId ?? '')}`"
+        - assert:
+            expr: "config.expectedReplyAll.every((needle) => normalizedReply.includes(normalizeLowercaseStringOrEmpty(needle)))"
+            message:
+              expr: "`terminal task status omitted required claims: ${outbound.text}`"
+        - assert:
+            expr: "config.expectedArtifactAll.every((needle) => normalizedArtifact.includes(normalizeLowercaseStringOrEmpty(needle)))"
+            message:
+              expr: "`task evidence artifact omitted required claims: ${artifact}`"
+        - assert:
+            expr: "!config.forbiddenNeedles.some((needle) => normalizedReply.includes(normalizeLowercaseStringOrEmpty(needle)))"
+            message:
+              expr: "`terminal task status overclaimed external progress: ${outbound.text}`"
+        - assert:
+            expr: "JSON.stringify(outbound.toolCalls?.map((toolCall) => toolCall.name) ?? []) === JSON.stringify(['read', 'read', 'write'])"
+            message:
+              expr: "`expected real ordered read/read/write telemetry, got ${JSON.stringify(outbound.toolCalls ?? null)}`"
+        - assert:
+            expr: "JSON.stringify(outbound.toolCalls ?? []).includes('[redacted]') && !JSON.stringify(outbound.toolCalls ?? []).includes('PERSONAL_TASK_LEDGER.md') && !JSON.stringify(outbound.toolCalls ?? []).includes('FOLLOWTHROUGH_NOTE.md') && !JSON.stringify(outbound.toolCalls ?? []).includes(config.artifactFile)"
+            message:
+              expr: "`task tool telemetry leaked raw arguments: ${JSON.stringify(outbound.toolCalls ?? null)}`"
+        - assert:
+            expr: "artifactStat.mtimeMs <= outbound.timestamp"
+            message:
+              expr: "`final Done claim preceded its status artifact: artifactMtime=${artifactStat.mtimeMs} outboundTimestamp=${outbound.timestamp}`"
+      detailsExpr: "`${outbound.text}; artifactBeforeReply=${artifactStat.mtimeMs <= outbound.timestamp}; tools=${outbound.toolCalls?.map((toolCall) => toolCall.name).join(',') ?? 'missing'}`"
@@ -0,0 +1,121 @@
+// Telemetry runtime boundary tests cover real QA-channel evidence and honest task followthrough.
+import fs from "node:fs/promises";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import {
+  runQaSuite,
+  startQaLabServer,
+  validateQaEvidenceSummaryJson,
+} from "../../../../extensions/qa-lab/api.js";
+import { useAutoCleanupTempDirTracker } from "../../../helpers/temp-dir.js";
+
+const SCENARIO_IDS = ["telemetry-qa-lab-bus", "telemetry-task-evidence-followthrough"] as const;
+const PRIMARY_COVERAGE_IDS = [
+  "observability.telemetry-evidence",
+  "observability.telemetry-no-fake-progress",
+  "observability.telemetry-qa-bus",
+  "observability.telemetry-qa-lab",
+  "observability.telemetry-task-followthrough",
+] as const;
+const tempDirs = useAutoCleanupTempDirTracker(afterEach);
+
+describe("telemetry runtime boundaries", () => {
+  it("writes exact QA Lab evidence for real bus telemetry and task followthrough", async () => {
+    const repoRoot = process.cwd();
+    const artifactRoot = path.join(repoRoot, ".artifacts", "qa-e2e");
+    await fs.mkdir(artifactRoot, { recursive: true });
+    const outputDir = tempDirs.make("openclaw-telemetry-runtime-", artifactRoot);
+    const runtime = await runQaSuite({
+      alternateModel: "mock-openai/gpt-5.6-luna",
+      concurrency: 1,
+      outputDir,
+      primaryModel: "mock-openai/gpt-5.6-luna",
+      providerMode: "mock-openai",
+      repoRoot,
+      scenarioIds: [...SCENARIO_IDS],
+      startLab: startQaLabServer,
+    });
+
+    expect(runtime.executionKind).toBe("flow");
+    expect(runtime.result.scenarios).toEqual(
+      SCENARIO_IDS.map(() =>
+        expect.objectContaining({
+          name: expect.any(String),
+          status: "pass",
+          steps: [expect.objectContaining({ status: "pass" })],
+        }),
+      ),
+    );
+
+    const summary = JSON.parse(await fs.readFile(runtime.result.summaryPath, "utf8")) as {
+      counts?: { failed?: number; passed?: number; skipped?: number; total?: number };
+      run?: {
+        providerMode?: string;
+        primaryModel?: string;
+        scenarioIds?: string[];
+      };
+      scenarios?: Array<{ name?: string; status?: string }>;
+    };
+    expect(summary.counts).toEqual({ total: 2, passed: 2, failed: 0, skipped: 0 });
+    expect(summary.run).toMatchObject({
+      providerMode: "mock-openai",
+      primaryModel: "mock-openai/gpt-5.6-luna",
+      scenarioIds: [...SCENARIO_IDS],
+    });
+    expect(summary.scenarios?.map((scenario) => scenario.status)).toEqual(["pass", "pass"]);
+
+    const evidence = validateQaEvidenceSummaryJson(
+      JSON.parse(await fs.readFile(runtime.result.evidencePath, "utf8")),
+    );
+    expect(evidence.entries.map((entry) => entry.test.id)).toEqual([...SCENARIO_IDS]);
+    expect(
+      evidence.entries
+        .flatMap((entry) => entry.coverage)
+        .filter((coverage) => coverage.role === "primary")
+        .map((coverage) => coverage.id)
+        .toSorted(),
+    ).toEqual([...PRIMARY_COVERAGE_IDS].toSorted());
+
+    for (const entry of evidence.entries) {
+      expect(entry.result.status).toBe("pass");
+      expect(entry.execution).toMatchObject({
+        runner: "host",
+        provider: {
+          id: "openai",
+          live: false,
+          fixture: "mock-openai",
+          model: {
+            name: "gpt-5.6-luna",
+            ref: "mock-openai/gpt-5.6-luna",
+          },
+        },
+        channel: {
+          id: "qa-channel",
+          live: false,
+        },
+        packageSource: { kind: "source-checkout" },
+      });
+      expect(entry.execution?.environment).toEqual({
+        ref: expect.any(String),
+        os: expect.any(String),
+        nodeVersion: expect.stringMatching(/^v\d+/u),
+      });
+      expect(entry.execution?.artifacts).toEqual([
+        { kind: "summary", path: "qa-suite-summary.json", source: "qa-suite" },
+        { kind: "report", path: "qa-suite-report.md", source: "qa-suite" },
+      ]);
+    }
+
+    const artifactText = await Promise.all(
+      [runtime.result.summaryPath, runtime.result.evidencePath, runtime.result.reportPath].map(
+        async (artifactPath) => await fs.readFile(artifactPath, "utf8"),
+      ),
+    );
+    const serializedArtifacts = artifactText.join("\n");
+    expect(serializedArtifacts).not.toContain(repoRoot);
+    expect(serializedArtifacts).not.toContain("PERSONAL_TASK_LEDGER.md");
+    expect(serializedArtifacts).not.toContain("FOLLOWTHROUGH_NOTE.md");
+    expect(serializedArtifacts).not.toContain("QA_TELEMETRY_PROBE.md");
+    expect(path.basename(runtime.result.outputDir)).toMatch(/^openclaw-telemetry-runtime-/u);
+  }, 180_000);
+});