fix(qa): require persisted results in runtime parity (#113499)

* fix(qa): bind runtime parity to fixture session

* fix(qa): narrow session store reader input

* test(qa): isolate runtime session selection regression

* fix(ci): restore env surface ratchet

extensions/qa-lab/src/runtime-parity-session-selection.test.ts

@@ -0,0 +1,128 @@
+import path from "node:path";
+import { resolveStorePath, upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime";
+import { appendSessionTranscriptMessageByIdentity } from "openclaw/plugin-sdk/session-transcript-runtime";
+import {
+  appendSqliteTrajectoryRuntimeEvents,
+  formatSqliteSessionFileMarker,
+} from "openclaw/plugin-sdk/sqlite-runtime-testing";
+import { afterEach, describe, expect, it } from "vitest";
+import { captureRuntimeParityCell } from "./runtime-parity.js";
+import { createTempDirHarness } from "./temp-dir.test-helper.js";
+
+const tempDirs = createTempDirHarness();
+
+afterEach(async () => {
+  await tempDirs.cleanup();
+});
+
+async function seedSession(params: {
+  messages: Array<Record<string, unknown>>;
+  parentSessionKey?: string;
+  sessionId: string;
+  sessionKey: string;
+  tempRoot?: string;
+  trajectoryEvents?: Array<{ data?: Record<string, unknown>; type: string }>;
+  updatedAt: number;
+}) {
+  const tempRoot = params.tempRoot ?? (await tempDirs.makeTempDir("qa-runtime-selection-"));
+  const env = { ...process.env, OPENCLAW_STATE_DIR: path.join(tempRoot, "state") };
+  const storePath = resolveStorePath(undefined, { agentId: "qa", env });
+  await upsertSessionEntry({
+    agentId: "qa",
+    env,
+    sessionKey: params.sessionKey,
+    storePath,
+    entry: {
+      sessionId: params.sessionId,
+      sessionFile: formatSqliteSessionFileMarker({
+        agentId: "qa",
+        sessionId: params.sessionId,
+        storePath,
+      }),
+      updatedAt: params.updatedAt,
+      ...(params.parentSessionKey ? { parentSessionKey: params.parentSessionKey } : {}),
+    },
+  });
+  for (const message of params.messages) {
+    await appendSessionTranscriptMessageByIdentity({
+      agentId: "qa",
+      env,
+      sessionId: params.sessionId,
+      sessionKey: params.sessionKey,
+      storePath,
+      message: message as never,
+    });
+  }
+  if (params.trajectoryEvents?.length) {
+    appendSqliteTrajectoryRuntimeEvents(
+      { agentId: "qa", env, sessionId: params.sessionId, storePath },
+      params.trajectoryEvents.map((event, index) => ({
+        traceSchema: "openclaw-trajectory",
+        schemaVersion: 1,
+        traceId: params.sessionId,
+        source: "runtime",
+        type: event.type,
+        ts: new Date(index + 1).toISOString(),
+        seq: index + 1,
+        sessionId: params.sessionId,
+        sessionKey: params.sessionKey,
+        runId: "run-1",
+        data: event.data,
+      })),
+    );
+  }
+  return tempRoot;
+}
+
+describe("runtime parity session selection", () => {
+  it("keeps fixture-owned tool sessions when Codex attaches parent metadata", async () => {
+    const now = Date.now();
+    const rootSessionKey = "agent:qa:unrelated-root";
+    const tempRoot = await seedSession({
+      sessionId: "unrelated-root",
+      sessionKey: rootSessionKey,
+      messages: [{ role: "assistant", content: "Setup complete." }],
+      updatedAt: now,
+    });
+    await seedSession({
+      tempRoot,
+      sessionId: "web-fetch-fixture",
+      sessionKey: "agent:qa:runtime-tool:web_fetch:failure",
+      parentSessionKey: rootSessionKey,
+      messages: [{ role: "user", content: "failure target=web_fetch" }],
+      updatedAt: now - 1_000,
+      trajectoryEvents: [
+        {
+          type: "tool.call",
+          data: {
+            toolCallId: "web-fetch-1",
+            name: "web_fetch",
+            arguments: { __qaFailureMode: "denied-input" },
+          },
+        },
+        {
+          type: "tool.result",
+          data: {
+            toolCallId: "web-fetch-1",
+            name: "web_fetch",
+            status: "failed",
+            success: false,
+            result: { error: "url required" },
+          },
+        },
+      ],
+    });
+
+    const cell = await captureRuntimeParityCell({
+      runtime: "codex",
+      gateway: { tempRoot },
+      scenarioResult: {
+        status: "pass",
+        details: "RUNTIME_PARITY_SESSION_KEY=agent:qa:runtime-tool:web_fetch:failure",
+      },
+      wallClockMs: 10,
+    });
+
+    expect(cell.toolCalls).toEqual([expect.objectContaining({ tool: "web_fetch" })]);
+  });
+});

extensions/qa-lab/src/runtime-parity.test.ts

@@ -545,7 +545,6 @@ describe("runtime parity", () => {
     expect(missingCell.transcriptBytes).toBe("");
     expect(missingCell.toolCalls).toEqual([]);
   });
-
   it("keeps an explicitly identified orphan result separate", async () => {
     const tempRoot = await seedRuntimeParityTranscript({
       sessionId: "orphan-trajectory-result",

extensions/qa-lab/src/runtime-parity.ts

@@ -1,5 +1,4 @@
 import {
-  listSessionEntries,
   loadTranscriptEventsSync,
   resolveStorePath,
 } from "openclaw/plugin-sdk/session-store-runtime";
@@ -21,6 +20,7 @@ import {
 } from "./gateway-log-sentinel.js";
 import { discardIgnoredResponseBody } from "./ignored-response-body.js";
 import * as parity from "./parity-shared.js";
+import { readRawQaSessionStore } from "./suite-runtime-agent-session.js";
 
 export type RuntimeId = "openclaw" | "codex";
 
@@ -1202,36 +1202,34 @@ function runtimeParitySessionEnv(stateDir: string): NodeJS.ProcessEnv {
   return { ...process.env, OPENCLAW_STATE_DIR: stateDir };
 }
 
-function readRuntimeParitySessionEntries(params: {
-  stateDir: string;
+async function readRuntimeParitySessionEntries(params: {
+  gateway: QaGatewayLike;
   agentId: string;
   preferredSessionKeys?: ReadonlySet<string>;
-}): RuntimeParitySessionCandidate[] {
-  try {
-    const entries = listSessionEntries({
-      agentId: params.agentId,
-      env: runtimeParitySessionEnv(params.stateDir),
-      readOnly: true,
-    })
-      .filter(({ entry }) => readNonEmptyString(entry.sessionId))
-      .map(({ entry, sessionKey }) => ({
-        entry: entry as RuntimeParitySessionEntry,
-        sessionKey,
-      }))
-      .filter(({ entry }) => !readNonEmptyString(entry.heartbeatIsolatedBaseSessionKey));
-    const selectedEntries = params.preferredSessionKeys
-      ? entries.filter(({ sessionKey }) => params.preferredSessionKeys?.has(sessionKey))
-      : entries;
-    const rootEntries = selectedEntries.filter(({ entry }) => isRuntimeParityRootSession(entry));
-    const candidates = rootEntries.length > 0 ? rootEntries : selectedEntries;
-    return candidates.toSorted((left, right) => {
-      const leftCreatedAt = left.entry.createdAt ?? left.entry.updatedAt ?? 0;
-      const rightCreatedAt = right.entry.createdAt ?? right.entry.updatedAt ?? 0;
-      return leftCreatedAt - rightCreatedAt || left.sessionKey.localeCompare(right.sessionKey);
-    });
-  } catch {
-    return [];
-  }
+}): Promise<RuntimeParitySessionCandidate[]> {
+  // This feeds release evidence: after bounded FTS-settle retries, a persistent
+  // store failure must fail capture instead of becoming an empty false green.
+  const store = await readRawQaSessionStore(
+    { gateway: params.gateway },
+    { agentId: params.agentId },
+  );
+  const entries = Object.entries(store)
+    .filter(([, entry]) => readNonEmptyString(entry.sessionId))
+    .map(([sessionKey, entry]) => ({
+      entry: entry as RuntimeParitySessionEntry,
+      sessionKey,
+    }))
+    .filter(({ entry }) => !readNonEmptyString(entry.heartbeatIsolatedBaseSessionKey));
+  const selectedEntries = params.preferredSessionKeys
+    ? entries.filter(({ sessionKey }) => params.preferredSessionKeys?.has(sessionKey))
+    : entries;
+  const rootEntries = selectedEntries.filter(({ entry }) => isRuntimeParityRootSession(entry));
+  const candidates = rootEntries.length > 0 ? rootEntries : selectedEntries;
+  return candidates.toSorted((left, right) => {
+    const leftCreatedAt = left.entry.createdAt ?? left.entry.updatedAt ?? 0;
+    const rightCreatedAt = right.entry.createdAt ?? right.entry.updatedAt ?? 0;
+    return leftCreatedAt - rightCreatedAt || left.sessionKey.localeCompare(right.sessionKey);
+  });
 }
 
 async function loadRuntimeParityCaptureSources(params: {
@@ -1242,8 +1240,8 @@ async function loadRuntimeParityCaptureSources(params: {
   const stateDir = `${params.gateway.tempRoot}/state`;
   const env = runtimeParitySessionEnv(stateDir);
   const storePath = resolveStorePath(undefined, { agentId: params.agentId, env });
-  const sessionEntries = readRuntimeParitySessionEntries({
-    stateDir,
+  const sessionEntries = await readRuntimeParitySessionEntries({
+    gateway: params.gateway,
     agentId: params.agentId,
     ...(params.preferredSessionKeys?.length
       ? { preferredSessionKeys: new Set(params.preferredSessionKeys) }

extensions/qa-lab/src/suite-runtime-agent-process.test.ts

@@ -784,13 +784,15 @@ describe("qa suite runtime agent process helpers", () => {
       readSessionTranscriptSummaryMock
         .mockResolvedValueOnce({
           assistantToolCallCounts: {},
+          completedToolCallCounts: {},
           successfulToolCallCounts: {},
           finalText: "",
         })
         .mockResolvedValueOnce({
           assistantToolCallCounts: { web_fetch: 1 },
+          completedToolCallCounts: { web_fetch: 1 },
           successfulToolCallCounts: { web_fetch: 1 },
-          finalText: "done",
+          finalText: "",
         });
       const env = {
         gateway: { call: gatewayCall },
@@ -821,6 +823,53 @@ describe("qa suite runtime agent process helpers", () => {
     }
   });
 
+  it("waits for a persisted failed tool result after the call is visible", async () => {
+    vi.useFakeTimers();
+    try {
+      const gatewayCall = vi
+        .fn()
+        .mockResolvedValueOnce({ runId: "run-failed-tool-evidence" })
+        .mockResolvedValueOnce({ status: "completed" });
+      readSessionTranscriptSummaryMock
+        .mockResolvedValueOnce({
+          assistantToolCallCounts: { session_status: 1 },
+          completedToolCallCounts: {},
+          successfulToolCallCounts: {},
+          finalText: "",
+        })
+        .mockResolvedValueOnce({
+          assistantToolCallCounts: { session_status: 1 },
+          completedToolCallCounts: { session_status: 1 },
+          successfulToolCallCounts: {},
+          finalText: "",
+        });
+      const env = {
+        gateway: { call: gatewayCall },
+        transport: {
+          buildAgentDelivery: vi.fn(() => ({
+            channel: "qa-channel",
+            replyChannel: "reply-channel",
+            replyTo: "reply-target",
+          })),
+        },
+      } as never;
+
+      const pending = runAgentPrompt(env, {
+        sessionKey: "session-failed-tool-evidence",
+        message: "call session_status with invalid input",
+        transcriptToolName: "session_status",
+      });
+      await vi.advanceTimersByTimeAsync(50);
+
+      await expect(pending).resolves.toEqual({
+        started: { runId: "run-failed-tool-evidence" },
+        waited: { status: "completed" },
+      });
+      expect(readSessionTranscriptSummaryMock).toHaveBeenCalledTimes(2);
+    } finally {
+      vi.useRealTimers();
+    }
+  });
   it("waits for the latest assistant history reply", async () => {
     const gatewayCall = vi
       .fn()

extensions/qa-lab/src/suite-runtime-agent-process.ts

@@ -620,13 +620,9 @@ async function waitForPersistedTranscriptToolEvidence(
       const summary = await readSessionTranscriptSummary(env, params.sessionKey, {
         allowEmpty: true,
       });
-      const callCount = summary.assistantToolCallCounts[params.toolName] ?? 0;
+      const completedCount = summary.completedToolCallCounts[params.toolName] ?? 0;
       const successfulCount = summary.successfulToolCallCounts[params.toolName] ?? 0;
-      if (
-        summary.finalText &&
-        callCount > 0 &&
-        (!params.requireSuccessfulResult || successfulCount > 0)
-      ) {
+      if (completedCount > 0 && (!params.requireSuccessfulResult || successfulCount > 0)) {
         return;
       }
       lastError = undefined;

extensions/qa-lab/src/suite-runtime-agent-session.test.ts

@@ -188,6 +188,19 @@ describe("qa suite runtime agent session helpers", () => {
     });
   });
 
+  it("reads a requested agent session store", async () => {
+    const readEntries = vi.fn(() => []);
+
+    await expect(
+      readRawQaSessionStore({ gateway: { tempRoot: "/tmp/qa-agent-store" } } as never, {
+        agentId: "alternate",
+        readEntries,
+        retryDelaysMs: [],
+      }),
+    ).resolves.toEqual({});
+    expect(readEntries).toHaveBeenCalledWith(expect.objectContaining({ agentId: "alternate" }));
+  });
+
   it("retries transient FTS integrity mismatches while child transcripts settle", async () => {
     const readEntries = vi
       .fn()
@@ -356,6 +369,7 @@ describe("qa suite runtime agent session helpers", () => {
       ),
     ).resolves.toEqual({
       assistantToolCallCounts: { message: 1 },
+      completedToolCallCounts: {},
       eventCursor: 2,
       successfulToolCallCounts: {},
       finalText: "",
@@ -382,6 +396,7 @@ describe("qa suite runtime agent session helpers", () => {
       ),
     ).resolves.toEqual({
       assistantToolCallCounts: { message: 1 },
+      completedToolCallCounts: {},
       eventCursor: 3,
       successfulToolCallCounts: {},
       finalText: "Sent.",
@@ -436,6 +451,7 @@ describe("qa suite runtime agent session helpers", () => {
       ),
     ).resolves.toEqual({
       assistantToolCallCounts: { message: 1 },
+      completedToolCallCounts: {},
       eventCursor: 4,
       successfulToolCallCounts: {},
       finalText: "Sent.",
@@ -542,6 +558,7 @@ describe("qa suite runtime agent session helpers", () => {
       ),
     ).resolves.toMatchObject({
       assistantToolCallCounts: { update_plan: 2, write: 1 },
+      completedToolCallCounts: { update_plan: 2 },
       successfulToolCallCounts: { update_plan: 1 },
     });
   });
@@ -607,6 +624,7 @@ describe("qa suite runtime agent session helpers", () => {
       }),
     ).resolves.toEqual({
       assistantToolCallCounts: {},
+      completedToolCallCounts: {},
       eventCursor: 0,
       successfulToolCallCounts: {},
       finalText: "",

extensions/qa-lab/src/suite-runtime-agent-session.ts

@@ -48,6 +48,7 @@ const SESSION_STORE_FTS_SETTLE_RETRY_DELAYS_MS = [100, 250, 500, 1_000, 2_000] a
 type QaSessionTranscriptSummary = {
   assistantMirrors?: Array<{ identity: string; text: string }>;
   assistantToolCallCounts: Record<string, number>;
+  completedToolCallCounts: Record<string, number>;
   eventCursor: number;
   successfulToolCallCounts: Record<string, number>;
   finalText: string;
@@ -117,8 +118,10 @@ function summarizeSessionTranscriptEvents(
   const scanner = createDirectReplyTranscriptSentinelScanner();
   const assistantMirrors: Array<{ identity: string; text: string }> = [];
   const assistantToolCallCounts: Record<string, number> = {};
+  const completedToolCallCounts: Record<string, number> = {};
   const successfulToolCallCounts: Record<string, number> = {};
   const assistantToolNamesByCallId = new Map<string, string>();
+  const completedToolCallIds = new Set<string>();
   const successfulToolCallIds = new Set<string>();
   let finalText = "";
   let lastAssistantContentTypes: string[] = [];
@@ -136,6 +139,15 @@ function summarizeSessionTranscriptEvents(
     if (message.role === "toolResult") {
       const toolCallId = readNonEmptyString(message.toolCallId);
       const toolName = readNonEmptyString(message.toolName);
+      if (
+        toolCallId &&
+        toolName &&
+        assistantToolNamesByCallId.get(toolCallId) === toolName &&
+        !completedToolCallIds.has(toolCallId)
+      ) {
+        completedToolCallIds.add(toolCallId);
+        completedToolCallCounts[toolName] = (completedToolCallCounts[toolName] ?? 0) + 1;
+      }
       if (
         toolCallId &&
         toolName &&
@@ -186,6 +198,7 @@ function summarizeSessionTranscriptEvents(
   return {
     ...(assistantMirrors.length > 0 ? { assistantMirrors } : {}),
     assistantToolCallCounts,
+    completedToolCallCounts,
     eventCursor,
     successfulToolCallCounts,
     finalText,
@@ -201,6 +214,7 @@ function summarizeSessionTranscriptEvents(
 function emptySessionTranscriptSummary(eventCursor: number): QaSessionTranscriptSummary {
   return {
     assistantToolCallCounts: {},
+    completedToolCallCounts: {},
     eventCursor,
     successfulToolCallCounts: {},
     finalText: "",
@@ -351,19 +365,21 @@ async function seedQaSessionTranscript(
 }
 
 async function readRawQaSessionStore(
-  env: Pick<QaSuiteRuntimeEnv, "gateway">,
+  env: { gateway: Pick<QaSuiteRuntimeEnv["gateway"], "tempRoot"> },
   options: {
+    agentId?: string;
     readEntries?: typeof listSessionEntries;
     retryDelaysMs?: readonly number[];
   } = {},
 ) {
   const runtimeEnv = qaSessionRuntimeEnv(env.gateway.tempRoot);
+  const agentId = readNonEmptyString(options.agentId) ?? "qa";
   const readEntries = options.readEntries ?? listSessionEntries;
   const retryDelaysMs = options.retryDelaysMs ?? SESSION_STORE_FTS_SETTLE_RETRY_DELAYS_MS;
   for (let attempt = 0; attempt <= retryDelaysMs.length; attempt += 1) {
     try {
       return Object.fromEntries(
-        readEntries({ agentId: "qa", env: runtimeEnv }).map(({ sessionKey, entry }) => [
+        readEntries({ agentId, env: runtimeEnv }).map(({ sessionKey, entry }) => [
           sessionKey,
           entry as QaRawSessionStoreEntry,
         ]),

packages/ai/src/providers/openai-response-format.ts

@@ -1,6 +1,6 @@
 import { isRecord } from "@openclaw/normalization-core/record-coerce";
 
-const OPENCLAW_RESPONSE_FORMAT_NAME = "openclaw_response";
+const JSON_SCHEMA_RESPONSE_FORMAT_NAME = "openclaw_response";
 const OLLAMA_CLOUD_ORIGIN = "https://ollama.com";
 
 export function isKnownOpenAIJsonSchemaModelId(modelId: string | undefined): boolean {
@@ -62,7 +62,7 @@ export function resolveOpenAICompletionsResponseFormat(
   return {
     type: "json_schema",
     json_schema: {
-      name: OPENCLAW_RESPONSE_FORMAT_NAME,
+      name: JSON_SCHEMA_RESPONSE_FORMAT_NAME,
       schema: responseFormat,
     },
   };