diff --git a/extensions/qa-lab/src/runtime-parity-session-selection.test.ts b/extensions/qa-lab/src/runtime-parity-session-selection.test.ts new file mode 100644 index 000000000000..43ba26cab10d --- /dev/null +++ b/extensions/qa-lab/src/runtime-parity-session-selection.test.ts @@ -0,0 +1,128 @@ +import path from "node:path"; +import { resolveStorePath, upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { appendSessionTranscriptMessageByIdentity } from "openclaw/plugin-sdk/session-transcript-runtime"; +import { + appendSqliteTrajectoryRuntimeEvents, + formatSqliteSessionFileMarker, +} from "openclaw/plugin-sdk/sqlite-runtime-testing"; +import { afterEach, describe, expect, it } from "vitest"; +import { captureRuntimeParityCell } from "./runtime-parity.js"; +import { createTempDirHarness } from "./temp-dir.test-helper.js"; + +const tempDirs = createTempDirHarness(); + +afterEach(async () => { + await tempDirs.cleanup(); +}); + +async function seedSession(params: { + messages: Array>; + parentSessionKey?: string; + sessionId: string; + sessionKey: string; + tempRoot?: string; + trajectoryEvents?: Array<{ data?: Record; type: string }>; + updatedAt: number; +}) { + const tempRoot = params.tempRoot ?? (await tempDirs.makeTempDir("qa-runtime-selection-")); + const env = { ...process.env, OPENCLAW_STATE_DIR: path.join(tempRoot, "state") }; + const storePath = resolveStorePath(undefined, { agentId: "qa", env }); + await upsertSessionEntry({ + agentId: "qa", + env, + sessionKey: params.sessionKey, + storePath, + entry: { + sessionId: params.sessionId, + sessionFile: formatSqliteSessionFileMarker({ + agentId: "qa", + sessionId: params.sessionId, + storePath, + }), + updatedAt: params.updatedAt, + ...(params.parentSessionKey ? { parentSessionKey: params.parentSessionKey } : {}), + }, + }); + for (const message of params.messages) { + await appendSessionTranscriptMessageByIdentity({ + agentId: "qa", + env, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + storePath, + message: message as never, + }); + } + if (params.trajectoryEvents?.length) { + appendSqliteTrajectoryRuntimeEvents( + { agentId: "qa", env, sessionId: params.sessionId, storePath }, + params.trajectoryEvents.map((event, index) => ({ + traceSchema: "openclaw-trajectory", + schemaVersion: 1, + traceId: params.sessionId, + source: "runtime", + type: event.type, + ts: new Date(index + 1).toISOString(), + seq: index + 1, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + runId: "run-1", + data: event.data, + })), + ); + } + return tempRoot; +} + +describe("runtime parity session selection", () => { + it("keeps fixture-owned tool sessions when Codex attaches parent metadata", async () => { + const now = Date.now(); + const rootSessionKey = "agent:qa:unrelated-root"; + const tempRoot = await seedSession({ + sessionId: "unrelated-root", + sessionKey: rootSessionKey, + messages: [{ role: "assistant", content: "Setup complete." }], + updatedAt: now, + }); + await seedSession({ + tempRoot, + sessionId: "web-fetch-fixture", + sessionKey: "agent:qa:runtime-tool:web_fetch:failure", + parentSessionKey: rootSessionKey, + messages: [{ role: "user", content: "failure target=web_fetch" }], + updatedAt: now - 1_000, + trajectoryEvents: [ + { + type: "tool.call", + data: { + toolCallId: "web-fetch-1", + name: "web_fetch", + arguments: { __qaFailureMode: "denied-input" }, + }, + }, + { + type: "tool.result", + data: { + toolCallId: "web-fetch-1", + name: "web_fetch", + status: "failed", + success: false, + result: { error: "url required" }, + }, + }, + ], + }); + + const cell = await captureRuntimeParityCell({ + runtime: "codex", + gateway: { tempRoot }, + scenarioResult: { + status: "pass", + details: "RUNTIME_PARITY_SESSION_KEY=agent:qa:runtime-tool:web_fetch:failure", + }, + wallClockMs: 10, + }); + + expect(cell.toolCalls).toEqual([expect.objectContaining({ tool: "web_fetch" })]); + }); +}); diff --git a/extensions/qa-lab/src/runtime-parity.test.ts b/extensions/qa-lab/src/runtime-parity.test.ts index b14a37cb98bb..fc6bb12dc145 100644 --- a/extensions/qa-lab/src/runtime-parity.test.ts +++ b/extensions/qa-lab/src/runtime-parity.test.ts @@ -545,7 +545,6 @@ describe("runtime parity", () => { expect(missingCell.transcriptBytes).toBe(""); expect(missingCell.toolCalls).toEqual([]); }); - it("keeps an explicitly identified orphan result separate", async () => { const tempRoot = await seedRuntimeParityTranscript({ sessionId: "orphan-trajectory-result", diff --git a/extensions/qa-lab/src/runtime-parity.ts b/extensions/qa-lab/src/runtime-parity.ts index 5520bef85a92..b3c70073fce1 100644 --- a/extensions/qa-lab/src/runtime-parity.ts +++ b/extensions/qa-lab/src/runtime-parity.ts @@ -1,5 +1,4 @@ import { - listSessionEntries, loadTranscriptEventsSync, resolveStorePath, } from "openclaw/plugin-sdk/session-store-runtime"; @@ -21,6 +20,7 @@ import { } from "./gateway-log-sentinel.js"; import { discardIgnoredResponseBody } from "./ignored-response-body.js"; import * as parity from "./parity-shared.js"; +import { readRawQaSessionStore } from "./suite-runtime-agent-session.js"; export type RuntimeId = "openclaw" | "codex"; @@ -1202,36 +1202,34 @@ function runtimeParitySessionEnv(stateDir: string): NodeJS.ProcessEnv { return { ...process.env, OPENCLAW_STATE_DIR: stateDir }; } -function readRuntimeParitySessionEntries(params: { - stateDir: string; +async function readRuntimeParitySessionEntries(params: { + gateway: QaGatewayLike; agentId: string; preferredSessionKeys?: ReadonlySet; -}): RuntimeParitySessionCandidate[] { - try { - const entries = listSessionEntries({ - agentId: params.agentId, - env: runtimeParitySessionEnv(params.stateDir), - readOnly: true, - }) - .filter(({ entry }) => readNonEmptyString(entry.sessionId)) - .map(({ entry, sessionKey }) => ({ - entry: entry as RuntimeParitySessionEntry, - sessionKey, - })) - .filter(({ entry }) => !readNonEmptyString(entry.heartbeatIsolatedBaseSessionKey)); - const selectedEntries = params.preferredSessionKeys - ? entries.filter(({ sessionKey }) => params.preferredSessionKeys?.has(sessionKey)) - : entries; - const rootEntries = selectedEntries.filter(({ entry }) => isRuntimeParityRootSession(entry)); - const candidates = rootEntries.length > 0 ? rootEntries : selectedEntries; - return candidates.toSorted((left, right) => { - const leftCreatedAt = left.entry.createdAt ?? left.entry.updatedAt ?? 0; - const rightCreatedAt = right.entry.createdAt ?? right.entry.updatedAt ?? 0; - return leftCreatedAt - rightCreatedAt || left.sessionKey.localeCompare(right.sessionKey); - }); - } catch { - return []; - } +}): Promise { + // This feeds release evidence: after bounded FTS-settle retries, a persistent + // store failure must fail capture instead of becoming an empty false green. + const store = await readRawQaSessionStore( + { gateway: params.gateway }, + { agentId: params.agentId }, + ); + const entries = Object.entries(store) + .filter(([, entry]) => readNonEmptyString(entry.sessionId)) + .map(([sessionKey, entry]) => ({ + entry: entry as RuntimeParitySessionEntry, + sessionKey, + })) + .filter(({ entry }) => !readNonEmptyString(entry.heartbeatIsolatedBaseSessionKey)); + const selectedEntries = params.preferredSessionKeys + ? entries.filter(({ sessionKey }) => params.preferredSessionKeys?.has(sessionKey)) + : entries; + const rootEntries = selectedEntries.filter(({ entry }) => isRuntimeParityRootSession(entry)); + const candidates = rootEntries.length > 0 ? rootEntries : selectedEntries; + return candidates.toSorted((left, right) => { + const leftCreatedAt = left.entry.createdAt ?? left.entry.updatedAt ?? 0; + const rightCreatedAt = right.entry.createdAt ?? right.entry.updatedAt ?? 0; + return leftCreatedAt - rightCreatedAt || left.sessionKey.localeCompare(right.sessionKey); + }); } async function loadRuntimeParityCaptureSources(params: { @@ -1242,8 +1240,8 @@ async function loadRuntimeParityCaptureSources(params: { const stateDir = `${params.gateway.tempRoot}/state`; const env = runtimeParitySessionEnv(stateDir); const storePath = resolveStorePath(undefined, { agentId: params.agentId, env }); - const sessionEntries = readRuntimeParitySessionEntries({ - stateDir, + const sessionEntries = await readRuntimeParitySessionEntries({ + gateway: params.gateway, agentId: params.agentId, ...(params.preferredSessionKeys?.length ? { preferredSessionKeys: new Set(params.preferredSessionKeys) } diff --git a/extensions/qa-lab/src/suite-runtime-agent-process.test.ts b/extensions/qa-lab/src/suite-runtime-agent-process.test.ts index f5a497b1dd2a..6f1439805c6e 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-process.test.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-process.test.ts @@ -784,13 +784,15 @@ describe("qa suite runtime agent process helpers", () => { readSessionTranscriptSummaryMock .mockResolvedValueOnce({ assistantToolCallCounts: {}, + completedToolCallCounts: {}, successfulToolCallCounts: {}, finalText: "", }) .mockResolvedValueOnce({ assistantToolCallCounts: { web_fetch: 1 }, + completedToolCallCounts: { web_fetch: 1 }, successfulToolCallCounts: { web_fetch: 1 }, - finalText: "done", + finalText: "", }); const env = { gateway: { call: gatewayCall }, @@ -821,6 +823,53 @@ describe("qa suite runtime agent process helpers", () => { } }); + it("waits for a persisted failed tool result after the call is visible", async () => { + vi.useFakeTimers(); + try { + const gatewayCall = vi + .fn() + .mockResolvedValueOnce({ runId: "run-failed-tool-evidence" }) + .mockResolvedValueOnce({ status: "completed" }); + readSessionTranscriptSummaryMock + .mockResolvedValueOnce({ + assistantToolCallCounts: { session_status: 1 }, + completedToolCallCounts: {}, + successfulToolCallCounts: {}, + finalText: "", + }) + .mockResolvedValueOnce({ + assistantToolCallCounts: { session_status: 1 }, + completedToolCallCounts: { session_status: 1 }, + successfulToolCallCounts: {}, + finalText: "", + }); + const env = { + gateway: { call: gatewayCall }, + transport: { + buildAgentDelivery: vi.fn(() => ({ + channel: "qa-channel", + replyChannel: "reply-channel", + replyTo: "reply-target", + })), + }, + } as never; + + const pending = runAgentPrompt(env, { + sessionKey: "session-failed-tool-evidence", + message: "call session_status with invalid input", + transcriptToolName: "session_status", + }); + await vi.advanceTimersByTimeAsync(50); + + await expect(pending).resolves.toEqual({ + started: { runId: "run-failed-tool-evidence" }, + waited: { status: "completed" }, + }); + expect(readSessionTranscriptSummaryMock).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } + }); it("waits for the latest assistant history reply", async () => { const gatewayCall = vi .fn() diff --git a/extensions/qa-lab/src/suite-runtime-agent-process.ts b/extensions/qa-lab/src/suite-runtime-agent-process.ts index 8cb0a51b278c..3fbba57037fb 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-process.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-process.ts @@ -620,13 +620,9 @@ async function waitForPersistedTranscriptToolEvidence( const summary = await readSessionTranscriptSummary(env, params.sessionKey, { allowEmpty: true, }); - const callCount = summary.assistantToolCallCounts[params.toolName] ?? 0; + const completedCount = summary.completedToolCallCounts[params.toolName] ?? 0; const successfulCount = summary.successfulToolCallCounts[params.toolName] ?? 0; - if ( - summary.finalText && - callCount > 0 && - (!params.requireSuccessfulResult || successfulCount > 0) - ) { + if (completedCount > 0 && (!params.requireSuccessfulResult || successfulCount > 0)) { return; } lastError = undefined; diff --git a/extensions/qa-lab/src/suite-runtime-agent-session.test.ts b/extensions/qa-lab/src/suite-runtime-agent-session.test.ts index 760db2c730fa..7695d0bf15d5 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-session.test.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-session.test.ts @@ -188,6 +188,19 @@ describe("qa suite runtime agent session helpers", () => { }); }); + it("reads a requested agent session store", async () => { + const readEntries = vi.fn(() => []); + + await expect( + readRawQaSessionStore({ gateway: { tempRoot: "/tmp/qa-agent-store" } } as never, { + agentId: "alternate", + readEntries, + retryDelaysMs: [], + }), + ).resolves.toEqual({}); + expect(readEntries).toHaveBeenCalledWith(expect.objectContaining({ agentId: "alternate" })); + }); + it("retries transient FTS integrity mismatches while child transcripts settle", async () => { const readEntries = vi .fn() @@ -356,6 +369,7 @@ describe("qa suite runtime agent session helpers", () => { ), ).resolves.toEqual({ assistantToolCallCounts: { message: 1 }, + completedToolCallCounts: {}, eventCursor: 2, successfulToolCallCounts: {}, finalText: "", @@ -382,6 +396,7 @@ describe("qa suite runtime agent session helpers", () => { ), ).resolves.toEqual({ assistantToolCallCounts: { message: 1 }, + completedToolCallCounts: {}, eventCursor: 3, successfulToolCallCounts: {}, finalText: "Sent.", @@ -436,6 +451,7 @@ describe("qa suite runtime agent session helpers", () => { ), ).resolves.toEqual({ assistantToolCallCounts: { message: 1 }, + completedToolCallCounts: {}, eventCursor: 4, successfulToolCallCounts: {}, finalText: "Sent.", @@ -542,6 +558,7 @@ describe("qa suite runtime agent session helpers", () => { ), ).resolves.toMatchObject({ assistantToolCallCounts: { update_plan: 2, write: 1 }, + completedToolCallCounts: { update_plan: 2 }, successfulToolCallCounts: { update_plan: 1 }, }); }); @@ -607,6 +624,7 @@ describe("qa suite runtime agent session helpers", () => { }), ).resolves.toEqual({ assistantToolCallCounts: {}, + completedToolCallCounts: {}, eventCursor: 0, successfulToolCallCounts: {}, finalText: "", diff --git a/extensions/qa-lab/src/suite-runtime-agent-session.ts b/extensions/qa-lab/src/suite-runtime-agent-session.ts index 9a01b9c92030..ec4597d1614a 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-session.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-session.ts @@ -48,6 +48,7 @@ const SESSION_STORE_FTS_SETTLE_RETRY_DELAYS_MS = [100, 250, 500, 1_000, 2_000] a type QaSessionTranscriptSummary = { assistantMirrors?: Array<{ identity: string; text: string }>; assistantToolCallCounts: Record; + completedToolCallCounts: Record; eventCursor: number; successfulToolCallCounts: Record; finalText: string; @@ -117,8 +118,10 @@ function summarizeSessionTranscriptEvents( const scanner = createDirectReplyTranscriptSentinelScanner(); const assistantMirrors: Array<{ identity: string; text: string }> = []; const assistantToolCallCounts: Record = {}; + const completedToolCallCounts: Record = {}; const successfulToolCallCounts: Record = {}; const assistantToolNamesByCallId = new Map(); + const completedToolCallIds = new Set(); const successfulToolCallIds = new Set(); let finalText = ""; let lastAssistantContentTypes: string[] = []; @@ -136,6 +139,15 @@ function summarizeSessionTranscriptEvents( if (message.role === "toolResult") { const toolCallId = readNonEmptyString(message.toolCallId); const toolName = readNonEmptyString(message.toolName); + if ( + toolCallId && + toolName && + assistantToolNamesByCallId.get(toolCallId) === toolName && + !completedToolCallIds.has(toolCallId) + ) { + completedToolCallIds.add(toolCallId); + completedToolCallCounts[toolName] = (completedToolCallCounts[toolName] ?? 0) + 1; + } if ( toolCallId && toolName && @@ -186,6 +198,7 @@ function summarizeSessionTranscriptEvents( return { ...(assistantMirrors.length > 0 ? { assistantMirrors } : {}), assistantToolCallCounts, + completedToolCallCounts, eventCursor, successfulToolCallCounts, finalText, @@ -201,6 +214,7 @@ function summarizeSessionTranscriptEvents( function emptySessionTranscriptSummary(eventCursor: number): QaSessionTranscriptSummary { return { assistantToolCallCounts: {}, + completedToolCallCounts: {}, eventCursor, successfulToolCallCounts: {}, finalText: "", @@ -351,19 +365,21 @@ async function seedQaSessionTranscript( } async function readRawQaSessionStore( - env: Pick, + env: { gateway: Pick }, options: { + agentId?: string; readEntries?: typeof listSessionEntries; retryDelaysMs?: readonly number[]; } = {}, ) { const runtimeEnv = qaSessionRuntimeEnv(env.gateway.tempRoot); + const agentId = readNonEmptyString(options.agentId) ?? "qa"; const readEntries = options.readEntries ?? listSessionEntries; const retryDelaysMs = options.retryDelaysMs ?? SESSION_STORE_FTS_SETTLE_RETRY_DELAYS_MS; for (let attempt = 0; attempt <= retryDelaysMs.length; attempt += 1) { try { return Object.fromEntries( - readEntries({ agentId: "qa", env: runtimeEnv }).map(({ sessionKey, entry }) => [ + readEntries({ agentId, env: runtimeEnv }).map(({ sessionKey, entry }) => [ sessionKey, entry as QaRawSessionStoreEntry, ]), diff --git a/packages/ai/src/providers/openai-response-format.ts b/packages/ai/src/providers/openai-response-format.ts index 820ce3f7ac31..f2afbd634c10 100644 --- a/packages/ai/src/providers/openai-response-format.ts +++ b/packages/ai/src/providers/openai-response-format.ts @@ -1,6 +1,6 @@ import { isRecord } from "@openclaw/normalization-core/record-coerce"; -const OPENCLAW_RESPONSE_FORMAT_NAME = "openclaw_response"; +const JSON_SCHEMA_RESPONSE_FORMAT_NAME = "openclaw_response"; const OLLAMA_CLOUD_ORIGIN = "https://ollama.com"; export function isKnownOpenAIJsonSchemaModelId(modelId: string | undefined): boolean { @@ -62,7 +62,7 @@ export function resolveOpenAICompletionsResponseFormat( return { type: "json_schema", json_schema: { - name: OPENCLAW_RESPONSE_FORMAT_NAME, + name: JSON_SCHEMA_RESPONSE_FORMAT_NAME, schema: responseFormat, }, };