From 37cf0c6064006c95bcbb7f8329557edcfed6936c Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 27 Jul 2026 19:24:52 -0400 Subject: [PATCH] improve: reduce GPT-5.6 coding harness overhead (#114574) * improve: reduce GPT-5.6 coding harness overhead * refactor: extract runtime parity timing helpers * fix(qa): verify native apply-patch runtime parity * fix(qa): use mock patch execution evidence * fix(qa): preserve workspace templates in full syncs * fix(qa): make rejected patch fixtures deterministic * fix(qa): prove apply-patch workspace containment * fix(qa): expose runtime prompt-cache usage * fix(codex): confine forced private QA patch turns * fix(qa): verify native patch execution and sandbox precedence * fix(qa): execute native patches and prove disk mutation * fix(codex): avoid duplicate native patch registration * perf(agents): keep code mode catalog guidance compact * fix(qa): make native Codex patch evidence deterministic * fix(qa): recognize native sandbox denial errors * fix(qa): advertise native Codex mock model capabilities * fix(qa): seed native Codex mock model catalog at startup * fix(qa): recognize native Codex patch denial results * fix(ci): repair Codex harness parity checks * fix(qa): distinguish unknown prompt-cache measurements * fix(qa): accept native Codex project patch denials * fix(ci): isolate runtime parity report contracts * fix(codex): eliminate native runtime transport overhead * fix(qa): consume runtime parity report contract * fix(qa): compare matched runtime timing captures * fix(qa): type native response delta context * fix(qa): satisfy native codex websocket lint --- .gitignore | 3 + .../codex/src/app-server/config-runtime.ts | 17 +- .../codex/src/app-server/config.test.ts | 86 +++ extensions/codex/src/app-server/config.ts | 1 - .../src/app-server/dynamic-tool-build.test.ts | 12 +- .../src/app-server/dynamic-tool-profile.ts | 8 +- .../app-server/provider-capabilities.test.ts | 11 + .../src/app-server/provider-capabilities.ts | 30 +- .../src/app-server/thread-binding-policy.ts | 4 +- .../src/app-server/thread-lifecycle.test.ts | 38 + extensions/qa-lab/package.json | 1 + .../qa-lab/src/agentic-parity-cache-usage.ts | 108 +++ .../src/agentic-parity-report-test-helpers.ts | 81 +++ .../src/agentic-parity-report.cache.test.ts | 227 ++++++ .../qa-lab/src/agentic-parity-report.test.ts | 169 +++-- .../src/agentic-parity-report.timing.test.ts | 135 ++++ .../qa-lab/src/agentic-parity-report.ts | 146 ++-- .../src/agentic-parity-runtime-markdown.ts | 108 +++ .../agentic-parity-runtime-report-contract.ts | 45 ++ extensions/qa-lab/src/cli.runtime.ts | 3 +- extensions/qa-lab/src/gateway-child.test.ts | 61 ++ extensions/qa-lab/src/gateway-child.ts | 40 +- .../mock-openai/mock-openai-contracts.ts | 7 + .../mock-openai/mock-openai-directives.ts | 12 + .../mock-openai/mock-openai-events.ts | 15 +- .../mock-openai/mock-openai-input.ts | 10 +- .../mock-openai-responses-websocket.test.ts | 375 ++++++++++ .../mock-openai-responses-websocket.ts | 261 +++++++ .../mock-openai/mock-openai-tooling.ts | 52 ++ .../src/providers/mock-openai/server.test.ts | 264 +++++++ .../src/providers/mock-openai/server.ts | 121 +++- .../src/providers/shared/mock-model-config.ts | 40 + .../qa-lab/src/runtime-parity-timing.ts | 148 ++++ extensions/qa-lab/src/runtime-parity.ts | 5 + .../qa-lab/src/runtime-tool-fixture.test.ts | 684 +++++++++++++++++- extensions/qa-lab/src/runtime-tool-fixture.ts | 355 ++++++++- .../qa-lab/src/scenario-catalog.test.ts | 17 + extensions/qa-lab/src/suite-run-standard.ts | 34 +- .../qa-lab/src/suite-runtime-parity-result.ts | 3 + .../qa-lab/src/tool-coverage-report.test.ts | 9 +- pnpm-lock.yaml | 3 + qa/scenarios/runtime/tools/apply-patch.yaml | 29 +- src/agents/agent-tools.schema.test.ts | 21 + src/agents/bash-tools.schemas.ts | 17 +- src/agents/code-mode.test.ts | 25 + src/agents/code-mode.ts | 12 +- .../package-acceptance-workflow.test.ts | 2 + ...n-codex-app-server-attempt-extra.config.ts | 2 + 48 files changed, 3536 insertions(+), 321 deletions(-) create mode 100644 extensions/qa-lab/src/agentic-parity-cache-usage.ts create mode 100644 extensions/qa-lab/src/agentic-parity-report-test-helpers.ts create mode 100644 extensions/qa-lab/src/agentic-parity-report.cache.test.ts create mode 100644 extensions/qa-lab/src/agentic-parity-report.timing.test.ts create mode 100644 extensions/qa-lab/src/agentic-parity-runtime-markdown.ts create mode 100644 extensions/qa-lab/src/agentic-parity-runtime-report-contract.ts create mode 100644 extensions/qa-lab/src/providers/mock-openai/mock-openai-responses-websocket.test.ts create mode 100644 extensions/qa-lab/src/providers/mock-openai/mock-openai-responses-websocket.ts create mode 100644 extensions/qa-lab/src/runtime-parity-timing.ts diff --git a/.gitignore b/.gitignore index 602554d4273e..259b3b2d0420 100644 --- a/.gitignore +++ b/.gitignore @@ -109,6 +109,9 @@ docs/internal/ tmp/ IDENTITY.md USER.md +# Keep packaged workspace templates visible to Git-aware remote syncs. +!docs/reference/templates/IDENTITY.md +!docs/reference/templates/USER.md # Exception: oc-path real-world test fixtures need to be tracked even # though the bare names match the local-untracked rule above. !extensions/oc-path/src/oc-path/tests/fixtures/real/IDENTITY.md diff --git a/extensions/codex/src/app-server/config-runtime.ts b/extensions/codex/src/app-server/config-runtime.ts index 09d9e3c2cc3a..f48d411a758b 100644 --- a/extensions/codex/src/app-server/config-runtime.ts +++ b/extensions/codex/src/app-server/config-runtime.ts @@ -58,6 +58,7 @@ import { readNumberEnv, resolveArgs, } from "./config-utils.js"; +import { isForcedPrivateQaCodexRuntime } from "./dynamic-tool-profile.js"; import type { CodexSandboxPolicy } from "./protocol.js"; export function resolveCodexAppServerRuntimeOptions( @@ -190,11 +191,17 @@ export function resolveCodexAppServerRuntimeOptions( ? normalizedPolicyMode : (explicitPolicyMode ?? normalizedPolicyMode ?? defaultPolicy?.mode ?? "yolo"); const serviceTier = normalizeCodexServiceTier(config.serviceTier); - const resolvedSandbox = + const configuredRuntimeSandbox = forcedPolicy?.sandbox ?? configuredSandbox ?? defaultPolicy?.sandbox ?? (policyMode === "guardian" ? "workspace-write" : "danger-full-access"); + // Private QA may bound production yolo, but must never widen a configured + // read-only sandbox or override the ordinary policy precedence. + const resolvedSandbox = + isForcedPrivateQaCodexRuntime(env) && configuredRuntimeSandbox === "danger-full-access" + ? "workspace-write" + : configuredRuntimeSandbox; if (transport === "websocket" && !url) { throw new Error( "plugins.entries.codex.config.appServer.url is required when appServer.transport is websocket", @@ -490,6 +497,7 @@ export function codexAppServerStartOptionsKey( export function codexSandboxPolicyForTurn( mode: CodexAppServerSandboxMode, cwd: string, + env: NodeJS.ProcessEnv = process.env, ): CodexSandboxPolicy { if (mode === "danger-full-access") { return { type: "dangerFullAccess" }; @@ -497,12 +505,15 @@ export function codexSandboxPolicyForTurn( if (mode === "read-only") { return { type: "readOnly", networkAccess: false }; } + // Codex includes /tmp and TMPDIR in workspace-write by default. Private QA + // workspaces live there, so retaining either root defeats sibling containment. + const excludePrivateQaTempRoots = isForcedPrivateQaCodexRuntime(env); return { type: "workspaceWrite", writableRoots: [cwd], networkAccess: false, - excludeTmpdirEnvVar: false, - excludeSlashTmp: false, + excludeTmpdirEnvVar: excludePrivateQaTempRoots, + excludeSlashTmp: excludePrivateQaTempRoots, }; } diff --git a/extensions/codex/src/app-server/config.test.ts b/extensions/codex/src/app-server/config.test.ts index 32eccc4a052f..7bc0a82d5f1d 100644 --- a/extensions/codex/src/app-server/config.test.ts +++ b/extensions/codex/src/app-server/config.test.ts @@ -7,6 +7,7 @@ import { describe, expect, it, vi } from "vitest"; import { canUseCodexModelBackedApprovalsReviewerForModel, codexAppServerStartOptionsKey, + codexSandboxPolicyForTurn, readCodexPluginConfig, resolveCodexAppServerRuntimeOptions, resolveCodexAppServerStartOptionsForAgent, @@ -549,6 +550,91 @@ describe("Codex app-server config", () => { }); }); + it("confines only explicitly forced private-QA Codex runtime to workspace writes", () => { + const privateQaCodexEnv = { + OPENCLAW_BUILD_PRIVATE_QA: "1", + OPENCLAW_QA_FORCE_RUNTIME: "codex", + }; + const runtime = resolveRuntimeForTest({ + pluginConfig: { + appServer: { + mode: "yolo", + approvalPolicy: "never", + sandbox: "danger-full-access", + }, + }, + env: privateQaCodexEnv, + }); + + expectRuntimePolicy(runtime, { + approvalPolicy: "never", + sandbox: "workspace-write", + approvalsReviewer: "user", + }); + expect(codexSandboxPolicyForTurn(runtime.sandbox, "/qa/workspace", privateQaCodexEnv)).toEqual({ + type: "workspaceWrite", + writableRoots: ["/qa/workspace"], + networkAccess: false, + excludeTmpdirEnvVar: true, + excludeSlashTmp: true, + }); + }); + + it("preserves an explicitly read-only sandbox for forced private-QA Codex runtime", () => { + const privateQaCodexEnv = { + OPENCLAW_BUILD_PRIVATE_QA: "1", + OPENCLAW_QA_FORCE_RUNTIME: "codex", + }; + const runtime = resolveRuntimeForTest({ + pluginConfig: { + appServer: { + mode: "yolo", + approvalPolicy: "never", + sandbox: "read-only", + }, + }, + env: privateQaCodexEnv, + }); + + expectRuntimePolicy(runtime, { + approvalPolicy: "never", + sandbox: "read-only", + approvalsReviewer: "user", + }); + expect(codexSandboxPolicyForTurn(runtime.sandbox, "/qa/workspace", privateQaCodexEnv)).toEqual({ + type: "readOnly", + networkAccess: false, + }); + }); + + it.each([ + { label: "ordinary production", env: {} }, + { label: "private build without a forced runtime", env: { OPENCLAW_BUILD_PRIVATE_QA: "1" } }, + { + label: "forced runtime without a private build", + env: { OPENCLAW_QA_FORCE_RUNTIME: "codex" }, + }, + { + label: "forced private-QA OpenClaw runtime", + env: { OPENCLAW_BUILD_PRIVATE_QA: "1", OPENCLAW_QA_FORCE_RUNTIME: "openclaw" }, + }, + ])("preserves production yolo filesystem policy for $label", ({ env }) => { + const runtime = resolveRuntimeForTest({ pluginConfig: {}, env }); + + expectRuntimePolicy(runtime, { + approvalPolicy: "never", + sandbox: "danger-full-access", + approvalsReviewer: "user", + }); + expect(codexSandboxPolicyForTurn("workspace-write", "/qa/workspace", env)).toEqual({ + type: "workspaceWrite", + writableRoots: ["/qa/workspace"], + networkAccess: false, + excludeTmpdirEnvVar: false, + excludeSlashTmp: false, + }); + }); + it("does not change ordinary harness connection defaults when supervision is enabled", () => { const runtime = resolveRuntimeForTest({ pluginConfig: { supervision: { enabled: true } }, diff --git a/extensions/codex/src/app-server/config.ts b/extensions/codex/src/app-server/config.ts index ace3519d6795..2ee5f10383ac 100644 --- a/extensions/codex/src/app-server/config.ts +++ b/extensions/codex/src/app-server/config.ts @@ -5,7 +5,6 @@ export { } from "./config-contracts.js"; export type { CodexAppServerApprovalPolicy, - CodexAppServerConnectionClass, CodexAppServerHomeScope, CodexAppServerRuntimeOptions, CodexAppServerSandboxMode, diff --git a/extensions/codex/src/app-server/dynamic-tool-build.test.ts b/extensions/codex/src/app-server/dynamic-tool-build.test.ts index 7b850f10d5a0..f59a02678564 100644 --- a/extensions/codex/src/app-server/dynamic-tool-build.test.ts +++ b/extensions/codex/src/app-server/dynamic-tool-build.test.ts @@ -565,9 +565,15 @@ describe("Codex app-server dynamic tool build", () => { }); it("exposes app-server-owned tools directly for forced private QA Codex runtime", () => { - const tools = ["read", "write", "get_goal", "image_generate", "message"].map((name) => ({ - name, - })); + const tools = [ + "read", + "write", + "apply_patch", + "apply-patch", + "get_goal", + "image_generate", + "message", + ].map((name) => ({ name })); const privateQaCodexEnv = { OPENCLAW_BUILD_PRIVATE_QA: "1", OPENCLAW_QA_FORCE_RUNTIME: "codex", diff --git a/extensions/codex/src/app-server/dynamic-tool-profile.ts b/extensions/codex/src/app-server/dynamic-tool-profile.ts index 79d4fba19c73..cf335d236829 100644 --- a/extensions/codex/src/app-server/dynamic-tool-profile.ts +++ b/extensions/codex/src/app-server/dynamic-tool-profile.ts @@ -5,7 +5,7 @@ import type { CodexAppServerConnectionClass, CodexDynamicToolsLoading, CodexPluginConfig, -} from "./config.js"; +} from "./config-contracts.js"; /** Tool names owned by Codex app-server and normally excluded from OpenClaw dynamic tools. */ const CODEX_APP_SERVER_OWNED_DYNAMIC_TOOL_EXCLUDES = [ @@ -142,7 +142,11 @@ function filterCodexDynamicToolsWithOptions( for (const name of CODEX_NATIVE_GOAL_TOOL_EXCLUDES) { excludes.add(name); } - if (!isForcedPrivateQaCodexRuntime(env)) { + if (isForcedPrivateQaCodexRuntime(env)) { + // Native apply_patch is registered first; advertising a second handler + // makes Codex reject the duplicate before either QA patch can execute. + excludes.add("apply_patch"); + } else { for (const name of CODEX_APP_SERVER_OWNED_DYNAMIC_TOOL_EXCLUDES) { if (options.preserveOpenClawShell && CODEX_APP_SERVER_OWNED_SHELL_TOOL_EXCLUDES.has(name)) { continue; diff --git a/extensions/codex/src/app-server/provider-capabilities.test.ts b/extensions/codex/src/app-server/provider-capabilities.test.ts index 2e432c8be9a6..90ea91dcb1bc 100644 --- a/extensions/codex/src/app-server/provider-capabilities.test.ts +++ b/extensions/codex/src/app-server/provider-capabilities.test.ts @@ -156,6 +156,7 @@ describe("resolveCodexProviderWebSearchSupport", () => { const { clientFactory, request } = createClientFactory(false); await expect(resolveSupport(clientFactory, " OpenAI ")).resolves.toBe("supported"); + expect(clientFactory).not.toHaveBeenCalled(); expect(request).not.toHaveBeenCalled(); }); @@ -165,6 +166,16 @@ describe("resolveCodexProviderWebSearchSupport", () => { await expect(resolveSupport(clientFactory, "amazon-bedrock")).resolves.toBe("unsupported"); await expect(resolveSupport(clientFactory, "custom-provider")).resolves.toBe("unsupported"); await expect(resolveSupport(clientFactory, "lmstudio")).resolves.toBe("unsupported"); + expect(clientFactory).not.toHaveBeenCalled(); expect(request).not.toHaveBeenCalled(); }); + + it("does not wait for app-server startup when the override proves hosted support", async () => { + const clientFactory = vi.fn( + async () => await new Promise(() => {}), + ) as unknown as CodexAppServerClientFactory; + + await expect(resolveSupport(clientFactory, "openai")).resolves.toBe("supported"); + expect(clientFactory).not.toHaveBeenCalled(); + }); }); diff --git a/extensions/codex/src/app-server/provider-capabilities.ts b/extensions/codex/src/app-server/provider-capabilities.ts index d5400101071c..720e5638f51e 100644 --- a/extensions/codex/src/app-server/provider-capabilities.ts +++ b/extensions/codex/src/app-server/provider-capabilities.ts @@ -8,6 +8,18 @@ import { } from "./shared-client.js"; import type { CodexNativeWebSearchSupport } from "./web-search.js"; +function resolveOverriddenProviderWebSearchSupport( + modelProviderOverride: string | undefined, +): CodexNativeWebSearchSupport | undefined { + const provider = modelProviderOverride?.trim().toLowerCase(); + if (!provider) { + return undefined; + } + // The capability RPC describes the configured provider, not a thread + // override. OpenAI's hosted support is known; other overrides stay managed. + return provider === "openai" ? "supported" : "unsupported"; +} + async function readConfiguredProviderWebSearchSupport(params: { client: CodexAppServerClient; timeoutMs: number; @@ -30,15 +42,9 @@ export async function resolveCodexProviderWebSearchSupportForClient(params: { modelProviderOverride: string | undefined; signal: AbortSignal; }): Promise { - const modelProviderOverride = params.modelProviderOverride?.trim().toLowerCase(); - if (modelProviderOverride === "openai") { - return "supported"; - } - if (modelProviderOverride) { - // Codex's capability RPC only reports the configured provider, not a - // thread-scoped override. Keep managed search for overrides whose hosted - // capability cannot be proven from the configured-provider response. - return "unsupported"; + const overrideSupport = resolveOverriddenProviderWebSearchSupport(params.modelProviderOverride); + if (overrideSupport) { + return overrideSupport; } try { return await readConfiguredProviderWebSearchSupport(params); @@ -57,6 +63,12 @@ export async function resolveCodexProviderWebSearchSupport(params: { modelProviderOverride: string | undefined; signal: AbortSignal; }): Promise { + const overrideSupport = resolveOverriddenProviderWebSearchSupport(params.modelProviderOverride); + if (overrideSupport) { + // Never serialize the prewarmed app-server startup behind a capability + // probe whose answer is already fixed by the selected thread provider. + return overrideSupport; + } let client: CodexAppServerClient | undefined; try { client = await params.clientFactory({ diff --git a/extensions/codex/src/app-server/thread-binding-policy.ts b/extensions/codex/src/app-server/thread-binding-policy.ts index 6eac4b97187a..55ea2fc696cc 100644 --- a/extensions/codex/src/app-server/thread-binding-policy.ts +++ b/extensions/codex/src/app-server/thread-binding-policy.ts @@ -1,4 +1,4 @@ -import type { CodexAppServerRuntimeOptions } from "./config.js"; +import type { CodexAppServerConnectionClass } from "./config-contracts.js"; import { normalizeCodexDynamicToolName } from "./dynamic-tool-profile.js"; import type { CodexAppServerThreadBinding } from "./session-binding.js"; import type { @@ -7,7 +7,7 @@ import type { } from "./thread-lifecycle-types.js"; export function shouldRotateCodexAppServerBindingForRuntime(params: { - connectionClass: CodexAppServerRuntimeOptions["connectionClass"]; + connectionClass: CodexAppServerConnectionClass; current?: string; binding?: string; }): boolean { diff --git a/extensions/codex/src/app-server/thread-lifecycle.test.ts b/extensions/codex/src/app-server/thread-lifecycle.test.ts index 3249aaa59e74..f5a29a74e242 100644 --- a/extensions/codex/src/app-server/thread-lifecycle.test.ts +++ b/extensions/codex/src/app-server/thread-lifecycle.test.ts @@ -1105,6 +1105,44 @@ describe("Codex app-server native code mode config", () => { }); describe("Codex app-server turn input image sanitizing", () => { + it("excludes implicit temporary writable roots from forced private-QA Codex turns", () => { + vi.stubEnv("OPENCLAW_BUILD_PRIVATE_QA", "1"); + vi.stubEnv("OPENCLAW_QA_FORCE_RUNTIME", "codex"); + try { + const request = buildTurnStartParams(createAttemptParams({ provider: "openai" }), { + threadId: "thread-1", + cwd: "/tmp/qa/workspace", + appServer: createAppServerOptions() as never, + }); + + expect(request.sandboxPolicy).toEqual({ + type: "workspaceWrite", + writableRoots: ["/tmp/qa/workspace"], + networkAccess: false, + excludeTmpdirEnvVar: true, + excludeSlashTmp: true, + }); + } finally { + vi.unstubAllEnvs(); + } + }); + + it("preserves implicit temporary writable roots for ordinary Codex turns", () => { + const request = buildTurnStartParams(createAttemptParams({ provider: "openai" }), { + threadId: "thread-1", + cwd: "/tmp/qa/workspace", + appServer: createAppServerOptions() as never, + }); + + expect(request.sandboxPolicy).toEqual({ + type: "workspaceWrite", + writableRoots: ["/tmp/qa/workspace"], + networkAccess: false, + excludeTmpdirEnvVar: false, + excludeSlashTmp: false, + }); + }); + it("uses an explicit turn sandbox policy override when provided", () => { const request = buildTurnStartParams(createAttemptParams({ provider: "openai" }), { threadId: "thread-1", diff --git a/extensions/qa-lab/package.json b/extensions/qa-lab/package.json index 1b1a763b903b..3f70112185a1 100644 --- a/extensions/qa-lab/package.json +++ b/extensions/qa-lab/package.json @@ -14,6 +14,7 @@ "playwright-core": "1.61.1", "pretty-ms": "9.3.0", "semver": "7.8.5", + "ws": "8.21.1", "yaml": "2.9.0", "zod": "4.4.3" }, diff --git a/extensions/qa-lab/src/agentic-parity-cache-usage.ts b/extensions/qa-lab/src/agentic-parity-cache-usage.ts new file mode 100644 index 000000000000..67cb8b45489c --- /dev/null +++ b/extensions/qa-lab/src/agentic-parity-cache-usage.ts @@ -0,0 +1,108 @@ +import type { RuntimeId, RuntimeParityUsage } from "./runtime-parity.js"; + +export type QaRuntimeParityCacheUsage = { + totalTokens: number; + inputTokens: number; + outputTokens: number; + grossInputTokens: number | null; + uncachedInputTokens: number | null; + cachedInputTokens: number | null; + cacheWriteTokens: number | null; + cacheHitPercent: number | null; +}; + +type QaRuntimeParityCacheScenario = { + openclawUsage: QaRuntimeParityCacheUsage | null; + codexUsage: QaRuntimeParityCacheUsage | null; +}; + +export function summarizeRuntimeParityCacheUsage( + usage: Pick< + RuntimeParityUsage, + "inputTokens" | "outputTokens" | "totalTokens" | "cacheRead" | "cacheWrite" + >, +): QaRuntimeParityCacheUsage { + const cachedInputTokens = usage.cacheRead ?? null; + const cacheWriteTokens = usage.cacheWrite ?? null; + const uncachedInputTokens = + cacheWriteTokens === null ? null : usage.inputTokens + cacheWriteTokens; + const grossInputTokens = + cachedInputTokens === null || uncachedInputTokens === null + ? null + : uncachedInputTokens + cachedInputTokens; + return { + totalTokens: usage.totalTokens, + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + grossInputTokens, + uncachedInputTokens, + cachedInputTokens, + cacheWriteTokens, + cacheHitPercent: + grossInputTokens !== null && grossInputTokens > 0 && cachedInputTokens !== null + ? (cachedInputTokens / grossInputTokens) * 100 + : null, + }; +} + +export function aggregateRuntimeParityCacheUsage( + scenarios: readonly QaRuntimeParityCacheScenario[], + runtime: RuntimeId, +): QaRuntimeParityCacheUsage | null { + const captures = scenarios.flatMap((scenario) => { + const usage = runtime === "openclaw" ? scenario.openclawUsage : scenario.codexUsage; + return usage === null ? [] : [usage]; + }); + if (captures.length === 0) { + return null; + } + const measuredCaptures = captures.filter( + (capture) => capture.cachedInputTokens !== null && capture.cacheWriteTokens !== null, + ); + const inputTokens = captures.reduce((total, capture) => total + capture.inputTokens, 0); + const outputTokens = captures.reduce((total, capture) => total + capture.outputTokens, 0); + const totalTokens = captures.reduce((total, capture) => total + capture.totalTokens, 0); + if (measuredCaptures.length === 0) { + return { + inputTokens, + outputTokens, + totalTokens, + grossInputTokens: null, + uncachedInputTokens: null, + cachedInputTokens: null, + cacheWriteTokens: null, + cacheHitPercent: null, + }; + } + const cachedInputTokens = measuredCaptures.reduce( + (total, capture) => total + (capture.cachedInputTokens ?? 0), + 0, + ); + const cacheWriteTokens = measuredCaptures.reduce( + (total, capture) => total + (capture.cacheWriteTokens ?? 0), + 0, + ); + const uncachedInputTokens = measuredCaptures.reduce( + (total, capture) => total + capture.inputTokens + (capture.cacheWriteTokens ?? 0), + 0, + ); + const grossInputTokens = uncachedInputTokens + cachedInputTokens; + return { + inputTokens, + outputTokens, + totalTokens, + grossInputTokens, + uncachedInputTokens, + cachedInputTokens, + cacheWriteTokens, + cacheHitPercent: grossInputTokens > 0 ? (cachedInputTokens / grossInputTokens) * 100 : null, + }; +} + +export function formatRuntimeCacheHitPercent(value: number | null | undefined): string { + return value === null || value === undefined ? "N/A" : `${value.toFixed(1)}%`; +} + +export function formatRuntimeCacheCount(value: number | null | undefined): string { + return value === null || value === undefined ? "N/A" : String(value); +} diff --git a/extensions/qa-lab/src/agentic-parity-report-test-helpers.ts b/extensions/qa-lab/src/agentic-parity-report-test-helpers.ts new file mode 100644 index 000000000000..3444af502009 --- /dev/null +++ b/extensions/qa-lab/src/agentic-parity-report-test-helpers.ts @@ -0,0 +1,81 @@ +import type { QaRuntimeParitySuiteSummary } from "./agentic-parity-report.js"; + +export function makeRuntimeParitySummary(): QaRuntimeParitySuiteSummary { + return { + scenarios: [ + { + name: "Approval turn tool followthrough", + status: "pass", + steps: [], + runtimeParity: { + scenarioId: "approval-turn-tool-followthrough", + drift: "none", + cells: { + openclaw: { + runtime: "openclaw", + status: "pass", + transcriptBytes: '{"role":"assistant"}\n', + toolCalls: [{ tool: "read_file", argsHash: "a", resultHash: "r" }], + finalText: "done", + usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + wallClockMs: 20, + bootStateLines: [], + }, + codex: { + runtime: "codex", + status: "pass", + transcriptBytes: '{"role":"assistant"}\n', + toolCalls: [{ tool: "read_file", argsHash: "a", resultHash: "r" }], + finalText: "done", + usage: { inputTokens: 8, outputTokens: 4, totalTokens: 12 }, + wallClockMs: 18, + bootStateLines: [], + }, + }, + }, + }, + { + name: "Compaction retry after mutating tool", + status: "pass", + steps: [], + runtimeParity: { + scenarioId: "compaction-retry-after-mutating-tool", + drift: "tool-call-shape", + driftDetails: "tool call 1 differs", + cells: { + openclaw: { + runtime: "openclaw", + status: "pass", + transcriptBytes: '{"role":"assistant"}\n', + toolCalls: [{ tool: "read_file", argsHash: "a", resultHash: "r" }], + finalText: "done", + usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + wallClockMs: 20, + bootStateLines: [], + }, + codex: { + runtime: "codex", + status: "pass", + transcriptBytes: '{"role":"assistant"}\n', + toolCalls: [{ tool: "read_file", argsHash: "b", resultHash: "r" }], + finalText: "done", + usage: { inputTokens: 9, outputTokens: 4, totalTokens: 13 }, + wallClockMs: 19, + bootStateLines: [], + }, + }, + }, + }, + ], + counts: { + total: 2, + passed: 2, + failed: 0, + }, + run: { + providerMode: "mock-openai", + primaryModel: "openai/gpt-5.6-luna", + runtimePair: ["openclaw", "codex"], + }, + }; +} diff --git a/extensions/qa-lab/src/agentic-parity-report.cache.test.ts b/extensions/qa-lab/src/agentic-parity-report.cache.test.ts new file mode 100644 index 000000000000..ed9b9c179bcd --- /dev/null +++ b/extensions/qa-lab/src/agentic-parity-report.cache.test.ts @@ -0,0 +1,227 @@ +import { describe, expect, it } from "vitest"; +import { makeRuntimeParitySummary } from "./agentic-parity-report-test-helpers.js"; +import { + buildQaRuntimeParityReport, + renderQaRuntimeParityMarkdownReport, +} from "./agentic-parity-report.js"; + +function makeMeasuredRuntimeParitySummary() { + const summary = makeRuntimeParitySummary(); + for (const scenario of summary.scenarios) { + if (scenario.runtimeParity) { + for (const cell of Object.values(scenario.runtimeParity.cells)) { + cell.usage = { ...cell.usage, cacheRead: 0, cacheWrite: 0 }; + } + } + } + return summary; +} + +describe("qa runtime parity prompt-cache reporting", () => { + it("reports cached, uncached, and cache-write input without counting output as cacheable", () => { + const summary = makeMeasuredRuntimeParitySummary(); + const scenario = summary.scenarios[0]; + if (!scenario?.runtimeParity) { + throw new Error("runtime parity fixture missing"); + } + scenario.runtimeParity.cells.openclaw.usage = { + inputTokens: 10, + outputTokens: 5, + totalTokens: 85, + cacheRead: 60, + cacheWrite: 10, + }; + scenario.runtimeParity.cells.codex.usage = { + inputTokens: 8, + outputTokens: 4, + totalTokens: 20, + cacheRead: 8, + cacheWrite: 0, + }; + + const report = buildQaRuntimeParityReport({ summary }); + + expect(report.scenarios[0]?.openclawUsage).toEqual({ + totalTokens: 85, + inputTokens: 10, + outputTokens: 5, + grossInputTokens: 80, + uncachedInputTokens: 20, + cachedInputTokens: 60, + cacheWriteTokens: 10, + cacheHitPercent: 75, + }); + expect(report.scenarios[0]?.codexUsage).toEqual({ + totalTokens: 20, + inputTokens: 8, + outputTokens: 4, + grossInputTokens: 16, + uncachedInputTokens: 8, + cachedInputTokens: 8, + cacheWriteTokens: 0, + cacheHitPercent: 50, + }); + expect(report.usage.openclaw).toMatchObject({ + totalTokens: 100, + grossInputTokens: 90, + uncachedInputTokens: 30, + cachedInputTokens: 60, + cacheWriteTokens: 10, + }); + expect(report.usage.openclaw?.cacheHitPercent).toBeCloseTo((60 / 90) * 100); + expect(report.usage.codex).toMatchObject({ + totalTokens: 33, + grossInputTokens: 25, + uncachedInputTokens: 17, + cachedInputTokens: 8, + cacheWriteTokens: 0, + cacheHitPercent: 32, + }); + const markdown = renderQaRuntimeParityMarkdownReport(report); + expect(markdown).toContain("## Prompt Cache"); + expect(markdown).toContain("| openclaw | 90 | 30 | 60 | 10 | 10 | 100 | 66.7% |"); + expect(markdown).toContain("| codex | 25 | 17 | 8 | 0 | 8 | 33 | 32.0% |"); + expect(markdown).toContain( + "prompt cache: openclaw 75.0% (60 cached, 20 uncached input); codex 50.0% (8 cached, 8 uncached input)", + ); + }); + + it("computes aggregate cache hits from measured input rather than averaging percentages", () => { + const summary = makeMeasuredRuntimeParitySummary(); + const [first, second] = summary.scenarios; + if (!first?.runtimeParity || !second?.runtimeParity) { + throw new Error("runtime parity fixtures missing"); + } + first.runtimeParity.cells.openclaw.usage = { + inputTokens: 10, + outputTokens: 5, + totalTokens: 85, + cacheRead: 60, + cacheWrite: 10, + }; + second.runtimeParity.cells.openclaw.usage = { + inputTokens: 10, + outputTokens: 5, + totalTokens: 115, + cacheRead: 100, + cacheWrite: 0, + }; + + const report = buildQaRuntimeParityReport({ summary }); + expect(report.usage.openclaw).toMatchObject({ + totalTokens: 200, + grossInputTokens: 190, + uncachedInputTokens: 30, + cachedInputTokens: 160, + cacheWriteTokens: 10, + }); + expect(report.usage.openclaw?.cacheHitPercent).toBeCloseTo((160 / 190) * 100); + }); + + it("reports unavailable runtime captures as N/A instead of fabricated zero-token usage", () => { + const report = buildQaRuntimeParityReport({ + summary: { + scenarios: [{ name: "Missing runtime capture", status: "fail" }], + counts: { total: 1, passed: 0, failed: 1 }, + run: { providerMode: "live-frontier", runtimePair: ["openclaw", "codex"] }, + }, + }); + expect(report.usage).toEqual({ openclaw: null, codex: null }); + expect(report.scenarios[0]).toMatchObject({ openclawUsage: null, codexUsage: null }); + const markdown = renderQaRuntimeParityMarkdownReport(report); + expect(markdown).toContain("| openclaw | N/A | N/A | N/A | N/A | N/A | N/A | N/A |"); + expect(markdown).toContain("| codex | N/A | N/A | N/A | N/A | N/A | N/A | N/A |"); + }); + + it("keeps missing cache telemetry unknown while preserving measured token totals", () => { + const report = buildQaRuntimeParityReport({ summary: makeRuntimeParitySummary() }); + expect(report.scenarios[0]?.openclawUsage).toMatchObject({ + inputTokens: 10, + outputTokens: 5, + totalTokens: 15, + grossInputTokens: null, + uncachedInputTokens: null, + cachedInputTokens: null, + cacheWriteTokens: null, + cacheHitPercent: null, + }); + expect(report.usage.openclaw).toMatchObject({ + inputTokens: 20, + outputTokens: 10, + totalTokens: 30, + grossInputTokens: null, + uncachedInputTokens: null, + cachedInputTokens: null, + cacheWriteTokens: null, + cacheHitPercent: null, + }); + expect(renderQaRuntimeParityMarkdownReport(report)).toContain( + "| openclaw | N/A | N/A | N/A | N/A | 10 | 30 | N/A |", + ); + }); + + it("excludes unknown cache captures from the weighted cache-hit denominator", () => { + const summary = makeRuntimeParitySummary(); + const first = summary.scenarios[0]; + if (!first?.runtimeParity) { + throw new Error("runtime parity fixture missing"); + } + first.runtimeParity.cells.openclaw.usage = { + inputTokens: 10, + outputTokens: 5, + totalTokens: 85, + cacheRead: 60, + cacheWrite: 10, + }; + const report = buildQaRuntimeParityReport({ summary }); + expect(report.usage.openclaw).toMatchObject({ + inputTokens: 20, + outputTokens: 10, + totalTokens: 100, + grossInputTokens: 80, + uncachedInputTokens: 20, + cachedInputTokens: 60, + cacheWriteTokens: 10, + cacheHitPercent: 75, + }); + }); + + it("distinguishes a measured zero cache hit from missing cache telemetry", () => { + const report = buildQaRuntimeParityReport({ summary: makeMeasuredRuntimeParitySummary() }); + expect(report.usage.openclaw).toMatchObject({ + inputTokens: 20, + grossInputTokens: 20, + uncachedInputTokens: 20, + cachedInputTokens: 0, + cacheWriteTokens: 0, + cacheHitPercent: 0, + }); + expect(renderQaRuntimeParityMarkdownReport(report)).toContain( + "| openclaw | 20 | 20 | 0 | 0 | 10 | 30 | 0.0% |", + ); + }); + + it("reports measured zero-input cache hits as unavailable rather than infinity", () => { + const summary = makeMeasuredRuntimeParitySummary(); + for (const scenario of summary.scenarios) { + if (scenario.runtimeParity) { + scenario.runtimeParity.cells.openclaw.usage = { + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + cacheRead: 0, + cacheWrite: 0, + }; + } + } + const report = buildQaRuntimeParityReport({ summary }); + expect(report.usage.openclaw).toMatchObject({ + grossInputTokens: 0, + cachedInputTokens: 0, + cacheHitPercent: null, + }); + expect(renderQaRuntimeParityMarkdownReport(report)).toContain( + "| openclaw | 0 | 0 | 0 | 0 | 0 | 0 | N/A |", + ); + }); +}); diff --git a/extensions/qa-lab/src/agentic-parity-report.test.ts b/extensions/qa-lab/src/agentic-parity-report.test.ts index 666091ff542c..bfe63cfc98d4 100644 --- a/extensions/qa-lab/src/agentic-parity-report.test.ts +++ b/extensions/qa-lab/src/agentic-parity-report.test.ts @@ -1,5 +1,6 @@ // Qa Lab tests cover agentic parity report plugin behavior. import { describe, expect, it } from "vitest"; +import { makeRuntimeParitySummary } from "./agentic-parity-report-test-helpers.js"; import { buildQaAgenticParityComparison, buildQaRuntimeParityReport, @@ -42,86 +43,6 @@ function withScenarioOverride(name: string, override: Partial { + const summary = makeRuntimeParitySummary(); + for (const scenario of summary.scenarios) { + if (scenario.runtimeParity) { + scenario.runtimeParity.cells.openclaw.wallClockMs = 0; + scenario.runtimeParity.cells.codex.wallClockMs = 0; + } + } + + const report = buildQaRuntimeParityReport({ summary }); + + expect(report.pass).toBe(true); + expect(report.timing).toEqual({ + openclaw: { totalWallClockMs: 0, p50WallClockMs: 0, p90WallClockMs: 0 }, + codex: { totalWallClockMs: 0, p50WallClockMs: 0, p90WallClockMs: 0 }, + fasterRuntime: "tie", + speedupPercent: 0, + }); + expect(report.scenarios[0]).toMatchObject({ + fasterRuntime: "tie", + speedupPercent: 0, + }); + }); + + it("excludes missing runtime captures from wall-clock aggregates", () => { + const summary = makeRuntimeParitySummary(); + summary.scenarios.push({ name: "Missing runtime capture", status: "fail" }); + + const report = buildQaRuntimeParityReport({ summary }); + + expect(report.pass).toBe(false); + expect(report.timing.openclaw.totalWallClockMs).toBe(40); + expect(report.timing.codex.totalWallClockMs).toBe(37); + expect(report.scenarios[2]).toMatchObject({ + openclawWallClockMs: null, + codexWallClockMs: null, + fasterRuntime: null, + speedupPercent: null, + }); + }); + + it("reports missing runtime timing as unavailable instead of zero", () => { + const report = buildQaRuntimeParityReport({ + summary: { + scenarios: [{ name: "Missing runtime capture", status: "fail" }], + counts: { total: 1, passed: 0, failed: 1 }, + run: { providerMode: "live-frontier", runtimePair: ["openclaw", "codex"] }, + }, + }); + + expect(report.timing).toEqual({ + openclaw: { totalWallClockMs: null, p50WallClockMs: null, p90WallClockMs: null }, + codex: { totalWallClockMs: null, p50WallClockMs: null, p90WallClockMs: null }, + fasterRuntime: null, + speedupPercent: null, + }); + + const markdown = renderQaRuntimeParityMarkdownReport(report); + expect(markdown).toContain("| openclaw | N/A | N/A | N/A |"); + expect(markdown).toContain("| codex | N/A | N/A | N/A |"); + expect(markdown).toContain("- Faster runtime: N/A"); }); it("fails runtime parity reports when a runtime cell has a hard failure", () => { @@ -1070,7 +1074,12 @@ status=done`, expect(report).toContain("# OpenClaw Runtime Parity Report — openclaw vs codex"); expect(report).toContain("| Tool-call-shape drift | 1 |"); + expect(report).toContain("## Runtime Timing"); + expect(report).toContain("| openclaw | 40 ms | 20 ms | 20 ms |"); + expect(report).toContain("| codex | 37 ms | 18 ms | 19 ms |"); + expect(report).toContain("- Faster runtime: codex 8.1% faster"); expect(report).toContain("### Compaction retry after mutating tool"); expect(report).toContain("- drift: tool-call-shape"); + expect(report).toContain("- wall time: openclaw 20 ms; codex 19 ms; codex 5.3% faster"); }); }); diff --git a/extensions/qa-lab/src/agentic-parity-report.timing.test.ts b/extensions/qa-lab/src/agentic-parity-report.timing.test.ts new file mode 100644 index 000000000000..a34420333229 --- /dev/null +++ b/extensions/qa-lab/src/agentic-parity-report.timing.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from "vitest"; +import { makeRuntimeParitySummary } from "./agentic-parity-report-test-helpers.js"; +import { + buildQaRuntimeParityReport, + renderQaRuntimeParityMarkdownReport, +} from "./agentic-parity-report.js"; +import { + measureRuntimeParityCellTiming, + summarizeRuntimeParityTiming, +} from "./runtime-parity-timing.js"; + +describe("qa runtime parity timing reporting", () => { + it("excludes gateway bootstrap from measured runtime execution", () => { + expect( + measureRuntimeParityCellTiming({ + suiteStartedAt: new Date("2026-07-27T12:00:00.000Z"), + scenarioStartedAt: new Date("2026-07-27T12:00:12.000Z"), + scenarioFinishedAt: new Date("2026-07-27T12:00:15.500Z"), + }), + ).toEqual({ wallClockMs: 3_500, bootstrapWallClockMs: 12_000 }); + }); + + it("keeps minimum turn timing without inventing negative bootstrap", () => { + const startedAt = new Date("2026-07-27T12:00:00.000Z"); + expect( + measureRuntimeParityCellTiming({ + suiteStartedAt: startedAt, + scenarioStartedAt: startedAt, + scenarioFinishedAt: startedAt, + }), + ).toEqual({ wallClockMs: 1, bootstrapWallClockMs: 0 }); + }); + + it("excludes failed attempts and retry backoff from both turn and bootstrap timing", () => { + expect( + measureRuntimeParityCellTiming({ + suiteStartedAt: new Date("2026-07-27T12:00:00.000Z"), + bootstrapFinishedAt: new Date("2026-07-27T12:00:12.000Z"), + scenarioStartedAt: new Date("2026-07-27T12:00:20.000Z"), + scenarioFinishedAt: new Date("2026-07-27T12:00:22.500Z"), + }), + ).toEqual({ wallClockMs: 2_500, bootstrapWallClockMs: 12_000 }); + }); + + it("reports gateway bootstrap separately without changing runtime comparisons", () => { + const summary = makeRuntimeParitySummary(); + for (const scenario of summary.scenarios) { + if (scenario.runtimeParity) { + scenario.runtimeParity.cells.openclaw.bootstrapWallClockMs = 4_000; + scenario.runtimeParity.cells.codex.bootstrapWallClockMs = 12_000; + } + } + + const report = buildQaRuntimeParityReport({ summary }); + + expect(report.pass).toBe(true); + expect(report.timing.openclaw.totalWallClockMs).toBe(40); + expect(report.timing.codex.totalWallClockMs).toBe(37); + expect(report.timing.bootstrap).toEqual({ + openclaw: { totalWallClockMs: 8_000, p50WallClockMs: 4_000, p90WallClockMs: 4_000 }, + codex: { totalWallClockMs: 24_000, p50WallClockMs: 12_000, p90WallClockMs: 12_000 }, + }); + expect(report.scenarios[0]).toMatchObject({ + openclawWallClockMs: 20, + codexWallClockMs: 18, + openclawBootstrapWallClockMs: 4_000, + codexBootstrapWallClockMs: 12_000, + fasterRuntime: "codex", + }); + const markdown = renderQaRuntimeParityMarkdownReport(report); + expect(markdown).toContain("## Gateway Bootstrap (Excluded From Runtime Timing)"); + expect(markdown).toContain("| openclaw | 8000 ms | 4000 ms | 4000 ms |"); + expect(markdown).toContain("| codex | 24000 ms | 12000 ms | 12000 ms |"); + expect(markdown).toContain("- gateway bootstrap (excluded): openclaw 4000 ms; codex 12000 ms"); + }); + + it("reports when OpenClaw is faster without changing the parity verdict", () => { + const summary = makeRuntimeParitySummary(); + for (const scenario of summary.scenarios) { + if (scenario.runtimeParity) { + scenario.runtimeParity.cells.codex.wallClockMs = 30; + } + } + const report = buildQaRuntimeParityReport({ summary }); + expect(report.pass).toBe(true); + expect(report.timing.fasterRuntime).toBe("openclaw"); + expect(report.timing.speedupPercent).toBeCloseTo(50); + expect(report.scenarios[0]).toMatchObject({ + openclawWallClockMs: 20, + codexWallClockMs: 30, + fasterRuntime: "openclaw", + }); + expect(report.scenarios[0]?.speedupPercent).toBeCloseTo(50); + }); + + it("does not report an infinite speedup for a zero-duration runtime", () => { + const summary = makeRuntimeParitySummary(); + for (const scenario of summary.scenarios) { + if (scenario.runtimeParity) { + scenario.runtimeParity.cells.openclaw.wallClockMs = 0; + } + } + const report = buildQaRuntimeParityReport({ summary }); + expect(report.timing.fasterRuntime).toBe("openclaw"); + expect(report.timing.speedupPercent).toBeNull(); + expect(report.scenarios[0]).toMatchObject({ + fasterRuntime: "openclaw", + speedupPercent: null, + }); + }); + + it("compares only paired captures while retaining independently measured totals", () => { + const timing = summarizeRuntimeParityTiming([ + { openclawWallClockMs: 20, codexWallClockMs: 30 }, + { openclawWallClockMs: 1_000, codexWallClockMs: null }, + ]); + + expect(timing.openclaw.totalWallClockMs).toBe(1_020); + expect(timing.codex.totalWallClockMs).toBe(30); + expect(timing.fasterRuntime).toBe("openclaw"); + expect(timing.speedupPercent).toBeCloseTo(50); + }); + + it("does not compare independently measured totals without a complete pair", () => { + const timing = summarizeRuntimeParityTiming([ + { openclawWallClockMs: 20, codexWallClockMs: null }, + { openclawWallClockMs: null, codexWallClockMs: 30 }, + ]); + + expect(timing.openclaw.totalWallClockMs).toBe(20); + expect(timing.codex.totalWallClockMs).toBe(30); + expect(timing.fasterRuntime).toBeNull(); + expect(timing.speedupPercent).toBeNull(); + }); +}); diff --git a/extensions/qa-lab/src/agentic-parity-report.ts b/extensions/qa-lab/src/agentic-parity-report.ts index 0bd050b4f70e..325359f1e7fa 100644 --- a/extensions/qa-lab/src/agentic-parity-report.ts +++ b/extensions/qa-lab/src/agentic-parity-report.ts @@ -1,20 +1,29 @@ +import { + aggregateRuntimeParityCacheUsage, + summarizeRuntimeParityCacheUsage, +} from "./agentic-parity-cache-usage.js"; +import type { + QaRuntimeParityReport, + QaRuntimeParityScenarioReport, +} from "./agentic-parity-runtime-report-contract.js"; // Qa Lab plugin module implements agentic parity report behavior. import { QA_AGENTIC_PARITY_SCENARIO_TITLES, QA_AGENTIC_PARITY_TOOL_BACKED_SCENARIO_TITLES, } from "./agentic-parity.js"; -import type { - RuntimeId, - RuntimeParityDrift, - RuntimeParityResult, - RuntimeParityUsagePolicy, -} from "./runtime-parity.js"; +import { + compareRuntimeWallClockMs, + summarizeRuntimeParityTiming, +} from "./runtime-parity-timing.js"; +import type { RuntimeId, RuntimeParityDrift, RuntimeParityResult } from "./runtime-parity.js"; import { isRuntimeParityResultPass, resolveRuntimeParityUsagePolicy, runtimeParityCellStatus, } from "./runtime-parity.js"; +export { renderQaRuntimeParityMarkdownReport } from "./agentic-parity-runtime-markdown.js"; + type QaParityReportStep = { name: string; status: "pass" | "fail" | "skip"; @@ -63,35 +72,6 @@ export type QaRuntimeParitySuiteSummary = Omit; - scenarios: QaRuntimeParityScenarioReport[]; - pass: boolean; - failures: string[]; - notes: string[]; -}; - type QaAgenticParityMetrics = { totalScenarios: number; passedScenarios: number; @@ -666,8 +646,14 @@ export function buildQaRuntimeParityReport(params: { codexStatus: "missing", openclawTokens: 0, codexTokens: 0, + openclawUsage: null, + codexUsage: null, openclawToolCalls: 0, codexToolCalls: 0, + openclawWallClockMs: null, + codexWallClockMs: null, + fasterRuntime: null, + speedupPercent: null, } satisfies QaRuntimeParityScenarioReport; } driftCounts[parity.drift] += 1; @@ -687,8 +673,25 @@ export function buildQaRuntimeParityReport(params: { codexStatus, openclawTokens: openclawCell.usage.totalTokens, codexTokens: codexCell.usage.totalTokens, + openclawUsage: + runtimeParityUsage.expectation === "not-applicable" + ? null + : summarizeRuntimeParityCacheUsage(openclawCell.usage), + codexUsage: + runtimeParityUsage.expectation === "not-applicable" + ? null + : summarizeRuntimeParityCacheUsage(codexCell.usage), openclawToolCalls: openclawCell.toolCalls.length, codexToolCalls: codexCell.toolCalls.length, + openclawWallClockMs: openclawCell.wallClockMs, + codexWallClockMs: codexCell.wallClockMs, + ...(openclawCell.bootstrapWallClockMs === undefined + ? {} + : { openclawBootstrapWallClockMs: openclawCell.bootstrapWallClockMs }), + ...(codexCell.bootstrapWallClockMs === undefined + ? {} + : { codexBootstrapWallClockMs: codexCell.bootstrapWallClockMs }), + ...compareRuntimeWallClockMs(openclawCell.wallClockMs, codexCell.wallClockMs), } satisfies QaRuntimeParityScenarioReport; if (parityStatus === "fail") { failures.push( @@ -712,7 +715,6 @@ export function buildQaRuntimeParityReport(params: { if (scenarios.length === 0 || totalScenarios <= 0) { failures.push("Runtime parity report has no executed scenarios."); } - return { runtimePair, comparedAt: params.comparedAt ?? new Date().toISOString(), @@ -723,76 +725,18 @@ export function buildQaRuntimeParityReport(params: { failedScenarios, driftCounts, scenarios, + timing: summarizeRuntimeParityTiming(scenarios), + usage: { + openclaw: aggregateRuntimeParityCacheUsage(scenarios, "openclaw"), + codex: aggregateRuntimeParityCacheUsage(scenarios, "codex"), + }, pass: failures.length === 0 && failedScenarios === 0, failures, notes: [ "Runtime parity fails runtime, transport, and failure-mode drift; structural and tool-shape drift is recorded as advisory when both runtimes complete.", "Token totals here are assistant-message usage captured from the normalized transcript, not provider transport payloads.", + "Cache-hit percentages use cached input divided by cached, uncached, and cache-write input; output tokens are excluded from the denominator.", + "Wall-clock timings cover each complete QA runtime cell, including gateway, model, and tool execution; they are not provider-reported turn durations.", ], }; } - -export function renderQaRuntimeParityMarkdownReport(report: QaRuntimeParityReport): string { - const lines = [ - `# OpenClaw Runtime Parity Report — ${report.runtimePair[0]} vs ${report.runtimePair[1]}`, - "", - `- Compared at: ${report.comparedAt}`, - `- Provider mode: ${report.providerMode ?? "unknown"}`, - `- Primary model: ${report.primaryModel ?? "unknown"}`, - `- Verdict: ${report.pass ? "pass" : "fail"}`, - "", - "## Aggregate Metrics", - "", - "| Metric | Value |", - "| --- | ---: |", - `| Total scenarios | ${report.totalScenarios} |`, - `| Passed scenarios | ${report.passedScenarios} |`, - `| Failed scenarios | ${report.failedScenarios} |`, - `| No drift | ${report.driftCounts.none} |`, - `| Text-only drift | ${report.driftCounts["text-only"]} |`, - `| Tool-call-shape drift | ${report.driftCounts["tool-call-shape"]} |`, - `| Tool-result-shape drift | ${report.driftCounts["tool-result-shape"]} |`, - `| Structural drift | ${report.driftCounts.structural} |`, - `| Failure-mode drift | ${report.driftCounts["failure-mode"]} |`, - "", - ]; - - if (report.failures.length > 0) { - lines.push("## Gate Failures", ""); - for (const failure of report.failures) { - lines.push(`- ${failure}`); - } - lines.push(""); - } - - lines.push("## Scenario Comparison", ""); - for (const scenario of report.scenarios) { - const usageNotApplicable = scenario.runtimeParityUsage.expectation === "not-applicable"; - const openclawTokens = usageNotApplicable ? "N/A" : String(scenario.openclawTokens); - const codexTokens = usageNotApplicable ? "N/A" : String(scenario.codexTokens); - lines.push(`### ${scenario.name}`, ""); - lines.push(`- status: ${scenario.status}`); - lines.push(`- drift: ${scenario.drift}`); - lines.push( - `- openclaw: ${scenario.openclawStatus} (${scenario.openclawToolCalls} tool calls, ${openclawTokens} tokens)`, - ); - lines.push( - `- codex: ${scenario.codexStatus} (${scenario.codexToolCalls} tool calls, ${codexTokens} tokens)`, - ); - if (scenario.runtimeParityUsage.expectation === "not-applicable") { - lines.push(`- assistant-message usage: N/A (${scenario.runtimeParityUsage.reason})`); - } - if (scenario.driftDetails) { - lines.push(`- details: ${scenario.driftDetails}`); - } - lines.push(""); - } - - lines.push("## Notes", ""); - for (const note of report.notes) { - lines.push(`- ${note}`); - } - lines.push(""); - - return lines.join("\n"); -} diff --git a/extensions/qa-lab/src/agentic-parity-runtime-markdown.ts b/extensions/qa-lab/src/agentic-parity-runtime-markdown.ts new file mode 100644 index 000000000000..48d7f4e2648d --- /dev/null +++ b/extensions/qa-lab/src/agentic-parity-runtime-markdown.ts @@ -0,0 +1,108 @@ +import { + formatRuntimeCacheCount, + formatRuntimeCacheHitPercent, +} from "./agentic-parity-cache-usage.js"; +import type { QaRuntimeParityReport } from "./agentic-parity-runtime-report-contract.js"; +import { formatRuntimeSpeedComparison, formatRuntimeWallClockMs } from "./runtime-parity-timing.js"; + +export function renderQaRuntimeParityMarkdownReport(report: QaRuntimeParityReport): string { + const lines = [ + `# OpenClaw Runtime Parity Report — ${report.runtimePair[0]} vs ${report.runtimePair[1]}`, + "", + `- Compared at: ${report.comparedAt}`, + `- Provider mode: ${report.providerMode ?? "unknown"}`, + `- Primary model: ${report.primaryModel ?? "unknown"}`, + `- Verdict: ${report.pass ? "pass" : "fail"}`, + "", + "## Aggregate Metrics", + "", + "| Metric | Value |", + "| --- | ---: |", + `| Total scenarios | ${report.totalScenarios} |`, + `| Passed scenarios | ${report.passedScenarios} |`, + `| Failed scenarios | ${report.failedScenarios} |`, + `| No drift | ${report.driftCounts.none} |`, + `| Text-only drift | ${report.driftCounts["text-only"]} |`, + `| Tool-call-shape drift | ${report.driftCounts["tool-call-shape"]} |`, + `| Tool-result-shape drift | ${report.driftCounts["tool-result-shape"]} |`, + `| Structural drift | ${report.driftCounts.structural} |`, + `| Failure-mode drift | ${report.driftCounts["failure-mode"]} |`, + "", + "## Prompt Cache", + "", + "| Runtime | Gross input | Uncached input | Cached input | Cache writes | Output | Total tokens | Cache hit |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", + `| openclaw | ${formatRuntimeCacheCount(report.usage.openclaw?.grossInputTokens)} | ${formatRuntimeCacheCount(report.usage.openclaw?.uncachedInputTokens)} | ${formatRuntimeCacheCount(report.usage.openclaw?.cachedInputTokens)} | ${formatRuntimeCacheCount(report.usage.openclaw?.cacheWriteTokens)} | ${formatRuntimeCacheCount(report.usage.openclaw?.outputTokens)} | ${formatRuntimeCacheCount(report.usage.openclaw?.totalTokens)} | ${formatRuntimeCacheHitPercent(report.usage.openclaw?.cacheHitPercent)} |`, + `| codex | ${formatRuntimeCacheCount(report.usage.codex?.grossInputTokens)} | ${formatRuntimeCacheCount(report.usage.codex?.uncachedInputTokens)} | ${formatRuntimeCacheCount(report.usage.codex?.cachedInputTokens)} | ${formatRuntimeCacheCount(report.usage.codex?.cacheWriteTokens)} | ${formatRuntimeCacheCount(report.usage.codex?.outputTokens)} | ${formatRuntimeCacheCount(report.usage.codex?.totalTokens)} | ${formatRuntimeCacheHitPercent(report.usage.codex?.cacheHitPercent)} |`, + "", + "## Runtime Timing", + "", + "| Runtime | Total wall time | p50 per scenario | p90 per scenario |", + "| --- | ---: | ---: | ---: |", + `| openclaw | ${formatRuntimeWallClockMs(report.timing.openclaw.totalWallClockMs)} | ${formatRuntimeWallClockMs(report.timing.openclaw.p50WallClockMs)} | ${formatRuntimeWallClockMs(report.timing.openclaw.p90WallClockMs)} |`, + `| codex | ${formatRuntimeWallClockMs(report.timing.codex.totalWallClockMs)} | ${formatRuntimeWallClockMs(report.timing.codex.p50WallClockMs)} | ${formatRuntimeWallClockMs(report.timing.codex.p90WallClockMs)} |`, + "", + `- Faster runtime: ${formatRuntimeSpeedComparison(report.timing)}`, + "", + ]; + if (report.timing.bootstrap) { + lines.push( + "## Gateway Bootstrap (Excluded From Runtime Timing)", + "", + "| Runtime | Total bootstrap | p50 per scenario | p90 per scenario |", + "| --- | ---: | ---: | ---: |", + `| openclaw | ${formatRuntimeWallClockMs(report.timing.bootstrap.openclaw.totalWallClockMs)} | ${formatRuntimeWallClockMs(report.timing.bootstrap.openclaw.p50WallClockMs)} | ${formatRuntimeWallClockMs(report.timing.bootstrap.openclaw.p90WallClockMs)} |`, + `| codex | ${formatRuntimeWallClockMs(report.timing.bootstrap.codex.totalWallClockMs)} | ${formatRuntimeWallClockMs(report.timing.bootstrap.codex.p50WallClockMs)} | ${formatRuntimeWallClockMs(report.timing.bootstrap.codex.p90WallClockMs)} |`, + "", + ); + } + if (report.failures.length > 0) { + lines.push("## Gate Failures", ""); + for (const failure of report.failures) { + lines.push(`- ${failure}`); + } + lines.push(""); + } + lines.push("## Scenario Comparison", ""); + for (const scenario of report.scenarios) { + const usageNotApplicable = scenario.runtimeParityUsage.expectation === "not-applicable"; + const openclawTokens = usageNotApplicable ? "N/A" : String(scenario.openclawTokens); + const codexTokens = usageNotApplicable ? "N/A" : String(scenario.codexTokens); + lines.push(`### ${scenario.name}`, ""); + lines.push(`- status: ${scenario.status}`); + lines.push(`- drift: ${scenario.drift}`); + lines.push( + `- openclaw: ${scenario.openclawStatus} (${scenario.openclawToolCalls} tool calls, ${openclawTokens} tokens)`, + ); + lines.push( + `- codex: ${scenario.codexStatus} (${scenario.codexToolCalls} tool calls, ${codexTokens} tokens)`, + ); + lines.push( + `- wall time: openclaw ${formatRuntimeWallClockMs(scenario.openclawWallClockMs)}; codex ${formatRuntimeWallClockMs(scenario.codexWallClockMs)}; ${formatRuntimeSpeedComparison(scenario)}`, + ); + if ( + scenario.openclawBootstrapWallClockMs !== undefined || + scenario.codexBootstrapWallClockMs !== undefined + ) { + lines.push( + `- gateway bootstrap (excluded): openclaw ${formatRuntimeWallClockMs(scenario.openclawBootstrapWallClockMs ?? null)}; codex ${formatRuntimeWallClockMs(scenario.codexBootstrapWallClockMs ?? null)}`, + ); + } + lines.push( + `- prompt cache: openclaw ${formatRuntimeCacheHitPercent(scenario.openclawUsage?.cacheHitPercent)} (${formatRuntimeCacheCount(scenario.openclawUsage?.cachedInputTokens)} cached, ${formatRuntimeCacheCount(scenario.openclawUsage?.uncachedInputTokens)} uncached input); codex ${formatRuntimeCacheHitPercent(scenario.codexUsage?.cacheHitPercent)} (${formatRuntimeCacheCount(scenario.codexUsage?.cachedInputTokens)} cached, ${formatRuntimeCacheCount(scenario.codexUsage?.uncachedInputTokens)} uncached input)`, + ); + if (scenario.runtimeParityUsage.expectation === "not-applicable") { + lines.push(`- assistant-message usage: N/A (${scenario.runtimeParityUsage.reason})`); + } + if (scenario.driftDetails) { + lines.push(`- details: ${scenario.driftDetails}`); + } + lines.push(""); + } + lines.push("## Notes", ""); + for (const note of report.notes) { + lines.push(`- ${note}`); + } + lines.push(""); + return lines.join("\n"); +} diff --git a/extensions/qa-lab/src/agentic-parity-runtime-report-contract.ts b/extensions/qa-lab/src/agentic-parity-runtime-report-contract.ts new file mode 100644 index 000000000000..6471cc3558f6 --- /dev/null +++ b/extensions/qa-lab/src/agentic-parity-runtime-report-contract.ts @@ -0,0 +1,45 @@ +import type { QaRuntimeParityCacheUsage } from "./agentic-parity-cache-usage.js"; +import type { QaRuntimeTiming } from "./runtime-parity-timing.js"; +import type { RuntimeId, RuntimeParityDrift, RuntimeParityUsagePolicy } from "./runtime-parity.js"; + +export type QaRuntimeParityScenarioReport = { + name: string; + status: "pass" | "fail"; + runtimeParityUsage: RuntimeParityUsagePolicy; + drift: RuntimeParityDrift | "missing"; + driftDetails?: string; + openclawStatus: "pass" | "fail" | "missing"; + codexStatus: "pass" | "fail" | "missing"; + openclawTokens: number; + codexTokens: number; + openclawUsage: QaRuntimeParityCacheUsage | null; + codexUsage: QaRuntimeParityCacheUsage | null; + openclawToolCalls: number; + codexToolCalls: number; + openclawWallClockMs: number | null; + codexWallClockMs: number | null; + openclawBootstrapWallClockMs?: number; + codexBootstrapWallClockMs?: number; + fasterRuntime: RuntimeId | "tie" | null; + speedupPercent: number | null; +}; + +export type QaRuntimeParityReport = { + runtimePair: [RuntimeId, RuntimeId]; + comparedAt: string; + providerMode?: string; + primaryModel?: string; + totalScenarios: number; + passedScenarios: number; + failedScenarios: number; + driftCounts: Record; + scenarios: QaRuntimeParityScenarioReport[]; + timing: QaRuntimeTiming; + usage: { + openclaw: QaRuntimeParityCacheUsage | null; + codex: QaRuntimeParityCacheUsage | null; + }; + pass: boolean; + failures: string[]; + notes: string[]; +}; diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts index e7f9ebe71152..0c50dfa37c2a 100644 --- a/extensions/qa-lab/src/cli.runtime.ts +++ b/extensions/qa-lab/src/cli.runtime.ts @@ -16,6 +16,7 @@ import { type QaParitySuiteSummary, type QaRuntimeParitySuiteSummary, } from "./agentic-parity-report.js"; +import type { QaRuntimeParityReport } from "./agentic-parity-runtime-report-contract.js"; import { resolveQaParityPackScenarioIds } from "./agentic-parity.js"; import { createQaArtifactRunId } from "./artifact-run-id.js"; import { runQaCharacterEval, type QaCharacterModelOptions } from "./character-eval.js"; @@ -1115,7 +1116,7 @@ export async function runQaParityReportCommand(opts: { const summary = JSON.parse( await fs.readFile(summaryPath, "utf8"), ) as QaRuntimeParitySuiteSummary; - const reportPayload = buildQaRuntimeParityReport({ summary }); + const reportPayload: QaRuntimeParityReport = buildQaRuntimeParityReport({ summary }); const report = renderQaRuntimeParityMarkdownReport(reportPayload); const reportPath = path.join(outputDir, "qa-runtime-parity-report.md"); const runtimeSummaryPath = path.join(outputDir, "qa-runtime-parity-summary.json"); diff --git a/extensions/qa-lab/src/gateway-child.test.ts b/extensions/qa-lab/src/gateway-child.test.ts index d92e6d8ba884..55ba640ab222 100644 --- a/extensions/qa-lab/src/gateway-child.test.ts +++ b/extensions/qa-lab/src/gateway-child.test.ts @@ -229,6 +229,67 @@ describe("Gateway child fixture helpers", () => { }); }); + it("stages native Codex model metadata before starting the private mock runtime", async () => { + const tempRoot = await tempDirs.makeTempDir("qa-codex-model-catalog-"); + const modelCatalogPath = await testing.stageQaCodexMockModelCatalog({ + tempRoot, + forcedRuntime: "codex", + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.6-luna", + alternateModel: "mock-openai/gpt-5.6-luna-alt", + }); + + expect(modelCatalogPath).toBe(path.join(tempRoot, "codex-model-catalog.json")); + const catalog = JSON.parse(await readFile(modelCatalogPath!, "utf8")) as { + models: Array>; + }; + expect(catalog.models).toEqual([ + expect.objectContaining({ + slug: "gpt-5.6-luna", + apply_patch_tool_type: "freeform", + tool_mode: "direct", + }), + expect.objectContaining({ + slug: "gpt-5.6-luna-alt", + apply_patch_tool_type: "freeform", + tool_mode: "direct", + }), + ]); + expect( + testing.buildQaForcedRuntimeEnvPatch({ + forcedRuntime: "codex", + providerMode: "mock-openai", + providerBaseUrl: "http://127.0.0.1:44080/v1", + codexModelCatalogPath: modelCatalogPath, + }), + ).toEqual( + expect.objectContaining({ + OPENCLAW_CODEX_APP_SERVER_ARGS: `app-server -c openai_base_url=http://127.0.0.1:44080/v1 -c ${JSON.stringify(`model_catalog_json=${modelCatalogPath}`)} --listen stdio://`, + }), + ); + }); + + it("does not stage a Codex catalog for other runtimes or live providers", async () => { + const tempRoot = await tempDirs.makeTempDir("qa-codex-model-catalog-unused-"); + await expect( + testing.stageQaCodexMockModelCatalog({ + tempRoot, + forcedRuntime: "openclaw", + providerMode: "mock-openai", + }), + ).resolves.toBeUndefined(); + await expect( + testing.stageQaCodexMockModelCatalog({ + tempRoot, + forcedRuntime: "codex", + providerMode: "live-frontier", + }), + ).resolves.toBeUndefined(); + await expect( + readFile(path.join(tempRoot, "codex-model-catalog.json"), "utf8"), + ).rejects.toThrow(); + }); + it("resolves source and built Gateway CLI commands", async () => { const repoRoot = await tempDirs.makeTempDir("qa-gateway-command-"); await mkdir(path.join(repoRoot, "src"), { recursive: true }); diff --git a/extensions/qa-lab/src/gateway-child.ts b/extensions/qa-lab/src/gateway-child.ts index 7d18d91e8020..57e8bce01cb0 100644 --- a/extensions/qa-lab/src/gateway-child.ts +++ b/extensions/qa-lab/src/gateway-child.ts @@ -68,6 +68,7 @@ import { stageQaLiveAnthropicSetupToken, } from "./providers/live-frontier/auth.js"; import { stageQaMockAuthProfiles } from "./providers/shared/mock-auth.js"; +import { listMockCodexModelInfos } from "./providers/shared/mock-model-config.js"; import { seedQaAgentWorkspace } from "./qa-agent-workspace.js"; import { buildQaGatewayConfig, type QaThinkingLevel } from "./qa-gateway-config.js"; import type { QaTransportAdapter } from "./qa-transport.js"; @@ -432,10 +433,33 @@ export function buildQaRuntimeEnv(params: { return scrubQaGatewayChildSecretEnv(normalizedEnv); } +async function stageQaCodexMockModelCatalog(params: { + tempRoot: string; + forcedRuntime?: RuntimeId; + providerMode: QaProviderMode; + primaryModel?: string; + alternateModel?: string; +}): Promise { + if (params.forcedRuntime !== "codex" || params.providerMode !== "mock-openai") { + return undefined; + } + const modelCatalogPath = path.join(params.tempRoot, "codex-model-catalog.json"); + const selectedModelRefs = [params.primaryModel, params.alternateModel].filter( + (model): model is string => typeof model === "string" && model.length > 0, + ); + await fs.writeFile( + modelCatalogPath, + `${JSON.stringify({ models: listMockCodexModelInfos(selectedModelRefs) }, null, 2)}\n`, + { encoding: "utf8", mode: 0o600 }, + ); + return modelCatalogPath; +} + function buildQaForcedRuntimeEnvPatch(params: { forcedRuntime?: RuntimeId; providerMode: QaProviderMode; providerBaseUrl?: string; + codexModelCatalogPath?: string; }): NodeJS.ProcessEnv | undefined { if (!params.forcedRuntime) { return undefined; @@ -451,7 +475,11 @@ function buildQaForcedRuntimeEnvPatch(params: { if (!providerBaseUrl) { throw new Error("forced Codex mock QA requires the managed mock provider URL"); } - patch.OPENCLAW_CODEX_APP_SERVER_ARGS = `app-server -c openai_base_url=${providerBaseUrl} --listen stdio://`; + if (!params.codexModelCatalogPath) { + throw new Error("forced Codex mock QA requires the staged native model catalog"); + } + const modelCatalogOverride = JSON.stringify(`model_catalog_json=${params.codexModelCatalogPath}`); + patch.OPENCLAW_CODEX_APP_SERVER_ARGS = `app-server -c openai_base_url=${providerBaseUrl} -c ${modelCatalogOverride} --listen stdio://`; patch.OPENAI_API_KEY = QA_MOCK_OPENAI_API_KEY; patch.CODEX_API_KEY = QA_MOCK_OPENAI_API_KEY; return patch; @@ -593,6 +621,7 @@ async function waitForQaGatewayRestartBoundary(params: { export const testing = { assertQaArtifactDirWithinRepo, + buildQaForcedRuntimeEnvPatch, buildQaRuntimeEnv, cleanupQaGatewayTempRoots, fetchLocalGatewayHealth, @@ -609,6 +638,7 @@ export const testing = { stageQaLiveApiKeyProfiles, stageQaLiveAnthropicSetupToken, stageQaMockAuthProfiles, + stageQaCodexMockModelCatalog, resolveQaLiveCliAuthEnv, waitForQaGatewayRestartBoundary, resolveQaOwnerPluginIdsForProviderIds, @@ -1047,6 +1077,13 @@ export async function startQaGatewayChild(params: { fs.mkdir(xdgCacheHome, { recursive: true }), ]); const providerMode = resolveQaGatewayChildProviderMode(params.providerMode); + const codexModelCatalogPath = await stageQaCodexMockModelCatalog({ + tempRoot, + forcedRuntime: params.forcedRuntime, + providerMode, + primaryModel: params.primaryModel, + alternateModel: params.alternateModel, + }); const resolvedProvider = getQaProvider(providerMode); const liveProviderIds = resolvedProvider.usesModelProviderPlugins ? [params.primaryModel, params.alternateModel] @@ -1309,6 +1346,7 @@ export async function startQaGatewayChild(params: { forcedRuntime: params.forcedRuntime, providerMode, providerBaseUrl: params.providerBaseUrl, + codexModelCatalogPath, }), }, forwardHostHomeForClaudeCli: liveProviderIds.includes("claude-cli"), diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts index d816f54c18be..9ae5cf39216c 100644 --- a/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts @@ -7,6 +7,7 @@ import { writeJson } from "../shared/http-json.js"; export type ResponsesInputItem = Record; export type StreamEvent = + | { type: "response.created"; response: { id: string } } | { type: "response.output_item.added"; item: Record } | { type: "response.output_text.delta"; @@ -23,6 +24,12 @@ export type StreamEvent = text: string; } | { type: "response.function_call_arguments.delta"; delta: string } + | { + type: "response.custom_tool_call_input.delta"; + item_id: string; + call_id: string; + delta: string; + } | { type: "response.output_item.done"; item: Record } | { type: "response.completed"; diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-directives.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-directives.ts index 31773c382b21..573275a52e46 100644 --- a/extensions/qa-lab/src/providers/mock-openai/mock-openai-directives.ts +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-directives.ts @@ -206,6 +206,18 @@ export function hasToolDefinition(body: Record, name: string) { return [...tools, ...dynamicTools].some((tool) => toolDefinitionMentionsName(tool, name)); } +export function hasDeclaredCustomTool(body: Record, name: string) { + const tools = Array.isArray(body.tools) ? body.tools : []; + return tools.some( + (tool) => + tool !== null && + typeof tool === "object" && + !Array.isArray(tool) && + (tool as Record).type === "custom" && + (tool as Record).name === name, + ); +} + function toolDefinitionMentionsName(value: unknown, name: string, depth = 0): boolean { if (depth > 6 || !value || typeof value !== "object") { return false; diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-events.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-events.ts index 194c1dc1d6b8..563bef2d6df1 100644 --- a/extensions/qa-lab/src/providers/mock-openai/mock-openai-events.ts +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-events.ts @@ -98,7 +98,10 @@ export function extractPlannedToolName(events: StreamEvent[]) { continue; } const item = event.item as { type?: unknown; name?: unknown }; - if (item.type === "function_call" && typeof item.name === "string") { + if ( + (item.type === "function_call" || item.type === "custom_tool_call") && + typeof item.name === "string" + ) { return item.name; } } @@ -111,7 +114,10 @@ export function extractPlannedToolCallId(events: StreamEvent[]) { continue; } const item = event.item as { type?: unknown; call_id?: unknown }; - if (item.type === "function_call" && typeof item.call_id === "string") { + if ( + (item.type === "function_call" || item.type === "custom_tool_call") && + typeof item.call_id === "string" + ) { return item.call_id; } } @@ -123,7 +129,10 @@ export function extractPlannedToolArgs(events: StreamEvent[]) { if (event.type !== "response.output_item.done") { continue; } - const item = event.item as { type?: unknown; arguments?: unknown }; + const item = event.item as { type?: unknown; arguments?: unknown; input?: unknown }; + if (item.type === "custom_tool_call") { + return typeof item.input === "string" ? { input: item.input } : undefined; + } if (item.type !== "function_call" || typeof item.arguments !== "string") { continue; } diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-input.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-input.ts index f77731743f64..699c7a9208ba 100644 --- a/extensions/qa-lab/src/providers/mock-openai/mock-openai-input.ts +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-input.ts @@ -100,15 +100,19 @@ function stringifyFunctionCallOutput(output: unknown): string { return ""; } +function isResponsesToolCallOutput(item: ResponsesInputItem) { + return item.type === "function_call_output" || item.type === "custom_tool_call_output"; +} + function extractFunctionCallOutputText(item: ResponsesInputItem) { - if (item.type !== "function_call_output") { + if (!isResponsesToolCallOutput(item)) { return ""; } return stringifyFunctionCallOutput(item.output); } function extractFunctionCallOutputCallId(item: ResponsesInputItem) { - if (item.type !== "function_call_output") { + if (!isResponsesToolCallOutput(item)) { return ""; } const record = item as { @@ -124,7 +128,7 @@ function extractFunctionCallOutputCallId(item: ResponsesInputItem) { } function functionCallOutputIsStructuredError(item: ResponsesInputItem) { - if (item.type !== "function_call_output") { + if (!isResponsesToolCallOutput(item)) { return false; } return item.is_error === true || item.isError === true; diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-responses-websocket.test.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-responses-websocket.test.ts new file mode 100644 index 000000000000..cbc4c5a26a83 --- /dev/null +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-responses-websocket.test.ts @@ -0,0 +1,375 @@ +import { once } from "node:events"; +import { afterEach, describe, expect, it } from "vitest"; +import { WebSocket, type RawData } from "ws"; +import { startQaMockOpenAiServer } from "./server.js"; + +const cleanups: Array<() => Promise> = []; + +afterEach(async () => { + while (cleanups.length > 0) { + await cleanups.pop()?.(); + } +}); + +async function startServer() { + const server = await startQaMockOpenAiServer({ host: "127.0.0.1", port: 0 }); + cleanups.push(async () => server.stop()); + return server; +} + +async function connectResponsesWebSocket(baseUrl: string) { + const socket = new WebSocket(`${baseUrl.replace(/^http/u, "ws")}/v1/responses`); + cleanups.push(async () => { + if (socket.readyState !== WebSocket.CLOSED) { + socket.terminate(); + } + }); + await once(socket, "open"); + return socket; +} + +async function collectResponseEvents(socket: WebSocket, request: unknown) { + return await new Promise>>((resolve, reject) => { + const events: Array> = []; + const timeout = setTimeout(() => { + cleanup(); + reject(new Error("timed out waiting for the mock Responses WebSocket")); + }, 5_000); + const cleanup = () => { + clearTimeout(timeout); + socket.off("message", onMessage); + socket.off("error", onError); + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + const onMessage = (data: RawData) => { + const text = Buffer.isBuffer(data) + ? data.toString("utf8") + : Array.isArray(data) + ? Buffer.concat(data).toString("utf8") + : Buffer.from(data).toString("utf8"); + const event = JSON.parse(text) as Record; + events.push(event); + if (event.type === "response.completed" || event.type === "error") { + cleanup(); + resolve(events); + } + }; + socket.on("message", onMessage); + socket.on("error", onError); + socket.send(typeof request === "string" ? request : JSON.stringify(request)); + }); +} + +function readCompletedResponse(events: Array>) { + const completion = events.find((event) => event.type === "response.completed"); + expect(completion).toBeDefined(); + return completion?.response as Record; +} + +describe("QA mock OpenAI Responses WebSocket", () => { + it("streams the native response.create protocol and records normal QA request evidence", async () => { + const server = await startServer(); + const socket = await connectResponsesWebSocket(server.baseUrl); + const prompt = "Read README.md for the source and docs discovery report."; + + const events = await collectResponseEvents(socket, { + type: "response.create", + model: "gpt-5.6-sol", + stream: true, + tools: [{ type: "function", name: "read" }], + input: [{ role: "user", content: [{ type: "input_text", text: prompt }] }], + }); + + expect(events[0]).toMatchObject({ type: "response.created" }); + expect(events[0]).toMatchObject({ + sequence_number: 0, + response: { + object: "response", + model: "gpt-5.6-sol", + status: "in_progress", + output: [], + }, + }); + expect(events).toEqual( + expect.arrayContaining([ + expect.objectContaining({ type: "response.output_item.done" }), + expect.objectContaining({ type: "response.completed" }), + ]), + ); + const lastRequest = await fetch(`${server.baseUrl}/debug/last-request`).then((response) => + response.json(), + ); + expect(lastRequest).toMatchObject({ + model: "gpt-5.6-sol", + prompt, + plannedToolName: "read", + }); + }); + + it("reconstructs Codex previous-response deltas on the same reusable connection", async () => { + const server = await startServer(); + const socket = await connectResponsesWebSocket(server.baseUrl); + const prompt = "Read README.md for the source and docs discovery report."; + const firstEvents = await collectResponseEvents(socket, { + type: "response.create", + model: "gpt-5.6-sol", + stream: true, + tools: [{ type: "function", name: "read" }], + input: [{ role: "user", content: [{ type: "input_text", text: prompt }] }], + }); + const firstResponse = readCompletedResponse(firstEvents); + const output = firstResponse.output as Array>; + const toolCall = output.find((item) => item.type === "function_call"); + expect(toolCall?.call_id).toEqual(expect.any(String)); + + const secondEvents = await collectResponseEvents(socket, { + type: "response.create", + model: "gpt-5.6-sol", + stream: true, + previous_response_id: firstResponse.id, + input: [ + { + type: "function_call_output", + call_id: toolCall?.call_id, + output: "README: source and docs evidence captured", + }, + ], + }); + + expect(firstEvents.map((event) => event.sequence_number)).toEqual( + firstEvents.map((_, index) => index), + ); + expect(secondEvents.map((event) => event.sequence_number)).toEqual( + secondEvents.map((_, index) => index), + ); + + const requests = (await fetch(`${server.baseUrl}/debug/requests`).then((response) => + response.json(), + )) as Array>; + expect(requests).toHaveLength(2); + expect(requests[1]).toMatchObject({ + prompt, + toolOutput: "README: source and docs evidence captured", + toolOutputCallId: toolCall?.call_id, + }); + }); + + it("rejects a response ID after a newer response replaces the connection cache", async () => { + const server = await startServer(); + const socket = await connectResponsesWebSocket(server.baseUrl); + const prompt = "Read README.md for the source and docs discovery report."; + const firstEvents = await collectResponseEvents(socket, { + type: "response.create", + model: "gpt-5.6-sol", + stream: true, + store: false, + tools: [{ type: "function", name: "read" }], + input: [{ role: "user", content: [{ type: "input_text", text: prompt }] }], + }); + const firstResponse = readCompletedResponse(firstEvents); + const firstOutput = firstResponse.output as Array>; + const toolCall = firstOutput.find((item) => item.type === "function_call"); + + const secondEvents = await collectResponseEvents(socket, { + type: "response.create", + previous_response_id: firstResponse.id, + input: [ + { + type: "function_call_output", + call_id: toolCall?.call_id, + output: "README: source and docs evidence captured", + }, + ], + }); + + expect(readCompletedResponse(secondEvents).id).not.toBe(firstResponse.id); + await expect( + collectResponseEvents(socket, { + type: "response.create", + previous_response_id: firstResponse.id, + input: [], + }), + ).resolves.toEqual([ + expect.objectContaining({ + type: "error", + sequence_number: 0, + status: 400, + error: expect.objectContaining({ code: "previous_response_not_found" }), + }), + ]); + }); + + it("answers connection warmups without inventing a model or tool request", async () => { + const server = await startServer(); + const socket = await connectResponsesWebSocket(server.baseUrl); + + const events = await collectResponseEvents(socket, { + type: "response.create", + model: "gpt-5.6-sol", + stream: true, + generate: false, + input: [], + }); + + expect(events).toEqual([ + expect.objectContaining({ type: "response.created" }), + expect.objectContaining({ + type: "response.completed", + response: expect.objectContaining({ + object: "response", + model: "gpt-5.6-sol", + status: "completed", + output: [], + usage: expect.objectContaining({ + input_tokens_details: { cached_tokens: 0 }, + output_tokens_details: { reasoning_tokens: 0 }, + }), + }), + }), + ]); + const cursor = await fetch(`${server.baseUrl}/debug/request-cursor`).then((response) => + response.json(), + ); + expect(cursor).toEqual({ cursor: 0 }); + }); + + it("preserves the full warmup prompt when the first real response sends an empty delta", async () => { + const server = await startServer(); + const socket = await connectResponsesWebSocket(server.baseUrl); + const prompt = "Read README.md for the source and docs discovery report."; + + const warmup = await collectResponseEvents(socket, { + type: "response.create", + model: "gpt-5.6-sol", + stream: true, + generate: false, + tools: [{ type: "function", name: "read" }], + input: [{ role: "user", content: [{ type: "input_text", text: prompt }] }], + }); + + const events = await collectResponseEvents(socket, { + type: "response.create", + stream: true, + previous_response_id: readCompletedResponse(warmup).id, + input: [], + }); + + expect(events).toEqual( + expect.arrayContaining([ + expect.objectContaining({ type: "response.output_item.done" }), + expect.objectContaining({ type: "response.completed" }), + ]), + ); + const lastRequest = await fetch(`${server.baseUrl}/debug/last-request`).then((response) => + response.json(), + ); + expect(lastRequest).toMatchObject({ model: "gpt-5.6-sol", prompt, plannedToolName: "read" }); + const cursor = await fetch(`${server.baseUrl}/debug/request-cursor`).then((response) => + response.json(), + ); + expect(cursor).toEqual({ cursor: 1 }); + }); + + it("preserves the injected post-tool 503 failure on the WebSocket transport", async () => { + const server = await startServer(); + const socket = await connectResponsesWebSocket(server.baseUrl); + const prompt = "Provider HTTP 503 after tool QA check: read QA_KICKOFF_TASK.md, then reply."; + + const firstEvents = await collectResponseEvents(socket, { + type: "response.create", + model: "gpt-5.6-sol", + stream: true, + tools: [{ type: "function", name: "read" }], + input: [{ role: "user", content: [{ type: "input_text", text: prompt }] }], + }); + const firstResponse = readCompletedResponse(firstEvents); + const output = firstResponse.output as Array>; + const toolCall = output.find((item) => item.type === "function_call"); + + const failure = await collectResponseEvents(socket, { + type: "response.create", + model: "gpt-5.6-sol", + stream: true, + previous_response_id: firstResponse.id, + input: [ + { + type: "function_call_output", + call_id: toolCall?.call_id, + output: "QA mission loaded", + }, + ], + }); + + expect(failure).toEqual([ + expect.objectContaining({ + type: "error", + status: 503, + error: { type: "server_error", message: "Service Unavailable" }, + }), + ]); + await expect( + collectResponseEvents(socket, { + type: "response.create", + previous_response_id: firstResponse.id, + input: [], + }), + ).resolves.toEqual([ + expect.objectContaining({ + type: "error", + sequence_number: 0, + status: 400, + error: expect.objectContaining({ code: "previous_response_not_found" }), + }), + ]); + const requests = (await fetch(`${server.baseUrl}/debug/requests`).then((response) => + response.json(), + )) as Array>; + expect(requests).toHaveLength(2); + expect(requests[1]).toMatchObject({ prompt, toolOutput: "QA mission loaded" }); + }); + + it("rejects a response delta whose previous response belongs to another connection", async () => { + const server = await startServer(); + const socket = await connectResponsesWebSocket(server.baseUrl); + + const events = await collectResponseEvents(socket, { + type: "response.create", + model: "gpt-5.6-sol", + previous_response_id: "resp_qa_unknown", + input: [], + }); + + expect(events).toEqual([ + expect.objectContaining({ + type: "error", + status: 400, + error: { + type: "invalid_request_error", + code: "previous_response_not_found", + message: "The previous response was not found on this connection.", + }, + }), + ]); + }); + + it("returns a bounded protocol error for malformed WebSocket requests", async () => { + const server = await startServer(); + const socket = await connectResponsesWebSocket(server.baseUrl); + + const events = await collectResponseEvents(socket, "{invalid-json"); + + expect(events).toEqual([ + expect.objectContaining({ + type: "error", + status: 400, + error: { + type: "invalid_request_error", + message: "Expected a JSON response.create request.", + }, + }), + ]); + }); +}); diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-responses-websocket.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-responses-websocket.ts new file mode 100644 index 000000000000..4918f436b1f6 --- /dev/null +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-responses-websocket.ts @@ -0,0 +1,261 @@ +import type { Server } from "node:http"; +import { setTimeout as sleep } from "node:timers/promises"; +import { WebSocket, WebSocketServer, type RawData } from "ws"; +import type { ResponsesInputItem, StreamEvent } from "./mock-openai-contracts.js"; + +export type QaMockResponsesDispatchResult = { + events: StreamEvent[]; + failure?: { + status: number; + type: string; + message: string; + }; + previewPauseMs?: number; +}; + +type QaMockResponsesWebSocketDispatch = (params: { + body: Record; + raw: string; +}) => Promise; + +type QaMockResponsesWebSocketHistory = { + id: string; + body: Record; + input: ResponsesInputItem[]; +}; + +function readWebSocketText(data: RawData): string { + if (typeof data === "string") { + return data; + } + if (Array.isArray(data)) { + return Buffer.concat(data).toString("utf8"); + } + if (data instanceof ArrayBuffer) { + return Buffer.from(data).toString("utf8"); + } + return data.toString("utf8"); +} + +function readWebSocketRequest(raw: string): Record | undefined { + try { + const value: unknown = JSON.parse(raw); + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : undefined; + } catch { + return undefined; + } +} + +function sendWebSocketEvent(socket: WebSocket, event: unknown): void { + if (socket.readyState === WebSocket.OPEN) { + socket.send(JSON.stringify(event)); + } +} + +/** + * Give Codex the same Responses-over-WebSocket transport its built-in OpenAI + * provider uses in production. An SSE-only mock makes each native turn pay + * websocket reconnect/fallback retries before the real scenario even starts. + */ +export function attachQaMockResponsesWebSocketServer(params: { + server: Server; + dispatch: QaMockResponsesWebSocketDispatch; +}) { + const webSocketServer = new WebSocketServer({ + noServer: true, + maxPayload: 4 * 1024 * 1024, + }); + + params.server.on("upgrade", (request, socket, head) => { + const url = new URL(request.url ?? "/", "http://127.0.0.1"); + if (url.pathname !== "/v1/responses") { + socket.write("HTTP/1.1 404 Not Found\r\nConnection: close\r\n\r\n"); + socket.destroy(); + return; + } + + webSocketServer.handleUpgrade(request, socket, head, (connection) => { + webSocketServer.emit("connection", connection, request); + }); + }); + + webSocketServer.on("connection", (socket: WebSocket) => { + let cachedResponse: QaMockResponsesWebSocketHistory | undefined; + let warmupOrdinal = 0; + let sequenceNumber = 0; + let pending = Promise.resolve(); + const sendEvent = (event: Record) => { + sendWebSocketEvent(socket, { ...event, sequence_number: sequenceNumber++ }); + }; + + socket.on("error", () => { + // A disconnected client must not crash or outlive the private mock. + }); + + socket.on("message", (data: RawData, isBinary: boolean) => { + pending = pending + .then(async () => { + // Responses stream sequence numbers describe one response, not + // all responses that happen to reuse the same WebSocket. + sequenceNumber = 0; + const raw = readWebSocketText(data); + const request = isBinary ? undefined : readWebSocketRequest(raw); + if (!request || request.type !== "response.create") { + sendEvent({ + type: "error", + status: 400, + error: { + type: "invalid_request_error", + message: "Expected a JSON response.create request.", + }, + }); + return; + } + + const previousResponseId = + typeof request.previous_response_id === "string" + ? request.previous_response_id + : undefined; + if (previousResponseId && cachedResponse?.id !== previousResponseId) { + sendEvent({ + type: "error", + status: 400, + error: { + type: "invalid_request_error", + code: "previous_response_not_found", + message: "The previous response was not found on this connection.", + }, + }); + return; + } + const previousResponse = previousResponseId ? cachedResponse : undefined; + const previousInput = previousResponse?.input ?? []; + const nextInput = Array.isArray(request.input) + ? (request.input as ResponsesInputItem[]) + : []; + const inheritedRequest = previousResponse ? { ...previousResponse.body } : {}; + // These control fields belong to one frame; model, instructions, + // tool definitions, and other prompt context carry across deltas. + delete inheritedRequest.type; + delete inheritedRequest.previous_response_id; + delete inheritedRequest.generate; + delete inheritedRequest.input; + const body: Record & { input: ResponsesInputItem[] } = { + ...inheritedRequest, + ...request, + input: [...previousInput, ...nextInput], + }; + + if (request.generate === false) { + const id = `resp_qa_ws_warmup_${++warmupOrdinal}`; + const createdAt = Math.floor(Date.now() / 1_000); + const model = typeof body.model === "string" ? body.model : ""; + sendEvent({ + type: "response.created", + response: { + id, + object: "response", + created_at: createdAt, + model, + status: "in_progress", + output: [], + }, + }); + sendEvent({ + type: "response.completed", + response: { + id, + object: "response", + created_at: createdAt, + model, + status: "completed", + output: [], + usage: { + input_tokens: 0, + input_tokens_details: { cached_tokens: 0 }, + output_tokens: 0, + output_tokens_details: { reasoning_tokens: 0 }, + total_tokens: 0, + }, + }, + }); + // Codex's real first request often sends no input at all: its + // previous_response_id points at this prompt-bearing warmup. + cachedResponse = { id, body, input: body.input }; + return; + } + + // Codex sends only the delta after a cached response; scenario and + // debug matching need the complete prompt and inherited tool surface. + const dispatched = await params.dispatch({ body, raw: JSON.stringify(body) }); + if (dispatched.failure) { + // A failed continuation invalidates the connection's ephemeral + // store=false response; Codex must retry with a full new request. + cachedResponse = undefined; + sendEvent({ + type: "error", + status: dispatched.failure.status, + error: { + type: dispatched.failure.type, + message: dispatched.failure.message, + }, + }); + return; + } + const { events } = dispatched; + const completion = events.find((event) => event.type === "response.completed"); + if (completion?.type === "response.completed") { + if (!events.some((event) => event.type === "response.created")) { + sendEvent({ + type: "response.created", + response: { + id: completion.response.id, + object: "response", + created_at: Math.floor(Date.now() / 1_000), + model: typeof body.model === "string" ? body.model : "", + status: "in_progress", + output: [], + }, + }); + } + cachedResponse = { + id: completion.response.id, + body, + input: [...body.input, ...completion.response.output], + }; + } + for (const event of events) { + if (dispatched.previewPauseMs && event.type === "response.output_text.done") { + await sleep(dispatched.previewPauseMs); + } + sendEvent(event); + } + }) + .catch(() => { + cachedResponse = undefined; + sequenceNumber = 0; + sendEvent({ + type: "error", + status: 500, + error: { + type: "server_error", + message: "Mock Responses WebSocket dispatch failed.", + }, + }); + }); + }); + }); + + return { + async close(): Promise { + for (const socket of webSocketServer.clients) { + socket.terminate(); + } + await new Promise((resolve, reject) => { + webSocketServer.close((error) => (error ? reject(error) : resolve())); + }); + }, + }; +} diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-tooling.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-tooling.ts index a496f3497095..70db2f170c63 100644 --- a/extensions/qa-lab/src/providers/mock-openai/mock-openai-tooling.ts +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-tooling.ts @@ -125,6 +125,45 @@ export function buildToolCallEventsWithArgs( ]; } +export function buildCustomToolCallEventsWithInput(name: string, input: string): StreamEvent[] { + const call = buildMockFunctionCall(name, { input }); + const itemId = call.itemId.replace(/^fc_/, "ctc_"); + const item = { + type: "custom_tool_call", + id: itemId, + call_id: call.callId, + name, + input, + status: "completed", + }; + return [ + { + type: "response.created", + response: { id: call.responseId }, + }, + { + type: "response.output_item.added", + item: { ...item, input: "", status: "in_progress" }, + }, + { + type: "response.custom_tool_call_input.delta", + item_id: itemId, + call_id: call.callId, + delta: input, + }, + { type: "response.output_item.done", item }, + { + type: "response.completed", + response: { + id: call.responseId, + status: "completed", + output: [item], + usage: { input_tokens: 64, output_tokens: 16, total_tokens: 80 }, + }, + }, + ]; +} + export function extractRememberedFact(userTexts: string[]) { for (const text of userTexts) { const qaCanaryMatch = /\bqa canary code is\s+([A-Za-z0-9-]+)/i.exec(text); @@ -171,6 +210,19 @@ export function buildQaToolSearchArgs( if (failureMode && targetTool === "web_search") { return { query: QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY }; } + if (failureMode && targetTool === "apply_patch") { + return { + input: [ + "*** Begin Patch", + "*** Update File: ../runtime-tool-fixture-denied.txt", + "@@", + "-runtime-tool-fixture-denied-original", + "+runtime patch outside the workspace", + "*** End Patch", + "", + ].join("\n"), + }; + } if (failureMode) { return { __qaFailureMode: "denied-input" }; } diff --git a/extensions/qa-lab/src/providers/mock-openai/server.test.ts b/extensions/qa-lab/src/providers/mock-openai/server.test.ts index 223113fea40a..663ddb6f29e0 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.test.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.test.ts @@ -3971,6 +3971,227 @@ describe("qa mock openai server", () => { expect(String(toolPlanOutput.arguments)).toContain("OPENCLAW_QA_WEB_SEARCH_DENIED_INPUT"); }); + it.each([ + { + label: "workspace-local happy", + prompt: + "tool search qa check target=apply_patch. Call apply_patch exactly once and then summarize.", + operation: "Add File", + patchPath: "runtime-tool-fixture-patch.txt", + }, + { + label: "workspace-escaping failure", + prompt: + "tool search qa failure target=apply_patch. Exercise the denied-input path once and then summarize.", + operation: "Update File", + patchPath: "../runtime-tool-fixture-denied.txt", + }, + ])("plans a valid $label apply_patch envelope", async ({ prompt, operation, patchPath }) => { + const server = await startMockServer(); + const response = await postResponses(server, { + stream: false, + input: [makeUserInput(prompt)], + }); + + expect(response.status).toBe(200); + const payload = await response.json(); + expect(outputItem(payload)).toMatchObject({ type: "function_call", name: "apply_patch" }); + const args = outputToolArgs(payload); + expect(args).not.toHaveProperty("__qaFailureMode"); + expect(args.input).toBeTypeOf("string"); + expect(args.input).toContain("*** Begin Patch\n"); + expect(args.input).toContain(`*** ${operation}: ${patchPath}\n`); + if (operation === "Update File") { + expect(args.input).toContain("\n@@\n-runtime-tool-fixture-denied-original\n"); + } + expect(args.input).toContain("\n*** End Patch\n"); + }); + + it.each([ + { + label: "workspace-local happy", + prompt: + "tool search qa check target=apply_patch. Call apply_patch exactly once and then summarize.", + operation: "Add File", + patchPath: "runtime-tool-fixture-patch.txt", + }, + { + label: "workspace-escaping failure", + prompt: + "tool search qa failure target=apply_patch. Exercise the denied-input path once and then summarize.", + operation: "Update File", + patchPath: "../runtime-tool-fixture-denied.txt", + }, + ])("plans an actual $label native freeform patch", async (testCase) => { + const server = await startMockServer(); + const response = await postResponses(server, { + stream: false, + tools: [ + { + type: "custom", + name: "apply_patch", + format: { type: "grammar", syntax: "lark", definition: "start: /.+/" }, + }, + ], + input: [makeUserInput(testCase.prompt)], + }); + + expect(response.status).toBe(200); + const item = outputItem(await response.json()); + expect(item).toMatchObject({ type: "custom_tool_call", name: "apply_patch" }); + expect(item).not.toHaveProperty("arguments"); + expect(item.input).toEqual( + expect.stringContaining(`*** ${testCase.operation}: ${testCase.patchPath}\n`), + ); + if (testCase.operation === "Update File") { + expect(item.input).toEqual( + expect.stringContaining("\n@@\n-runtime-tool-fixture-denied-original\n"), + ); + } + + const debugResponse = await fetch(`${server.baseUrl}/debug/last-request`); + expect(debugResponse.status).toBe(200); + const debug = requireRecord(await debugResponse.json(), "native patch plan debug request"); + expect(debug.plannedToolName).toBe("apply_patch"); + expect(debug.plannedToolCallId).toBe(item.call_id); + expect(debug.plannedToolArgs).toEqual({ input: item.input }); + }); + + it("streams native Codex patch input as custom-tool SSE", async () => { + const server = await startMockServer(); + const response = await postResponses(server, { + stream: true, + tools: [ + { + type: "custom", + name: "apply_patch", + format: { type: "grammar", syntax: "lark", definition: "start: /.+/" }, + }, + ], + input: [ + makeUserInput( + "tool search qa check target=apply_patch. Call apply_patch exactly once and then summarize.", + ), + ], + }); + + expect(response.status).toBe(200); + const body = await response.text(); + const events = body + .split("\n") + .filter((line) => line.startsWith("data: {")) + .map( + (line) => + JSON.parse(line.slice("data: ".length)) as { + type: string; + response?: { id?: string; output?: Array> }; + item?: Record; + item_id?: string; + call_id?: string; + delta?: string; + }, + ); + expect(events.map((event) => event.type)).toEqual([ + "response.created", + "response.output_item.added", + "response.custom_tool_call_input.delta", + "response.output_item.done", + "response.completed", + ]); + const [created, added, delta, done, completed] = events; + expect(created?.response?.id).toBe(completed?.response?.id); + expect(added?.item).toMatchObject({ + type: "custom_tool_call", + name: "apply_patch", + input: "", + status: "in_progress", + }); + expect(done?.item).toMatchObject({ + type: "custom_tool_call", + name: "apply_patch", + status: "completed", + }); + expect(done?.item?.id).toEqual(expect.stringMatching(/^ctc_mock_apply_patch_/)); + expect(delta?.item_id).toBe(done?.item?.id); + expect(delta?.call_id).toBe(done?.item?.call_id); + expect(delta?.delta).toBe(done?.item?.input); + expect(delta?.delta).toContain("runtime-tool-fixture-patch.txt"); + expect(completed?.response?.output).toEqual([done?.item]); + }); + + it.each([ + { + label: "successful native patch", + prompt: + "tool search qa check target=apply_patch. Call apply_patch exactly once and then summarize.", + output: "Successfully applied patch", + expectedOutput: "Successfully applied patch", + structuredError: false, + }, + { + label: "denied native patch", + prompt: + "tool search qa failure target=apply_patch. Exercise the denied-input path once and then summarize.", + output: "Error: Path escapes sandbox root", + expectedOutput: "Error: Path escapes sandbox root", + structuredError: true, + }, + { + label: "upstream Codex native patch rejection without a wire error flag", + prompt: + "tool search qa failure target=apply_patch. Exercise the denied-input path once and then summarize.", + output: "patch rejected: writing outside of the project; rejected by user approval settings", + expectedOutput: + "patch rejected: writing outside of the project; rejected by user approval settings", + structuredError: false, + }, + { + label: "structured native patch output", + prompt: + "tool search qa check target=apply_patch. Call apply_patch exactly once and then summarize.", + output: [{ type: "input_text", text: "Successfully applied structured patch" }], + expectedOutput: "Successfully applied structured patch", + structuredError: false, + }, + ])("links and completes $label custom-tool outputs", async (testCase) => { + const server = await startMockServer(); + const planResponse = await postResponses(server, { + stream: false, + input: [makeUserInput(testCase.prompt)], + }); + expect(planResponse.status).toBe(200); + const plannedCall = outputItem(await planResponse.json()); + expect(plannedCall).toMatchObject({ type: "function_call", name: "apply_patch" }); + const callId = outputToolCallId(plannedCall, "native-patch-call"); + + const continuationResponse = await postResponses(server, { + stream: false, + input: [ + makeUserInput(testCase.prompt), + { + type: "custom_tool_call_output", + call_id: callId, + output: testCase.output, + ...(testCase.structuredError ? { is_error: true } : {}), + }, + ], + }); + expect(continuationResponse.status).toBe(200); + expect(outputItem(await continuationResponse.json()).type).toBe("message"); + + const debugResponse = await fetch(`${server.baseUrl}/debug/last-request`); + expect(debugResponse.status).toBe(200); + const debug = requireRecord(await debugResponse.json(), "custom patch debug request"); + expect(debug.toolOutput).toBe(testCase.expectedOutput); + expect(debug.toolOutputCallId).toBe(callId); + if (testCase.structuredError) { + expect(debug.toolOutputStructuredError).toBe(true); + } else { + expect(debug).not.toHaveProperty("toolOutputStructuredError"); + } + expect(debug).not.toHaveProperty("plannedToolName"); + }); + it("plans QA subagent handoff calls even when Codex dynamic tools are not in body.tools", async () => { const server = await startMockServer(); @@ -4411,6 +4632,49 @@ describe("qa mock openai server", () => { ); }); + it("advertises directly executable native Codex metadata alongside OpenAI models", async () => { + const server = await startMockServer({ + modelRefs: ["mock-openai/gpt-5.6-luna", "mock-openai/gpt-5.6-luna-alt"], + }); + + const response = await fetch(`${server.baseUrl}/v1/models?client_version=0.142.0`); + expect(response.status).toBe(200); + const body = (await response.json()) as { + data: Array<{ id: string; object: string }>; + models: Array>; + }; + expect(body.data).toEqual( + expect.arrayContaining([ + { id: "gpt-5.6-luna", object: "model" }, + { id: "gpt-5.6-luna-alt", object: "model" }, + ]), + ); + expect(body.models).toHaveLength(2); + expect(body.models).toEqual([ + expect.objectContaining({ + slug: "gpt-5.6-luna", + display_name: "gpt-5.6-luna", + apply_patch_tool_type: "freeform", + tool_mode: "direct", + shell_type: "shell_command", + visibility: "list", + supported_in_api: true, + base_instructions: expect.any(String), + truncation_policy: { mode: "tokens", limit: 10_000 }, + supported_reasoning_levels: expect.arrayContaining([ + { effort: "medium", description: "Balanced QA reasoning" }, + ]), + experimental_supported_tools: [], + input_modalities: ["text", "image"], + }), + expect.objectContaining({ + slug: "gpt-5.6-luna-alt", + apply_patch_tool_type: "freeform", + tool_mode: "direct", + }), + ]); + }); + it("serves deterministic OpenAI-compatible audio transcription responses", async () => { const server = await startMockServer(); diff --git a/extensions/qa-lab/src/providers/mock-openai/server.ts b/extensions/qa-lab/src/providers/mock-openai/server.ts index 4b548b2a358e..8722a9345b4e 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.ts @@ -4,7 +4,10 @@ import { setTimeout as sleep } from "node:timers/promises"; import { closeQaHttpServer } from "../../bus-server.js"; import { parseQaDebugRequestCursor } from "../shared/debug-request-cursor.js"; import { writeJson } from "../shared/http-json.js"; -import { listMockOpenAiServerModelIds } from "../shared/mock-model-config.js"; +import { + listMockCodexModelInfos, + listMockOpenAiServerModelIds, +} from "../shared/mock-model-config.js"; import { buildMessagesPayload } from "./mock-anthropic-messages.js"; import { buildAssistantText } from "./mock-openai-assistant-text.js"; import { @@ -91,6 +94,7 @@ import { shouldUseWhatsAppContactMarker, shouldUseWhatsAppStickerMarker, extractBlockStreamingMarkerDirectives, + hasDeclaredCustomTool, hasDeclaredTool, hasToolDefinition, isQaToolSearchFixture, @@ -135,9 +139,14 @@ import { extractLatestImageUserTurn, parseToolOutputJson, } from "./mock-openai-input.js"; +import { + attachQaMockResponsesWebSocketServer, + type QaMockResponsesDispatchResult, +} from "./mock-openai-responses-websocket.js"; import { readTargetFromPrompt, execCommandFromToolProgressPrompt, + buildCustomToolCallEventsWithInput, buildToolCallEventsWithArgs, extractOrbitCode, extractToolSearchTarget, @@ -247,6 +256,13 @@ async function buildResponsesPayload( const plannedArgs = targetTool ? buildQaToolSearchArgs(targetTool, QA_TOOL_SEARCH_FAILURE_PROMPT_RE.test(allInputText)) : {}; + if ( + targetTool === "apply_patch" && + hasDeclaredCustomTool(body, targetTool) && + typeof plannedArgs.input === "string" + ) { + return buildCustomToolCallEventsWithInput(targetTool, plannedArgs.input); + } if (targetTool && hasDeclaredTool(body, "tool_search_code")) { return buildToolCallEventsWithArgs("tool_search_code", { code: [ @@ -1299,6 +1315,59 @@ export async function startQaMockOpenAiServer(params?: { const inflightRequests = new Map(); let nextInflightRequestId = 1; const imageGenerationRequests: Array> = []; + const dispatchResponses = async (request: { + body: Record; + raw: string; + }): Promise => { + const input = Array.isArray(request.body.input) + ? (request.body.input as ResponsesInputItem[]) + : []; + if (isRemoteCompactionV2Request(input)) { + return { events: buildRemoteCompactionV2Events() }; + } + const prompt = extractLastUserText(input); + const allInputText = extractAllRequestTexts(input, request.body); + const inflightRequestId = nextInflightRequestId++; + inflightRequests.set(inflightRequestId, { prompt, allInputText }); + let events: StreamEvent[]; + try { + events = await buildResponsesPayload(request.body, scenarioState); + } finally { + inflightRequests.delete(inflightRequestId); + } + const resolvedModel = typeof request.body.model === "string" ? request.body.model : ""; + recordRequest({ + raw: request.raw, + body: request.body, + prompt, + allInputText, + instructions: extractInstructionsText(request.body) || undefined, + toolOutput: extractToolOutput(input), + model: resolvedModel, + providerVariant: resolveProviderVariant(resolvedModel), + imageInputCount: countImageInputs(input), + plannedToolCallId: extractPlannedToolCallId(events), + plannedToolName: extractPlannedToolName(events), + plannedToolArgs: extractPlannedToolArgs(events), + toolOutputCallId: extractToolOutputCallId(input) || undefined, + ...(extractToolOutputStructuredError(input) ? { toolOutputStructuredError: true } : {}), + }); + return { + events, + ...(QA_PROVIDER_HTTP_503_AFTER_TOOL_PROMPT_RE.test(allInputText) && extractToolOutput(input) + ? { + failure: { + status: 503, + type: "server_error", + message: "Service Unavailable", + }, + } + : {}), + ...(QA_FINAL_ONLY_MARKER_STREAMING_PROMPT_RE.test(allInputText) + ? { previewPauseMs: finalOnlyMarkerPauseMs } + : {}), + }; + }; const server = createServer((req, res) => { void (async () => { const url = new URL(req.url ?? "/", "http://127.0.0.1"); @@ -1312,6 +1381,7 @@ export async function startQaMockOpenAiServer(params?: { id, object: "model", })), + models: listMockCodexModelInfos(params?.modelRefs), }); return; } @@ -1439,45 +1509,17 @@ export async function startQaMockOpenAiServer(params?: { } return; } - const prompt = extractLastUserText(input); - const allInputText = extractAllRequestTexts(input, body); - const inflightRequestId = nextInflightRequestId++; - inflightRequests.set(inflightRequestId, { prompt, allInputText }); - let events: StreamEvent[]; - try { - events = await buildResponsesPayload(body, scenarioState); - } finally { - inflightRequests.delete(inflightRequestId); - } - const resolvedModel = typeof body.model === "string" ? body.model : ""; - recordRequest({ - raw, - body, - prompt, - allInputText, - instructions: extractInstructionsText(body) || undefined, - toolOutput: extractToolOutput(input), - model: resolvedModel, - providerVariant: resolveProviderVariant(resolvedModel), - imageInputCount: countImageInputs(input), - plannedToolCallId: extractPlannedToolCallId(events), - plannedToolName: extractPlannedToolName(events), - plannedToolArgs: extractPlannedToolArgs(events), - toolOutputCallId: extractToolOutputCallId(input) || undefined, - ...(extractToolOutputStructuredError(input) ? { toolOutputStructuredError: true } : {}), - }); - if ( - QA_PROVIDER_HTTP_503_AFTER_TOOL_PROMPT_RE.test(allInputText) && - extractToolOutput(input) - ) { - writeJson(res, 503, { + const dispatched = await dispatchResponses({ body, raw }); + if (dispatched.failure) { + writeJson(res, dispatched.failure.status, { error: { - type: "server_error", - message: "Service Unavailable", + type: dispatched.failure.type, + message: dispatched.failure.message, }, }); return; } + const { events } = dispatched; if (body.stream === false) { const completion = events.at(-1); if (!completion || completion.type !== "response.completed") { @@ -1487,8 +1529,8 @@ export async function startQaMockOpenAiServer(params?: { writeJson(res, 200, completion.response); return; } - if (QA_FINAL_ONLY_MARKER_STREAMING_PROMPT_RE.test(allInputText)) { - await writeSseWithPreviewPause(res, events, finalOnlyMarkerPauseMs); + if (dispatched.previewPauseMs !== undefined) { + await writeSseWithPreviewPause(res, events, dispatched.previewPauseMs); } else { writeSse(res, events); } @@ -1545,6 +1587,10 @@ export async function startQaMockOpenAiServer(params?: { writeJson(res, 404, { error: "not found" }); })(); }); + const responsesWebSocket = attachQaMockResponsesWebSocketServer({ + server, + dispatch: dispatchResponses, + }); await new Promise((resolve, reject) => { server.once("error", reject); @@ -1559,6 +1605,7 @@ export async function startQaMockOpenAiServer(params?: { return { baseUrl: `http://${host}:${address.port}`, async stop() { + await responsesWebSocket.close(); await closeQaHttpServer(server); }, }; diff --git a/extensions/qa-lab/src/providers/shared/mock-model-config.ts b/extensions/qa-lab/src/providers/shared/mock-model-config.ts index ab8e333bac51..215c167cb1ef 100644 --- a/extensions/qa-lab/src/providers/shared/mock-model-config.ts +++ b/extensions/qa-lab/src/providers/shared/mock-model-config.ts @@ -137,3 +137,43 @@ export function listMockOpenAiServerModelIds(selectedModelRefs: readonly string[ "claude-sonnet-4-6", ]; } + +/** Codex consumes full model metadata, while OpenAI clients consume the sibling `data` list. */ +export function listMockCodexModelInfos(selectedModelRefs: readonly string[] = []) { + return selectedOpenAiModelIds("mock-openai", selectedModelRefs).map((slug, priority) => ({ + slug, + display_name: slug, + description: "QA mock OpenAI coding model", + default_reasoning_level: "medium", + supported_reasoning_levels: [ + { effort: "low", description: "Fast QA reasoning" }, + { effort: "medium", description: "Balanced QA reasoning" }, + { effort: "high", description: "Thorough QA reasoning" }, + { effort: "xhigh", description: "Extra-thorough QA reasoning" }, + ], + shell_type: "shell_command", + visibility: "list", + supported_in_api: true, + priority, + availability_nux: null, + upgrade: null, + base_instructions: "You are Codex, a coding agent based on GPT-5.", + include_skills_usage_instructions: false, + supports_reasoning_summaries: true, + default_reasoning_summary: "none", + support_verbosity: true, + default_verbosity: "low", + apply_patch_tool_type: "freeform", + web_search_tool_type: "text_and_image", + truncation_policy: { mode: "tokens", limit: 10_000 }, + supports_parallel_tool_calls: true, + supports_image_detail_original: true, + context_window: 128_000, + max_context_window: 128_000, + effective_context_window_percent: 95, + experimental_supported_tools: [], + input_modalities: ["text", "image"], + supports_search_tool: true, + tool_mode: "direct", + })); +} diff --git a/extensions/qa-lab/src/runtime-parity-timing.ts b/extensions/qa-lab/src/runtime-parity-timing.ts new file mode 100644 index 000000000000..e4cca765f1d2 --- /dev/null +++ b/extensions/qa-lab/src/runtime-parity-timing.ts @@ -0,0 +1,148 @@ +import type { RuntimeId } from "./runtime-parity.js"; + +type QaRuntimeWallClockMetrics = { + totalWallClockMs: number | null; + p50WallClockMs: number | null; + p90WallClockMs: number | null; +}; + +type QaRuntimeSpeedComparison = { + fasterRuntime: RuntimeId | "tie" | null; + speedupPercent: number | null; +}; + +export type QaRuntimeTiming = QaRuntimeSpeedComparison & { + openclaw: QaRuntimeWallClockMetrics; + codex: QaRuntimeWallClockMetrics; + bootstrap?: { + openclaw: QaRuntimeWallClockMetrics; + codex: QaRuntimeWallClockMetrics; + }; +}; + +export type QaRuntimeParityCellTiming = { + wallClockMs: number; + bootstrapWallClockMs: number; +}; + +export function measureRuntimeParityCellTiming(params: { + suiteStartedAt: Date; + bootstrapFinishedAt?: Date; + scenarioStartedAt: Date; + scenarioFinishedAt: Date; +}): QaRuntimeParityCellTiming { + return { + // Gateway/provider startup is harness bootstrap, not an agent turn. Keep + // both measurements so a faster report cannot hide cold-start regressions. + wallClockMs: Math.max( + 1, + params.scenarioFinishedAt.getTime() - params.scenarioStartedAt.getTime(), + ), + bootstrapWallClockMs: Math.max( + 0, + (params.bootstrapFinishedAt ?? params.scenarioStartedAt).getTime() - + params.suiteStartedAt.getTime(), + ), + }; +} + +export function compareRuntimeWallClockMs( + openclawWallClockMs: number | null, + codexWallClockMs: number | null, +): QaRuntimeSpeedComparison { + if (openclawWallClockMs === null || codexWallClockMs === null) { + return { fasterRuntime: null, speedupPercent: null }; + } + if (openclawWallClockMs === codexWallClockMs) { + return { fasterRuntime: "tie", speedupPercent: 0 }; + } + const fasterRuntime = openclawWallClockMs < codexWallClockMs ? "openclaw" : "codex"; + const fasterWallClockMs = Math.min(openclawWallClockMs, codexWallClockMs); + const slowerWallClockMs = Math.max(openclawWallClockMs, codexWallClockMs); + return { + fasterRuntime, + speedupPercent: + fasterWallClockMs === 0 + ? null + : ((slowerWallClockMs - fasterWallClockMs) / fasterWallClockMs) * 100, + }; +} + +function summarizeRuntimeWallClock(values: number[]): QaRuntimeWallClockMetrics { + if (values.length === 0) { + return { totalWallClockMs: null, p50WallClockMs: null, p90WallClockMs: null }; + } + const sorted = values.toSorted((left, right) => left - right); + const percentile = (value: number) => + sorted[Math.min(sorted.length - 1, Math.ceil((value / 100) * sorted.length) - 1)] ?? null; + return { + totalWallClockMs: sorted.reduce((total, value) => total + value, 0), + p50WallClockMs: percentile(50), + p90WallClockMs: percentile(90), + }; +} + +export function summarizeRuntimeParityTiming( + scenarios: readonly { + openclawWallClockMs: number | null; + codexWallClockMs: number | null; + openclawBootstrapWallClockMs?: number | null; + codexBootstrapWallClockMs?: number | null; + }[], +): QaRuntimeTiming { + const openclaw = summarizeRuntimeWallClock( + scenarios.flatMap(({ openclawWallClockMs }) => + openclawWallClockMs === null ? [] : [openclawWallClockMs], + ), + ); + const codex = summarizeRuntimeWallClock( + scenarios.flatMap(({ codexWallClockMs }) => + codexWallClockMs === null ? [] : [codexWallClockMs], + ), + ); + const pairedTimingCaptures = scenarios.flatMap(({ openclawWallClockMs, codexWallClockMs }) => + openclawWallClockMs === null || codexWallClockMs === null + ? [] + : [{ openclawWallClockMs, codexWallClockMs }], + ); + const openclawBootstrapValues = scenarios.flatMap(({ openclawBootstrapWallClockMs }) => + openclawBootstrapWallClockMs == null ? [] : [openclawBootstrapWallClockMs], + ); + const codexBootstrapValues = scenarios.flatMap(({ codexBootstrapWallClockMs }) => + codexBootstrapWallClockMs == null ? [] : [codexBootstrapWallClockMs], + ); + return { + openclaw, + codex, + ...(openclawBootstrapValues.length > 0 || codexBootstrapValues.length > 0 + ? { + bootstrap: { + openclaw: summarizeRuntimeWallClock(openclawBootstrapValues), + codex: summarizeRuntimeWallClock(codexBootstrapValues), + }, + } + : {}), + ...compareRuntimeWallClockMs( + pairedTimingCaptures.length > 0 + ? pairedTimingCaptures.reduce((total, capture) => total + capture.openclawWallClockMs, 0) + : null, + pairedTimingCaptures.length > 0 + ? pairedTimingCaptures.reduce((total, capture) => total + capture.codexWallClockMs, 0) + : null, + ), + }; +} + +export function formatRuntimeWallClockMs(value: number | null): string { + return value === null ? "N/A" : `${value} ms`; +} + +export function formatRuntimeSpeedComparison(comparison: QaRuntimeSpeedComparison): string { + if (comparison.fasterRuntime === null || comparison.speedupPercent === null) { + return "N/A"; + } + if (comparison.fasterRuntime === "tie") { + return "tie"; + } + return `${comparison.fasterRuntime} ${comparison.speedupPercent.toFixed(1)}% faster`; +} diff --git a/extensions/qa-lab/src/runtime-parity.ts b/extensions/qa-lab/src/runtime-parity.ts index bee716606763..2ed9a83bf640 100644 --- a/extensions/qa-lab/src/runtime-parity.ts +++ b/extensions/qa-lab/src/runtime-parity.ts @@ -57,6 +57,7 @@ export type RuntimeParityCell = { finalText: string; usage: RuntimeParityUsage; wallClockMs: number; + bootstrapWallClockMs?: number; transportErrorClass?: string; runtimeErrorClass?: string; bootStateLines: string[]; @@ -160,6 +161,7 @@ type RuntimeParityCaptureParams = { gateway: QaGatewayLike; scenarioResult: QaSuiteScenarioLike; wallClockMs: number; + bootstrapWallClockMs?: number; agentId?: string; mockBaseUrl?: string; }; @@ -1470,6 +1472,9 @@ export async function captureRuntimeParityCell( finalText: extractFinalAssistantText(transcriptRecords), usage: aggregateUsage(transcriptRecords), wallClockMs: params.wallClockMs, + ...(params.bootstrapWallClockMs === undefined + ? {} + : { bootstrapWallClockMs: params.bootstrapWallClockMs }), ...(scenarioErrorClass || sentinelErrorClass ? { runtimeErrorClass: scenarioErrorClass ?? sentinelErrorClass } : {}), diff --git a/extensions/qa-lab/src/runtime-tool-fixture.test.ts b/extensions/qa-lab/src/runtime-tool-fixture.test.ts index 54e3aa78b6f4..e21c1ce3ff37 100644 --- a/extensions/qa-lab/src/runtime-tool-fixture.test.ts +++ b/extensions/qa-lab/src/runtime-tool-fixture.test.ts @@ -13,11 +13,13 @@ import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js"; const tempRoots: string[] = []; async function makeEnv(overrides: Partial = {}): Promise { - const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "runtime-tool-fixture-")); - tempRoots.push(workspaceDir); + const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "runtime-tool-fixture-")); + const workspaceDir = path.join(tempRoot, "workspace"); + await fs.mkdir(workspaceDir); + tempRoots.push(tempRoot); return { - outputDir: workspaceDir, - repoRoot: workspaceDir, + outputDir: tempRoot, + repoRoot: tempRoot, providerMode: "mock-openai", primaryModel: "openai/gpt-5.6-luna", alternateModel: "openai/gpt-5.6-luna", @@ -26,7 +28,7 @@ async function makeEnv(overrides: Partial = {}): Promise, + params: { sessionKey: string }, + contents: string | null = "runtime patch\n", +) { + if (params.sessionKey.endsWith(":happy") && contents !== null) { + await fs.writeFile( + path.join(env.gateway.workspaceDir, "runtime-tool-fixture-patch.txt"), + contents, + "utf8", + ); + } + return {}; +} + async function runMockRuntimeToolFixtureWithOutputs(params: { toolName: string; happyArgs: Record; failureArgs: Record; happyOutput: string; failureOutput: string; + happyPatchContents?: string | null; }) { const env = await makeEnv({ mock: { baseUrl: "http://127.0.0.1:9999" }, @@ -160,7 +260,12 @@ async function runMockRuntimeToolFixtureWithOutputs(params: { { createSession: vi.fn(async (_env, _label, key) => key!), readEffectiveTools: vi.fn(async () => new Set([params.toolName])), - runAgentPrompt: vi.fn(async () => ({})), + runAgentPrompt: vi.fn(async (runEnv, promptParams) => { + if (params.toolName === "apply_patch") { + return simulateRuntimePatchHappyTurn(runEnv, promptParams, params.happyPatchContents); + } + return {}; + }), fetchJson, ensureImageGenerationConfigured: vi.fn(), }, @@ -617,6 +722,573 @@ describe("runtime tool fixture", () => { expect(transcriptToolNames).toEqual([undefined, undefined]); }); + it.each([ + "apply_patch failed: path escapes sandbox root", + "Operation not permitted (os error 1)", + "patch rejected: writing outside of the project; rejected by user approval settings", + ])("verifies native Codex patch success and workspace denial: %s", async (failureOutput) => { + const env = await makeEnv(); + env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME = "codex"; + await writeCodexNativePatchEvidence(env, failureOutput); + const promptEvidence: Array<{ + requireSuccessfulTranscriptToolResult?: boolean; + transcriptToolName?: string; + }> = []; + + const details = await runRuntimeToolFixture( + env, + { + toolName: "apply_patch", + toolCoverage: { + bucket: "codex-native-workspace", + expectedLayer: "codex-native-workspace", + required: true, + }, + }, + { + createSession: vi.fn(async (_env, _label, key) => key!), + readEffectiveTools: vi.fn(async () => new Set()), + runAgentPrompt: vi.fn(async (_env, params) => { + promptEvidence.push({ + transcriptToolName: params.transcriptToolName, + requireSuccessfulTranscriptToolResult: params.requireSuccessfulTranscriptToolResult, + }); + return simulateRuntimePatchHappyTurn(_env, params); + }), + fetchJson: vi.fn(), + ensureImageGenerationConfigured: vi.fn(), + }, + ); + + expect(promptEvidence).toEqual([ + { transcriptToolName: "apply_patch", requireSuccessfulTranscriptToolResult: true }, + { transcriptToolName: "apply_patch", requireSuccessfulTranscriptToolResult: undefined }, + ]); + expect(details).toContain("apply_patch live provider happy planned args"); + expect(details).toContain("runtime-tool-fixture-patch.txt"); + expect(details).toContain("../runtime-tool-fixture-denied.txt"); + await expect( + fs.access(path.resolve(env.gateway.workspaceDir, "../runtime-tool-fixture-denied.txt")), + ).rejects.toThrow(); + await expect( + fs.access(path.join(env.gateway.workspaceDir, "runtime-tool-fixture-patch.txt")), + ).rejects.toThrow(); + }); + + it("rejects native patch transcripts that claim success without creating the workspace file", async () => { + const env = await makeEnv(); + env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME = "codex"; + await writeCodexNativePatchEvidence(env); + + await expect( + runRuntimeToolFixture( + env, + { + toolName: "apply_patch", + toolCoverage: { + bucket: "codex-native-workspace", + expectedLayer: "codex-native-workspace", + required: true, + }, + }, + { + createSession: vi.fn(async (_env, _label, key) => key!), + readEffectiveTools: vi.fn(async () => new Set()), + runAgentPrompt: vi.fn(async () => ({})), + fetchJson: vi.fn(), + ensureImageGenerationConfigured: vi.fn(), + }, + ), + ).rejects.toThrow( + "expected apply_patch to create runtime-tool-fixture-patch.txt with exact contents", + ); + }); + + it("rejects native Codex patch failures that only report missing patch context", async () => { + const env = await makeEnv(); + env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME = "codex"; + await writeCodexNativePatchEvidence( + env, + "apply_patch failed: failed to find expected lines in runtime-tool-fixture-denied.txt", + ); + + await expect( + runRuntimeToolFixture( + env, + { + toolName: "apply_patch", + toolCoverage: { + bucket: "codex-native-workspace", + expectedLayer: "codex-native-workspace", + required: true, + }, + }, + { + createSession: vi.fn(async (_env, _label, key) => key!), + readEffectiveTools: vi.fn(async () => new Set()), + runAgentPrompt: vi.fn(simulateRuntimePatchHappyTurn), + fetchJson: vi.fn(), + ensureImageGenerationConfigured: vi.fn(), + }, + ), + ).rejects.toThrow( + "expected live apply_patch failure to explicitly reject the workspace boundary", + ); + }); + + it("rejects native Codex patch failures without a linked failure result", async () => { + const env = await makeEnv(); + env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME = "codex"; + await writeCodexNativePatchEvidence(env, "apply_patch completed", { + failureStructuredError: false, + }); + + await expect( + runRuntimeToolFixture( + env, + { + toolName: "apply_patch", + toolCoverage: { + bucket: "codex-native-workspace", + expectedLayer: "codex-native-workspace", + required: true, + }, + }, + { + createSession: vi.fn(async (_env, _label, key) => key!), + readEffectiveTools: vi.fn(async () => new Set()), + runAgentPrompt: vi.fn(simulateRuntimePatchHappyTurn), + fetchJson: vi.fn(), + ensureImageGenerationConfigured: vi.fn(), + }, + ), + ).rejects.toThrow("expected live failure-path tool failure output for apply_patch"); + }); + + it.each([ + { + label: "happy-path file", + options: { happyPath: "runtime-tool-fixture-wrong.txt" }, + expectedError: "expected linked live apply_patch to add runtime-tool-fixture-patch.txt", + }, + { + label: "failure-path file", + options: { failurePath: "../runtime-tool-fixture-wrong.txt" }, + expectedError: + "expected linked live apply_patch to update ../runtime-tool-fixture-denied.txt", + }, + { + label: "failure-path operation", + options: { failureKind: "add" }, + expectedError: + "expected linked live apply_patch to update ../runtime-tool-fixture-denied.txt", + }, + ])("rejects linked native Codex patch evidence for the wrong $label", async (testCase) => { + const env = await makeEnv(); + env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME = "codex"; + await writeCodexNativePatchEvidence( + env, + "apply_patch failed: path escapes sandbox root", + testCase.options, + ); + + await expect( + runRuntimeToolFixture( + env, + { + toolName: "apply_patch", + toolCoverage: { + bucket: "codex-native-workspace", + expectedLayer: "codex-native-workspace", + required: true, + }, + }, + { + createSession: vi.fn(async (_env, _label, key) => key!), + readEffectiveTools: vi.fn(async () => new Set()), + runAgentPrompt: vi.fn(simulateRuntimePatchHappyTurn), + fetchJson: vi.fn(), + ensureImageGenerationConfigured: vi.fn(), + }, + ), + ).rejects.toThrow(testCase.expectedError); + }); + + it("validates the native patch call linked to its result instead of the first plan", async () => { + const env = await makeEnv(); + env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME = "codex"; + await writeQaSessionTranscript(env, "agent:qa:runtime-tool:apply_patch:happy", [ + { + role: "assistant", + content: [ + { + type: "toolCall", + id: "native-patch-unlinked-decoy", + name: "apply_patch", + arguments: { + changes: [{ path: "runtime-tool-fixture-wrong.txt", kind: { type: "add" } }], + }, + }, + ], + }, + ]); + await writeCodexNativePatchEvidence(env); + + await expect( + runRuntimeToolFixture( + env, + { + toolName: "apply_patch", + toolCoverage: { + bucket: "codex-native-workspace", + expectedLayer: "codex-native-workspace", + required: true, + }, + }, + { + createSession: vi.fn(async (_env, _label, key) => key!), + readEffectiveTools: vi.fn(async () => new Set()), + runAgentPrompt: vi.fn(simulateRuntimePatchHappyTurn), + fetchJson: vi.fn(), + ensureImageGenerationConfigured: vi.fn(), + }, + ), + ).resolves.toContain("apply_patch live provider happy planned args"); + }); + + it("fails closed and cleans up when a patch changes the outside-workspace sentinel", async () => { + const env = await makeEnv(); + const sentinelPath = path.resolve( + env.gateway.workspaceDir, + "../runtime-tool-fixture-denied.txt", + ); + + await expect( + runRuntimeToolFixture( + env, + { + toolName: "apply_patch", + toolCoverage: { + bucket: "openclaw-dynamic-integration", + expectedLayer: "openclaw-dynamic", + }, + }, + { + createSession: vi.fn(async (_env, _label, key) => key!), + readEffectiveTools: vi.fn(async () => new Set(["apply_patch"])), + runAgentPrompt: vi.fn(async (_env, params) => { + if (params.sessionKey.endsWith(":failure")) { + expect(await fs.readFile(sentinelPath, "utf8")).toBe( + "runtime-tool-fixture-denied-original\n", + ); + await fs.writeFile(sentinelPath, "runtime patch outside the workspace\n", "utf8"); + } + return simulateRuntimePatchHappyTurn(_env, params); + }), + fetchJson: vi.fn(), + ensureImageGenerationConfigured: vi.fn(), + }, + ), + ).rejects.toThrow("apply_patch modified or removed the outside-workspace sentinel"); + + await expect(fs.access(sentinelPath)).rejects.toThrow(); + }); + + it("fails closed when required native Codex patch execution has no linked transcript", async () => { + const env = await makeEnv(); + env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME = "codex"; + await writeQaSessionTranscript(env, "agent:qa:runtime-tool:apply_patch:happy", [ + { role: "assistant", content: "The patch was applied." }, + ]); + await writeQaSessionTranscript(env, "agent:qa:runtime-tool:apply_patch:failure", [ + { role: "assistant", content: "The unsafe patch was rejected." }, + ]); + + await expect( + runRuntimeToolFixture( + env, + { + toolName: "apply_patch", + toolCoverage: { + bucket: "codex-native-workspace", + expectedLayer: "codex-native-workspace", + required: true, + }, + }, + { + createSession: vi.fn(async (_env, _label, key) => key!), + readEffectiveTools: vi.fn(async () => new Set()), + runAgentPrompt: vi.fn(simulateRuntimePatchHappyTurn), + fetchJson: vi.fn(), + ensureImageGenerationConfigured: vi.fn(), + }, + ), + ).rejects.toThrow("expected live happy-path tool call for apply_patch"); + }); + + it.each([ + { label: "dynamically exposed", dynamicPatchExposed: true }, + { label: "native-only", dynamicPatchExposed: false }, + ])("verifies $label private-QA Codex patch calls without skipping them", async (testCase) => { + const env = await makeEnv({ + mock: { baseUrl: "http://127.0.0.1:9999" }, + }); + env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME = "codex"; + const promptEvidence: Array<{ + requireSuccessfulTranscriptToolResult?: boolean; + transcriptToolName?: string; + }> = []; + const happyCallId = "private-qa-patch-happy"; + const failureCallId = "private-qa-patch-failure"; + const fetchJson = vi + .fn() + .mockResolvedValueOnce({ cursor: 0 }) + .mockResolvedValueOnce([ + { + allInputText: "target=apply_patch", + plannedToolCallId: happyCallId, + plannedToolName: "apply_patch", + plannedToolArgs: { + input: + "*** Begin Patch\n*** Add File: runtime-tool-fixture-patch.txt\n+runtime patch\n*** End Patch\n", + }, + }, + { + allInputText: "target=apply_patch", + toolOutputCallId: happyCallId, + toolOutput: "Successfully applied patch", + }, + { + allInputText: "failure target=apply_patch", + plannedToolCallId: failureCallId, + plannedToolName: "apply_patch", + plannedToolArgs: { + input: + "*** Begin Patch\n*** Update File: ../runtime-tool-fixture-denied.txt\n@@\n-runtime-tool-fixture-denied-original\n+runtime patch outside the workspace\n*** End Patch\n", + }, + }, + { + allInputText: "failure target=apply_patch", + toolOutputCallId: failureCallId, + toolOutput: "Error: Path escapes sandbox root", + }, + ]); + + const details = await runRuntimeToolFixture( + env, + { + toolName: "apply_patch", + toolCoverage: { + bucket: "codex-native-workspace", + expectedLayer: "codex-native-workspace", + required: true, + }, + promptSnippet: "target=apply_patch", + failurePromptSnippet: "failure target=apply_patch", + }, + { + createSession: vi.fn(async (_env, _label, key) => key!), + readEffectiveTools: vi.fn( + async () => new Set(testCase.dynamicPatchExposed ? ["apply_patch"] : []), + ), + runAgentPrompt: vi.fn(async (_env, params) => { + promptEvidence.push({ + transcriptToolName: params.transcriptToolName, + requireSuccessfulTranscriptToolResult: params.requireSuccessfulTranscriptToolResult, + }); + return simulateRuntimePatchHappyTurn(_env, params); + }), + fetchJson, + ensureImageGenerationConfigured: vi.fn(), + }, + ); + + expect(promptEvidence).toEqual([ + { transcriptToolName: undefined, requireSuccessfulTranscriptToolResult: undefined }, + { transcriptToolName: undefined, requireSuccessfulTranscriptToolResult: undefined }, + ]); + expect(details).toContain("apply_patch mock provider happy planned args"); + expect(details).toContain("runtime-tool-fixture-patch.txt"); + expect(details).toContain("../runtime-tool-fixture-denied.txt"); + expect(details).not.toContain("codex-native-workspace apply_patch"); + await expect( + fs.access(path.resolve(env.gateway.workspaceDir, "../runtime-tool-fixture-denied.txt")), + ).rejects.toThrow(); + await expect( + fs.access(path.join(env.gateway.workspaceDir, "runtime-tool-fixture-patch.txt")), + ).rejects.toThrow(); + }); + + it.each([ + "Operation not permitted", + "Operation not permitted (os error 1)", + "EPERM: sandbox denied the requested patch", + "patch rejected: writing outside of the project; rejected by user approval settings", + ])("accepts native sandbox denial as a mock patch failure: %s", async (failureOutput) => { + await expect( + runMockRuntimeToolFixtureWithOutputs({ + toolName: "apply_patch", + happyArgs: { + input: + "*** Begin Patch\n*** Add File: runtime-tool-fixture-patch.txt\n+runtime patch\n*** End Patch\n", + }, + failureArgs: { + input: + "*** Begin Patch\n*** Update File: ../runtime-tool-fixture-denied.txt\n@@\n-runtime-tool-fixture-denied-original\n+runtime patch outside the workspace\n*** End Patch\n", + }, + happyOutput: "Successfully applied patch", + failureOutput, + }), + ).resolves.toContain("apply_patch mock provider failure planned args"); + }); + + it("rejects mock patch failures that only report missing patch context", async () => { + await expect( + runMockRuntimeToolFixtureWithOutputs({ + toolName: "apply_patch", + happyArgs: { + input: + "*** Begin Patch\n*** Add File: runtime-tool-fixture-patch.txt\n+runtime patch\n*** End Patch\n", + }, + failureArgs: { + input: + "*** Begin Patch\n*** Update File: ../runtime-tool-fixture-denied.txt\n@@\n-runtime-tool-fixture-denied-original\n+runtime patch outside the workspace\n*** End Patch\n", + }, + happyOutput: "Successfully applied patch", + failureOutput: "Error: failed to find expected lines in runtime-tool-fixture-denied.txt", + }), + ).rejects.toThrow( + "expected mock apply_patch failure to explicitly reject the workspace boundary", + ); + }); + + it.each([ + { label: "no workspace mutation", happyPatchContents: null }, + { label: "incorrect workspace contents", happyPatchContents: "a fabricated patch\n" }, + ])("rejects successful linked mock patch claims with $label", async (testCase) => { + await expect( + runMockRuntimeToolFixtureWithOutputs({ + toolName: "apply_patch", + happyArgs: { + input: + "*** Begin Patch\n*** Add File: runtime-tool-fixture-patch.txt\n+runtime patch\n*** End Patch\n", + }, + failureArgs: { + input: + "*** Begin Patch\n*** Update File: ../runtime-tool-fixture-denied.txt\n@@\n-runtime-tool-fixture-denied-original\n+runtime patch outside the workspace\n*** End Patch\n", + }, + happyOutput: "Successfully applied patch", + failureOutput: "Error: Path escapes sandbox root", + happyPatchContents: testCase.happyPatchContents, + }), + ).rejects.toThrow( + "expected apply_patch to create runtime-tool-fixture-patch.txt with exact contents", + ); + }); + + it.each([ + { + label: "happy-path file", + happyInput: + "*** Begin Patch\n*** Add File: runtime-tool-fixture-wrong.txt\n+runtime patch\n*** End Patch\n", + failureInput: + "*** Begin Patch\n*** Update File: ../runtime-tool-fixture-denied.txt\n@@\n-runtime-tool-fixture-denied-original\n+runtime patch outside the workspace\n*** End Patch\n", + expectedError: "expected linked mock apply_patch to add runtime-tool-fixture-patch.txt", + }, + { + label: "failure-path file", + happyInput: + "*** Begin Patch\n*** Add File: runtime-tool-fixture-patch.txt\n+runtime patch\n*** End Patch\n", + failureInput: + "*** Begin Patch\n*** Update File: ../runtime-tool-fixture-wrong.txt\n@@\n-runtime-tool-fixture-denied-original\n+runtime patch outside the workspace\n*** End Patch\n", + expectedError: + "expected linked mock apply_patch to update ../runtime-tool-fixture-denied.txt", + }, + { + label: "failure-path context", + happyInput: + "*** Begin Patch\n*** Add File: runtime-tool-fixture-patch.txt\n+runtime patch\n*** End Patch\n", + failureInput: + "*** Begin Patch\n*** Update File: ../runtime-tool-fixture-denied.txt\n@@\n-context-that-does-not-exist\n+runtime patch outside the workspace\n*** End Patch\n", + expectedError: + "expected linked mock apply_patch to update ../runtime-tool-fixture-denied.txt", + }, + ])("rejects linked mock patch evidence for the wrong $label", async (testCase) => { + await expect( + runMockRuntimeToolFixtureWithOutputs({ + toolName: "apply_patch", + happyArgs: { input: testCase.happyInput }, + failureArgs: { input: testCase.failureInput }, + happyOutput: "Successfully applied patch", + failureOutput: "Error: Path escapes sandbox root", + }), + ).rejects.toThrow(testCase.expectedError); + }); + + it("rejects unlinked private-QA Codex patch results without waiting for a transcript", async () => { + const env = await makeEnv({ + mock: { baseUrl: "http://127.0.0.1:9999" }, + }); + env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME = "codex"; + const promptEvidence: Array<{ + requireSuccessfulTranscriptToolResult?: boolean; + transcriptToolName?: string; + }> = []; + const fetchJson = vi + .fn() + .mockResolvedValueOnce({ cursor: 0 }) + .mockResolvedValueOnce([ + { + allInputText: "target=apply_patch", + plannedToolCallId: "private-qa-patch-happy", + plannedToolName: "apply_patch", + plannedToolArgs: { + input: + "*** Begin Patch\n*** Add File: runtime-tool-fixture-patch.txt\n+runtime patch\n*** End Patch\n", + }, + }, + { + allInputText: "target=apply_patch", + toolOutputCallId: "unrelated-patch-call", + toolOutput: "Successfully applied patch", + }, + ]); + + await expect( + runRuntimeToolFixture( + env, + { + toolName: "apply_patch", + toolCoverage: { + bucket: "codex-native-workspace", + expectedLayer: "codex-native-workspace", + required: true, + }, + promptSnippet: "target=apply_patch", + failurePromptSnippet: "failure target=apply_patch", + }, + { + createSession: vi.fn(async (_env, _label, key) => key!), + readEffectiveTools: vi.fn(async () => new Set(["apply_patch"])), + runAgentPrompt: vi.fn(async (_env, params) => { + promptEvidence.push({ + transcriptToolName: params.transcriptToolName, + requireSuccessfulTranscriptToolResult: params.requireSuccessfulTranscriptToolResult, + }); + return simulateRuntimePatchHappyTurn(_env, params); + }), + fetchJson, + ensureImageGenerationConfigured: vi.fn(), + }, + ), + ).rejects.toThrow("expected mock happy-path tool output for apply_patch"); + + expect(promptEvidence).toEqual([ + { transcriptToolName: undefined, requireSuccessfulTranscriptToolResult: undefined }, + { transcriptToolName: undefined, requireSuccessfulTranscriptToolResult: undefined }, + ]); + }); + it("skips Codex-native async planned-only fixtures without treating the plan as proof", async () => { const env = await makeEnv({ mock: { baseUrl: "http://127.0.0.1:9999" }, diff --git a/extensions/qa-lab/src/runtime-tool-fixture.ts b/extensions/qa-lab/src/runtime-tool-fixture.ts index 359168ede4fa..c681505fe932 100644 --- a/extensions/qa-lab/src/runtime-tool-fixture.ts +++ b/extensions/qa-lab/src/runtime-tool-fixture.ts @@ -33,6 +33,7 @@ type QaRuntimeToolFixtureConfig = Record & { }; type QaRuntimeToolFixtureRequest = { + body?: unknown; allInputText?: string; plannedToolCallId?: string; plannedToolName?: string; @@ -57,6 +58,12 @@ type QaRuntimeToolFixtureTranscriptToolResult = { }; const RUNTIME_PARITY_SESSION_KEY_DETAIL_PREFIX = "RUNTIME_PARITY_SESSION_KEY="; +const RUNTIME_PATCH_HAPPY_FILENAME = "runtime-tool-fixture-patch.txt"; +const RUNTIME_PATCH_HAPPY_CONTENTS = "runtime patch\n"; +const RUNTIME_PATCH_DENIED_FILENAME = "runtime-tool-fixture-denied.txt"; +const RUNTIME_PATCH_DENIED_CONTENTS = "runtime-tool-fixture-denied-original\n"; +const RUNTIME_PATCH_WORKSPACE_DENIAL_RE = + /(?:path\s+escapes\s+(?:the\s+)?(?:sandbox|workspace)(?:\s+root)?|outside(?:\s+of)?\s+(?:the\s+)?(?:project|sandbox|workspace|allowed\s+(?:sandbox|workspace|root)|writable\s+roots?)(?:\s+root)?|workspace[- ]only|permission\s+denied|operation\s+not\s+permitted|\bos\s+error\s+1\b|\b(?:EACCES|EPERM)\b)/iu; function runtimeParitySessionKeyDetails(...sessionKeys: string[]) { return sessionKeys.map( @@ -164,6 +171,148 @@ function requestHasFailureLikeToolOutput(request: QaRuntimeToolFixtureRequest) { ); } +function isWorkspaceBoundaryFailureToolOutput(text: unknown) { + return typeof text === "string" && RUNTIME_PATCH_WORKSPACE_DENIAL_RE.test(text); +} + +function formatRuntimePatchFailureOutput(request: QaRuntimeToolFixtureRequest): string { + const text = + typeof request.toolOutput === "string" + ? request.toolOutput + .replace( + /\b(?:bearer\s+[a-z\d._~+/-]+=*|(?:api[_-]?key|access[_-]?token|authorization|password|secret)\s*[:=]\s*["']?[^\s"',;]+)/giu, + "[REDACTED]", + ) + .replace( + /\b(?:sk|sess|ghp|gho|github_pat|xox[baprs])[-_][a-z\d_-]{8,}\b/giu, + "[REDACTED]", + ) + .slice(0, 240) + : undefined; + return JSON.stringify({ text, structuredError: request.toolOutputStructuredError === true }); +} + +function matchesRuntimePatchInput(input: unknown, operation: "add" | "update"): boolean { + if (typeof input !== "string") { + return false; + } + const lines = input.replace(/\r\n?/gu, "\n").split("\n"); + while (lines.at(-1) === "") { + lines.pop(); + } + if (lines[0] !== "*** Begin Patch" || lines.at(-1) !== "*** End Patch") { + return false; + } + const fileHeaders = lines.filter((line) => /^\*\*\* (?:Add|Update|Delete) File: /u.test(line)); + const expectedHeader = + operation === "add" + ? `*** Add File: ${RUNTIME_PATCH_HAPPY_FILENAME}` + : `*** Update File: ../${RUNTIME_PATCH_DENIED_FILENAME}`; + if (fileHeaders.length !== 1 || fileHeaders[0] !== expectedHeader) { + return false; + } + return operation === "add" + ? lines.includes("+runtime patch") + : lines.includes("@@") && + lines.includes(`-${RUNTIME_PATCH_DENIED_CONTENTS.trimEnd()}`) && + lines.includes("+runtime patch outside the workspace"); +} + +function matchesRuntimePatchArguments(params: { + args: unknown; + workspaceDir: string; + operation: "add" | "update"; +}): boolean { + if (!isRecord(params.args)) { + return false; + } + if (typeof params.args.input === "string") { + return matchesRuntimePatchInput(params.args.input, params.operation); + } + const changes = params.args.changes; + if (!Array.isArray(changes) || changes.length !== 1 || !isRecord(changes[0])) { + return false; + } + const change = changes[0]; + if (typeof change.path !== "string") { + return false; + } + const kind = change.kind; + const operation = isRecord(kind) ? kind.type : kind; + const expectedPath = + params.operation === "add" + ? path.resolve(params.workspaceDir, RUNTIME_PATCH_HAPPY_FILENAME) + : path.resolve(params.workspaceDir, "..", RUNTIME_PATCH_DENIED_FILENAME); + return ( + operation === params.operation && + path.resolve(params.workspaceDir, change.path) === expectedPath + ); +} + +async function formatRuntimePatchMutationDiagnostics(params: { + env: QaSuiteRuntimeEnv; + deps: QaRuntimeToolFixtureDeps; + requestCursor: number; +}) { + const workspaceEntries = await fs + .readdir(params.env.gateway.workspaceDir) + .then((entries) => entries.toSorted().slice(0, 16)) + .catch(() => [] as string[]); + const tempRootEntries = await fs + .readdir(params.env.gateway.tempRoot) + .then((entries) => entries.toSorted().slice(0, 16)) + .catch(() => [] as string[]); + const gatewayPatchLogs = (params.env.gateway.logs?.() ?? "") + .split(/\r?\n/u) + .filter((line) => + /custom tool call output is missing|apply[._ -]?patch|failed to parse responseitem|duplicate|tool registration/iu.test( + line, + ), + ) + .slice(-6) + .map((line) => + line + .replace( + /\b(?:bearer\s+[a-z\d._~+/-]+=*|(?:api[_-]?key|access[_-]?token|authorization|password|secret)\s*[:=]\s*["']?[^\s"',;]+)/giu, + "[REDACTED]", + ) + .replace(/\b(?:sk|sess|ghp|gho|github_pat|xox[baprs])[-_][a-z\d_-]{8,}\b/giu, "[REDACTED]") + .slice(0, 200), + ); + const mockRequests = params.env.mock + ? await params.deps + .fetchJson(qaMockRequestsAfterUrl(params.env.mock.baseUrl, params.requestCursor)) + .then(readQaRuntimeToolFixtureRequests) + .then((requests) => + requests.slice(-4).map((request) => { + const body = isRecord(request.body) ? request.body : undefined; + const tools = Array.isArray(body?.tools) ? body.tools : []; + return { + plannedToolName: request.plannedToolName, + plannedToolCallId: request.plannedToolCallId, + toolOutputCallId: request.toolOutputCallId, + toolOutput: request.toolOutput?.slice(0, 256), + patchToolTypes: tools.flatMap((tool) => + isRecord(tool) && tool.name === "apply_patch" && typeof tool.type === "string" + ? [tool.type] + : [], + ), + }; + }), + ) + .catch(() => []) + : []; + return [ + `workspace=${params.env.gateway.workspaceDir}`, + `workspaceEntries=${JSON.stringify(workspaceEntries)}`, + `tempRootEntries=${JSON.stringify(tempRootEntries)}`, + ...(gatewayPatchLogs.length > 0 + ? [`gatewayPatchLogs=${JSON.stringify(gatewayPatchLogs)}`] + : []), + ...(params.env.mock ? [`mockPatchRequests=${JSON.stringify(mockRequests)}`] : []), + ].join("; "); +} + function readNonEmptyString(value: unknown) { return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; } @@ -276,7 +425,7 @@ function readBooleanTrue(value: unknown) { } const FAILURE_LIKE_TOOL_RESULT_RE = - /\b(?:denied|enoent|error|exception|fail(?:ed|ure)?|forbidden|invalid|missing|not found|permission)\b/iu; + /\b(?:denied|enoent|error|exception|fail(?:ed|ure)?|forbidden|invalid|missing|not found|permission|reject(?:ed|ion)?)\b/iu; const REQUIRED_FIELD_TOOL_RESULT_RE = /(?:^|[\n:,({[]\s*)["']?[A-Z_][A-Z0-9_.[\]-]*["']?\s+(?:is\s+)?required\b/iu; @@ -290,6 +439,7 @@ function isFailureLikeToolResult(params: { return ( isStructuredFailureToolResult(params) || isHardFailureToolOutputText(params.text) || + isWorkspaceBoundaryFailureToolOutput(params.text) || FAILURE_LIKE_TOOL_RESULT_RE.test(params.text) || REQUIRED_FIELD_TOOL_RESULT_RE.test(params.text) ); @@ -419,19 +569,22 @@ function readTranscriptToolEvidence(transcriptBytes: string, toolName: string) { // Ignore malformed transcript rows and keep live fixture evidence deterministic. } } - const outputResult = calls - .map((call) => - results.find((result) => + const linkedEvidence = calls + .map((call) => ({ + call, + result: results.find((result) => transcriptToolResultLinksCall({ call, result, targetCallCount: calls.length, }), ), - ) - .find((result) => result && result.text.trim().length > 0); + })) + .find(({ result }) => result && result.text.trim().length > 0); + const outputResult = linkedEvidence?.result; return { plannedRequest: calls[0], + executedRequest: linkedEvidence?.call, outputRequest: outputResult, failureOutputRequest: outputResult?.failure ? outputResult : undefined, }; @@ -677,7 +830,10 @@ export async function runRuntimeToolFixture( }); const dynamicExposureIntentionallyExcluded = env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === "codex" && - metadata.expectedLayer === "codex-native-workspace"; + metadata.expectedLayer === "codex-native-workspace" && + !tools.has(toolName); + const requireCodexNativePatchCoverage = + dynamicExposureIntentionallyExcluded && metadata.required && toolName === "apply_patch"; const expectedAvailable = readBoolean(config.expectedAvailable, true); if (!tools.has(toolName) && !dynamicExposureIntentionallyExcluded) { if (!expectedAvailable) { @@ -710,9 +866,13 @@ export async function runRuntimeToolFixture( `failure target=${toolName}`, ); const happyPathOutputRequired = readBoolean(config.happyPathOutputRequired, true); + // Private QA can expose an actual dynamic patch tool. Real Codex instead + // mirrors native file changes into linked apply_patch transcript evidence. + const requireNativePatchTranscriptEvidence = !env.mock && requireCodexNativePatchCoverage; const requireTranscriptEvidence = metadata.required && - !dynamicExposureIntentionallyExcluded && + (!env.mock || toolName !== "apply_patch") && + (!dynamicExposureIntentionallyExcluded || requireNativePatchTranscriptEvidence) && !isKnownHarnessGap(config.knownHarnessGap); const mockBaseUrl = env.mock?.baseUrl; const requestCursorBefore = mockBaseUrl @@ -721,24 +881,82 @@ export async function runRuntimeToolFixture( ) : 0; - await runFixtureOperation(() => - deps.runAgentPrompt(env, { - sessionKey: happySessionKey, - message: happyPrompt, - timeoutMs: liveTurnTimeoutMs(env, 45_000), - ...(happyPathOutputRequired && requireTranscriptEvidence - ? { transcriptToolName: toolName, requireSuccessfulTranscriptToolResult: true } - : {}), - }), - ); - await runFixtureOperation(() => - deps.runAgentPrompt(env, { - sessionKey: failureSessionKey, - message: failurePrompt, - timeoutMs: liveTurnTimeoutMs(env, 45_000), - ...(requireTranscriptEvidence ? { transcriptToolName: toolName } : {}), - }), - ); + await runFixtureOperation(async () => { + const runHappyPrompt = () => + deps.runAgentPrompt(env, { + sessionKey: happySessionKey, + message: happyPrompt, + timeoutMs: liveTurnTimeoutMs(env, 45_000), + ...(happyPathOutputRequired && requireTranscriptEvidence + ? { transcriptToolName: toolName, requireSuccessfulTranscriptToolResult: true } + : {}), + }); + if (toolName !== "apply_patch" || !metadata.required) { + return runHappyPrompt(); + } + const happyPatchPath = path.join(env.gateway.workspaceDir, RUNTIME_PATCH_HAPPY_FILENAME); + const readHappyPatchContents = async () => + fs.readFile(happyPatchPath, "utf8").catch((error: unknown) => { + if (isRecord(error) && error.code === "ENOENT") { + return undefined; + } + throw error; + }); + if ((await readHappyPatchContents()) !== undefined) { + throw new Error( + `apply_patch happy-path target already exists: ${RUNTIME_PATCH_HAPPY_FILENAME}`, + ); + } + try { + const result = await runHappyPrompt(); + if ((await readHappyPatchContents()) !== RUNTIME_PATCH_HAPPY_CONTENTS) { + const diagnostics = await formatRuntimePatchMutationDiagnostics({ + env, + deps, + requestCursor: requestCursorBefore, + }); + throw new Error( + `expected apply_patch to create ${RUNTIME_PATCH_HAPPY_FILENAME} with exact contents; ${diagnostics}`, + ); + } + return result; + } finally { + await fs.rm(happyPatchPath, { force: true }); + } + }); + await runFixtureOperation(async () => { + const runFailurePrompt = () => + deps.runAgentPrompt(env, { + sessionKey: failureSessionKey, + message: failurePrompt, + timeoutMs: liveTurnTimeoutMs(env, 45_000), + ...(requireTranscriptEvidence ? { transcriptToolName: toolName } : {}), + }); + if (toolName !== "apply_patch") { + return runFailurePrompt(); + } + const deniedPatchPath = path.resolve( + env.gateway.workspaceDir, + "..", + RUNTIME_PATCH_DENIED_FILENAME, + ); + // Matching outside context makes failure evidence prove containment, not + // merely that apply_patch could not find a file or match a hunk. + await fs.writeFile(deniedPatchPath, RUNTIME_PATCH_DENIED_CONTENTS, { + encoding: "utf8", + flag: "wx", + }); + try { + const result = await runFailurePrompt(); + const sentinelContents = await fs.readFile(deniedPatchPath, "utf8").catch(() => undefined); + if (sentinelContents !== RUNTIME_PATCH_DENIED_CONTENTS) { + throw new Error("apply_patch modified or removed the outside-workspace sentinel"); + } + return result; + } finally { + await fs.rm(deniedPatchPath, { force: true }); + } + }); if (!env.mock) { const happyRequest = await runFixtureOperation(() => @@ -775,6 +993,19 @@ export async function runRuntimeToolFixture( new Error(`expected live happy-path successful tool output for ${toolName}`), ); } + if ( + toolName === "apply_patch" && + metadata.required && + !matchesRuntimePatchArguments({ + args: happyRequest.executedRequest?.args, + workspaceDir: env.gateway.workspaceDir, + operation: "add", + }) + ) { + throw fixtureError( + new Error(`expected linked live apply_patch to add ${RUNTIME_PATCH_HAPPY_FILENAME}`), + ); + } const failureRequest = await runFixtureOperation(() => readLiveToolEvidence({ env, @@ -802,6 +1033,27 @@ export async function runRuntimeToolFixture( new Error(`expected live failure-path tool failure output for ${toolName}`), ); } + if ( + toolName === "apply_patch" && + metadata.required && + !matchesRuntimePatchArguments({ + args: failureRequest.executedRequest?.args, + workspaceDir: env.gateway.workspaceDir, + operation: "update", + }) + ) { + throw fixtureError( + new Error(`expected linked live apply_patch to update ../${RUNTIME_PATCH_DENIED_FILENAME}`), + ); + } + if ( + toolName === "apply_patch" && + !isWorkspaceBoundaryFailureToolOutput(failureRequest.failureOutputRequest?.text) + ) { + throw fixtureError( + new Error("expected live apply_patch failure to explicitly reject the workspace boundary"), + ); + } return withSessionDetails( [ `${toolName} live provider happy planned args (diagnostic only): ${JSON.stringify(happyRequest.plannedRequest?.args ?? {})}`, @@ -875,7 +1127,7 @@ export async function runRuntimeToolFixture( ); } if (!happyRequest && !happyPlannedOnly) { - if (dynamicExposureIntentionallyExcluded) { + if (dynamicExposureIntentionallyExcluded && !requireCodexNativePatchCoverage) { skipFixture( formatCodexNativeWorkspaceDetails({ toolName, @@ -904,8 +1156,22 @@ export async function runRuntimeToolFixture( new Error(`expected mock happy-path successful tool output for ${toolName}`), ); } + if ( + toolName === "apply_patch" && + metadata.required && + happyRequest && + !matchesRuntimePatchArguments({ + args: happyRequest.plannedRequest.plannedToolArgs, + workspaceDir: env.gateway.workspaceDir, + operation: "add", + }) + ) { + throw fixtureError( + new Error(`expected linked mock apply_patch to add ${RUNTIME_PATCH_HAPPY_FILENAME}`), + ); + } if (!failureRequest) { - if (dynamicExposureIntentionallyExcluded) { + if (dynamicExposureIntentionallyExcluded && !requireCodexNativePatchCoverage) { skipFixture( formatCodexNativeWorkspaceDetails({ toolName, @@ -931,10 +1197,39 @@ export async function runRuntimeToolFixture( if (isKnownHarnessGap(config.knownHarnessGap)) { skipFixture(formatKnownHarnessGapDetails(toolName, config)); } - throw fixtureError(new Error(`expected mock failure-path tool failure output for ${toolName}`)); + const patchFailureDiagnostics = + toolName === "apply_patch" + ? `; received ${formatRuntimePatchFailureOutput(failureRequest.outputRequest)}` + : ""; + throw fixtureError( + new Error( + `expected mock failure-path tool failure output for ${toolName}${patchFailureDiagnostics}`, + ), + ); + } + if ( + toolName === "apply_patch" && + metadata.required && + !matchesRuntimePatchArguments({ + args: failureRequest.plannedRequest.plannedToolArgs, + workspaceDir: env.gateway.workspaceDir, + operation: "update", + }) + ) { + throw fixtureError( + new Error(`expected linked mock apply_patch to update ../${RUNTIME_PATCH_DENIED_FILENAME}`), + ); + } + if ( + toolName === "apply_patch" && + !isWorkspaceBoundaryFailureToolOutput(failureRequest.outputRequest.toolOutput) + ) { + throw fixtureError( + new Error("expected mock apply_patch failure to explicitly reject the workspace boundary"), + ); } - if (dynamicExposureIntentionallyExcluded) { + if (dynamicExposureIntentionallyExcluded && !requireCodexNativePatchCoverage) { skipFixture( formatCodexNativeWorkspaceDetails({ toolName, diff --git a/extensions/qa-lab/src/scenario-catalog.test.ts b/extensions/qa-lab/src/scenario-catalog.test.ts index 21dc973c4dad..5bad4b44a92d 100644 --- a/extensions/qa-lab/src/scenario-catalog.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.test.ts @@ -375,8 +375,25 @@ describe("qa scenario catalog", () => { toolCoverage: { bucket: "codex-native-workspace", expectedLayer: "codex-native-workspace", + required: true, }, }); + expect(readQaScenarioExecutionConfig(applyPatch.id)).not.toHaveProperty("knownHarnessGap"); + expect(readQaScenarioExecutionConfig(applyPatch.id)?.happyPrompt).toContain( + "runtime-tool-fixture-patch.txt", + ); + expect(readQaScenarioExecutionConfig(applyPatch.id)?.failurePrompt).toContain( + "../runtime-tool-fixture-denied.txt", + ); + expect(readQaScenarioExecutionConfig(applyPatch.id)?.failurePrompt).toContain( + "runtime-tool-fixture-denied-original", + ); + expect(readQaScenarioExecutionConfig(applyPatch.id)?.failurePrompt).toContain( + "runtime patch outside the workspace", + ); + expect(readQaScenarioExecutionConfig(applyPatch.id)?.failurePrompt).not.toContain( + "missing-context", + ); expect(readQaScenarioExecutionConfig(messageTool.id)).toMatchObject({ toolName: "message", expectedAvailable: false, diff --git a/extensions/qa-lab/src/suite-run-standard.ts b/extensions/qa-lab/src/suite-run-standard.ts index 4e6bfb1c5d08..609498b2db01 100644 --- a/extensions/qa-lab/src/suite-run-standard.ts +++ b/extensions/qa-lab/src/suite-run-standard.ts @@ -1,11 +1,14 @@ import path from "node:path"; -import { setTimeout as sleep } from "node:timers/promises"; import { disposeRegisteredAgentHarnesses } from "openclaw/plugin-sdk/agent-harness"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { startQaGatewayChild } from "./gateway-child.js"; import type { QaLabLatestReport, QaLabScenarioOutcome } from "./lab-server.types.js"; import { sanitizeQaProgressValue as sanitizeQaSuiteProgressValue } from "./progress-format.js"; import { startQaProviderServer } from "./providers/server-runtime.js"; +import { + measureRuntimeParityCellTiming, + type QaRuntimeParityCellTiming, +} from "./runtime-parity-timing.js"; import { captureRuntimeParityCell } from "./runtime-parity.js"; import { type QaSuiteGatewayHeapSnapshot, @@ -176,8 +179,8 @@ export async function runQaFlowSuiteStandard( await waitForGatewayHealthy(activeEnv, transportReadyTimeoutMs); await waitForTransportReady(activeEnv, transportReadyTimeoutMs); }); - await sleep(1_000); const scenarios: QaSuiteScenarioResult[] = []; + let runtimeParityCellTiming: QaRuntimeParityCellTiming | undefined; const liveScenarioOutcomes: QaLabScenarioOutcome[] = selectedScenarios.map((scenario) => ({ id: scenario.id, name: scenario.title, @@ -240,7 +243,19 @@ export async function runQaFlowSuiteStandard( scenarios: [...liveScenarioOutcomes], }); - const runSelectedScenario = () => runScenarioDefinition(activeEnv, scenario); + const scenarioBootstrapFinishedAt = new Date(); + let scenarioExecutionStartedAt = scenarioBootstrapFinishedAt; + let scenarioExecutionFinishedAt = scenarioBootstrapFinishedAt; + const runSelectedScenario = async () => { + // Retry backoff and unsuccessful attempts are not part of the final + // runtime turn, and they must not be relabeled as gateway bootstrap. + scenarioExecutionStartedAt = new Date(); + try { + return await runScenarioDefinition(activeEnv, scenario); + } finally { + scenarioExecutionFinishedAt = new Date(); + } + }; const scenarioRetryCount = scenario.execution.kind === "flow" ? scenario.execution.retryCount : undefined; let result: QaSuiteScenarioResult = @@ -273,6 +288,14 @@ export async function runQaFlowSuiteStandard( ], }; } + if (params?.captureRuntimeParityCell && selectedScenarios.length === 1) { + runtimeParityCellTiming = measureRuntimeParityCellTiming({ + suiteStartedAt: startedAt, + bootstrapFinishedAt: scenarioBootstrapFinishedAt, + scenarioStartedAt: scenarioExecutionStartedAt, + scenarioFinishedAt: scenarioExecutionFinishedAt, + }); + } sampleGatewayProcessRss(`scenario:${scenario.id}:finish`); scenarios.push(result); writeQaSuiteProgress( @@ -304,12 +327,13 @@ export async function runQaFlowSuiteStandard( params?.captureRuntimeParityCell && params.forcedRuntime && selectedScenarios.length === 1 && - runtimeParityScenario + runtimeParityScenario && + runtimeParityCellTiming ? await captureRuntimeParityCell({ runtime: params.forcedRuntime, gateway: activeGateway, scenarioResult: runtimeParityScenario, - wallClockMs: Math.max(1, Date.now() - startedAt.getTime()), + ...runtimeParityCellTiming, mockBaseUrl: activeMock?.baseUrl, }) : undefined; diff --git a/extensions/qa-lab/src/suite-runtime-parity-result.ts b/extensions/qa-lab/src/suite-runtime-parity-result.ts index 95c699ae113d..62eeec18bafb 100644 --- a/extensions/qa-lab/src/suite-runtime-parity-result.ts +++ b/extensions/qa-lab/src/suite-runtime-parity-result.ts @@ -13,6 +13,9 @@ function formatRuntimeParityCellDetails(cell: RuntimeParityCell) { return [ `runtime=${cell.runtime}`, `wallMs=${cell.wallClockMs}`, + ...(cell.bootstrapWallClockMs === undefined + ? [] + : [`bootstrapMs=${cell.bootstrapWallClockMs}`]), `toolCalls=${cell.toolCalls.length}`, `finalChars=${cell.finalText.length}`, `tokens=${cell.usage.totalTokens}`, diff --git a/extensions/qa-lab/src/tool-coverage-report.test.ts b/extensions/qa-lab/src/tool-coverage-report.test.ts index 2a28c48be71e..a23c06319870 100644 --- a/extensions/qa-lab/src/tool-coverage-report.test.ts +++ b/extensions/qa-lab/src/tool-coverage-report.test.ts @@ -892,15 +892,10 @@ describe("qa tool coverage report", () => { expect.objectContaining({ bucket: "codex-native-workspace", expectedLayer: "codex-native-workspace", - required: false, - }), - ); - expect(applyPatchRow).toEqual( - expect.objectContaining({ - tracking: - "#80320 Codex app-server intentionally owns apply_patch natively; this fixture still needs valid patch-shaped fault injection before it can prove product behavior.", + required: true, }), ); + expect(applyPatchRow?.tracking).toBeUndefined(); expect(report.rows.find((row) => row.tool === "sessions_spawn")).toEqual( expect.objectContaining({ required: true, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a59d06d1f080..f4b056cd0848 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1595,6 +1595,9 @@ importers: semver: specifier: 7.8.5 version: 7.8.5 + ws: + specifier: 8.21.1 + version: 8.21.1 yaml: specifier: 2.9.0 version: 2.9.0 diff --git a/qa/scenarios/runtime/tools/apply-patch.yaml b/qa/scenarios/runtime/tools/apply-patch.yaml index a0402c0ac014..42edd5c30e1f 100644 --- a/qa/scenarios/runtime/tools/apply-patch.yaml +++ b/qa/scenarios/runtime/tools/apply-patch.yaml @@ -7,11 +7,11 @@ scenario: coverage: secondary: - agent-runtime.tool-apply-patch - objective: Verify apply_patch behavior is tracked across OpenClaw and Codex while Codex owns patching natively. + objective: Verify OpenClaw and Codex execute valid workspace patches and reject workspace-escaping patches. successCriteria: - - OpenClaw may expose OpenClaw apply_patch while Codex app-server mode may omit duplicate OpenClaw dynamic apply_patch. - - Mock provider apply_patch plans are reported as fixture intent, not as actual runtime tool calls. - - The row stays report-only until fault injection uses valid patch-shaped inputs. + - OpenClaw executes a valid workspace patch and rejects a matching-context workspace-escaping patch without modifying the outside sentinel. + - Private mock QA verifies linked apply_patch calls and results without treating provider plans as execution evidence. + - Live Codex verifies projected native file-change calls and results without requiring a duplicate dynamic apply_patch tool. docsRefs: - qa/scenarios/index.yaml codeRefs: @@ -27,15 +27,22 @@ scenario: actualTool: apply_patch bucket: codex-native-workspace expectedLayer: codex-native-workspace - required: false - tracking: "#80320" + required: true codexDefaultImpact: P4 qaImpact: P2 - action: fix fixture fault injection - reason: Codex app-server intentionally owns apply_patch natively; this fixture still needs valid patch-shaped fault injection before it can prove product behavior. - knownHarnessGap: - issue: "#80320" - reason: Codex-native apply_patch is intentionally not an OpenClaw dynamic tool; QA fault injection still uses synthetic failure-path inputs. + action: require linked successful and workspace-denied patch execution + reason: Codex owns workspace patching natively while private QA may expose an OpenClaw dynamic apply_patch tool. + happyPrompt: >- + tool search qa check target=apply_patch. Call apply_patch exactly once + with a complete patch that adds runtime-tool-fixture-patch.txt containing + runtime patch, then summarize the actual tool result. + failurePrompt: >- + tool search qa failure target=apply_patch. Call apply_patch exactly once + with a complete patch that attempts to update + ../runtime-tool-fixture-denied.txt by replacing its existing + runtime-tool-fixture-denied-original line with + runtime patch outside the workspace, then summarize the actual + workspace-escape rejection. promptSnippet: "target=apply_patch" failurePromptSnippet: "failure target=apply_patch" diff --git a/src/agents/agent-tools.schema.test.ts b/src/agents/agent-tools.schema.test.ts index e43d516023d2..4b64ff793194 100644 --- a/src/agents/agent-tools.schema.test.ts +++ b/src/agents/agent-tools.schema.test.ts @@ -15,6 +15,7 @@ import { } from "./agent-tools.before-tool-call.js"; import { normalizeToolParameters } from "./agent-tools.schema.js"; import type { AnyAgentTool } from "./agent-tools.types.js"; +import { execSchema } from "./bash-tools.schemas.js"; import { BEFORE_TOOL_CALL_HOOK_CONTEXT, BEFORE_TOOL_CALL_SOURCE_TOOL, @@ -34,6 +35,26 @@ const TEST_USAGE = { cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, }; +describe("direct exec tool schema", () => { + it("keeps model-facing descriptions compact without hiding runtime constraints", () => { + const fields = execSchema.properties as Record; + const describeField = (name: string) => fields[name]?.description ?? ""; + const descriptions = Object.values(fields).map((field) => field.description ?? ""); + + expect(descriptions.join("").length).toBeLessThan(550); + expect(describeField("workdir")).toContain("Blank/whitespace"); + expect(describeField("yieldMs")).toContain("Milliseconds"); + expect(describeField("timeout")).toContain("seconds"); + expect(describeField("pty")).toContain("PTY"); + expect(describeField("elevated")).toContain("if allowed"); + expect(describeField("security")).toContain("tools.exec.security"); + expect(describeField("security")).toContain("host approvals"); + expect(describeField("ask")).toContain("tools.exec.ask"); + expect(describeField("ask")).toContain("channel-origin"); + expect(describeField("ask")).toContain("ask=off"); + }); +}); + describe("normalizeToolParameterSchema", () => { it("reuses normalized schemas for the same schema object and provider options", () => { const schema = { diff --git a/src/agents/bash-tools.schemas.ts b/src/agents/bash-tools.schemas.ts index 233596b3a0ac..57ca835ed80a 100644 --- a/src/agents/bash-tools.schemas.ts +++ b/src/agents/bash-tools.schemas.ts @@ -14,31 +14,29 @@ export const execSchema = Type.Object({ command: Type.String({ description: "Shell command to execute" }), workdir: Type.Optional( Type.String({ - description: - "Working directory. Blank/whitespace values are invalid; omit to use the default cwd.", + description: "Working directory; omit for default. Blank/whitespace is invalid.", }), ), env: Type.Optional(Type.Record(Type.String(), Type.String())), yieldMs: Type.Optional( Type.Number({ - description: "Milliseconds to wait before backgrounding (default 10000)", + description: "Milliseconds before backgrounding; default 10000.", }), ), background: Type.Optional(Type.Boolean({ description: "Run in background immediately" })), timeout: Type.Optional( Type.Number({ - description: "Timeout in seconds (optional, kills process on expiry)", + description: "Timeout in seconds; kills process on expiry.", }), ), pty: Type.Optional( Type.Boolean({ - description: - "Run in a pseudo-terminal (PTY) when available (TTY-required CLIs, coding agents)", + description: "Use PTY for TTY-required CLIs and coding agents.", }), ), elevated: Type.Optional( Type.Boolean({ - description: "Run on the host with elevated permissions (if allowed)", + description: "Run on host with elevated permissions if allowed.", }), ), host: optionalStringEnum(EXEC_TOOL_HOST_VALUES, { @@ -46,14 +44,13 @@ export const execSchema = Type.Object({ }), security: Type.Optional( Type.String({ - description: - "Ignored for normal calls; exec security is set by tools.exec.security and host approvals.", + description: "Ignored per call; tools.exec.security and host approvals decide.", }), ), ask: Type.Optional( Type.String({ description: - "Baseline ask comes from tools.exec.ask and host approvals; channel-origin calls ignore per-call ask when effective host ask is off.", + "Uses tools.exec.ask and host approvals; channel-origin calls cannot override host ask=off.", }), ), node: Type.Optional( diff --git a/src/agents/code-mode.test.ts b/src/agents/code-mode.test.ts index 2107eb808116..d1c98709dfa9 100644 --- a/src/agents/code-mode.test.ts +++ b/src/agents/code-mode.test.ts @@ -460,6 +460,31 @@ describe("Code Mode", () => { ); }); + it("keeps code-mode exec guidance compact without advertising unavailable namespaces", () => { + const { config, catalogRef, tools } = createCodeModeHarness(); + const compacted = applyCodeModeCatalog({ + tools: [...tools, pluginTool("fake_noop", "Noop")], + config, + sessionId: "session-code-mode", + sessionKey: "agent:main:main", + runId: "run-code-mode", + catalogRef, + }); + + const execTool = expectDefined(compacted.tools[0], "exec tool test invariant"); + const parameters = execTool.parameters as { + properties?: Record>; + }; + const codeDescription = parameters.properties?.code?.description; + + expect(execTool.description.length).toBeLessThan(2_400); + expect(execTool.description).toContain("parallelize independent work only"); + expect(codeDescription).toEqual(expect.any(String)); + expect(String(codeDescription).length).toBeLessThan(320); + expect(codeDescription).not.toContain("MCP namespace globals"); + expect(codeDescription).not.toContain("`API` virtual declaration files"); + }); + it("primes the exec schema with exact native tool ids and compact contracts", () => { const { config, catalogRef, tools } = createCodeModeHarness(); const alpha = pluginTool("alpha_tool", "Another deferred description."); diff --git a/src/agents/code-mode.ts b/src/agents/code-mode.ts index 9112b412c88b..14ad2ea04377 100644 --- a/src/agents/code-mode.ts +++ b/src/agents/code-mode.ts @@ -77,10 +77,10 @@ function renderCodeModeCatalogIndex(lines: readonly string[], total: number): st ? `${omitted} additional OpenClaw/plugin tools omitted from this prompt index. Use ALL_TOOLS or tools.search inside exec to find them.` : "Use these exact ids with tools.callValue; use ALL_TOOLS or tools.search inside exec when lookup is ambiguous."; return [ - "OpenClaw/plugin tool quick index (exact ids plus compact input and declared output hints; descriptions are intentionally deferred):", - "Each line is `id input -> output`; `-> ?` means the output shape is unknown.", - "OUTPUT DECLARED RULE: use the named fields in the first exec; keep dependent reads, checks, and follow-up calls in that exec instead of returning a raw value only to inspect an already-declared shape.", - "OUTPUT UNKNOWN RULE: when the needed tool is `-> ?`, including a final dependent call after declared-output calls, return that tool's raw value unchanged. Do not wrap it in the requested answer shape or read guessed fields; filter or map only in a later exec after observing its shape.", + "OpenClaw/plugin tool quick index (exact ids; descriptions are intentionally deferred):", + "Each line is `id input -> output`; `-> ?` means unknown.", + "OUTPUT DECLARED RULE: use declared fields for dependent calls in the first exec.", + "OUTPUT UNKNOWN RULE: return the raw tool value unchanged; inspect or map it only in a later exec.", ...lines, "", footer, @@ -146,7 +146,7 @@ function createCodeModeExecDescription( : ""; const catalogIndex = catalog ? formatCodeModeCatalogIndex(catalog) : ""; return ( - "Run JavaScript or TypeScript in OpenClaw code mode. Use `return` to pass the final value back to the agent; awaited calls without a returned value complete as `null`. Quick-index arrows show trusted declared output hints; `-> ?` means never guess result field names. When the needed tool has an unknown output, including a final dependent call after declared-output calls, the first exec must return the raw tool value unchanged with `return await tools.callValue(id, args);`; do not wrap it in the requested answer shape or read guessed fields; filter or map it only in a later exec after observing its shape. When the arrow declares the fields you need, select, call, and process them in the first exec; do not spend another exec inspecting that declared shape. Within that exec, perform dependent reads, checks, and follow-up calls in order; nested calls still enforce normal tool policy and approvals. Parallelize only independent work. `ALL_TOOLS` is the complete compact catalog with exact ids, input hints, and declared output hints. Select from it directly when practical, use `tools.search(query: string, options?)` when lookup is ambiguous, and use `tools.describe(id: string)` only when the compact input hint is insufficient. Never invent or transform a tool id. `tools.callValue(id: string, args?)` executes a tool and returns its JSON value directly; `tools.call(id: string, args?)` preserves the raw `{ tool, result }` envelope. Example: `const hit = ALL_TOOLS.find((entry) => entry.description.includes('weather')) ?? (await tools.search('weather'))[0]; return await tools.callValue(hit.id, {});`. Node.js modules and `require`/`import` are NOT available; for any shell, file, network, or external action, use enabled catalog tools allowed by policy from inside your code." + + "Run JavaScript or TypeScript in OpenClaw code mode. Use `return` to pass the final value back; otherwise the result is `null`. Quick-index arrows show trusted declared output hints; `-> ?` means never guess result field names. For declared fields, process them in the first exec; do not spend another exec inspecting them. Perform dependent reads, checks, and follow-up calls in order; parallelize independent work only. For an unknown output, including a final dependent call after declared-output calls, return the raw tool value unchanged; do not wrap it in the requested answer shape or guess fields; filter or map it only in a later exec. Nested calls enforce normal tool policy and approvals. `ALL_TOOLS` is the complete compact catalog. Select exact ids directly or with `tools.search(query: string, options?)`; use `tools.describe(id: string)` only when needed. Never invent or transform a tool id. `tools.callValue(id: string, args?)` returns its JSON value directly; `tools.call(id: string, args?)` preserves `{ tool, result }`. Example: `const hit = ALL_TOOLS.find((entry) => entry.description.includes('weather')) ?? (await tools.search('weather'))[0]; return await tools.callValue(hit.id, {});`. Node.js modules and `require`/`import` are NOT available; use enabled catalog tools allowed by policy for shell, file, network, or external actions." + apiGuidance + mcpGuidance + swarmGuidance + @@ -167,7 +167,7 @@ export function createCodeModeTools(ctx: CodeModeToolContext): AnyAgentTool[] { code: Type.Optional( Type.String({ description: - "JavaScript or TypeScript source for one complete workflow. Select exact ids from `ALL_TOOLS` or `tools.search`; never invent ids. `tools.search` takes a query string, not an object. Keep dependent operations in this program, never put dependent calls in Promise.all, and return the final value. `API` virtual declaration files and MCP namespace globals are also available in scope; Node built-in modules are not.", + "JavaScript or TypeScript for one complete workflow. Select exact ids from `ALL_TOOLS` or `tools.search`; never invent ids. `tools.search` takes a query string, not an object. Keep dependent calls in order; never put dependent calls in Promise.all. Return the final value. Node built-in modules are not available.", }), ), command: Type.Optional( diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index c89fbffa2be0..ded38ab7f288 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -4524,6 +4524,8 @@ wait_for_run plugin-clawhub-new.yml 123 "${expectedSha}" || status=$? ".github/release/clawhub-cli/package-lock.json", ".gitignore", "apps/android/.gitignore", + "docs/reference/templates/IDENTITY.md", + "docs/reference/templates/USER.md", "extensions/qa-lab/src/mantis/cli.ts", ]) { const result = spawnSync("git", ["check-ignore", "--no-index", path], { diff --git a/test/vitest/vitest.extension-codex-app-server-attempt-extra.config.ts b/test/vitest/vitest.extension-codex-app-server-attempt-extra.config.ts index d827931767ec..d01dcaebea62 100644 --- a/test/vitest/vitest.extension-codex-app-server-attempt-extra.config.ts +++ b/test/vitest/vitest.extension-codex-app-server-attempt-extra.config.ts @@ -20,6 +20,8 @@ function createExtensionCodexAppServerAttemptExtraVitestConfig( { dir: "extensions", env, + // Prewarm is owned by the light attempt shard, including narrowed runs. + exclude: ["extensions/codex/src/app-server/run-attempt-client-prewarm.test.ts"], fileParallelism: false, name: "extension-codex-app-server-attempt-extra", passWithNoTests: true,