mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
improve: reduce GPT-5.6 coding harness overhead (#114574)
* improve: reduce GPT-5.6 coding harness overhead * refactor: extract runtime parity timing helpers * fix(qa): verify native apply-patch runtime parity * fix(qa): use mock patch execution evidence * fix(qa): preserve workspace templates in full syncs * fix(qa): make rejected patch fixtures deterministic * fix(qa): prove apply-patch workspace containment * fix(qa): expose runtime prompt-cache usage * fix(codex): confine forced private QA patch turns * fix(qa): verify native patch execution and sandbox precedence * fix(qa): execute native patches and prove disk mutation * fix(codex): avoid duplicate native patch registration * perf(agents): keep code mode catalog guidance compact * fix(qa): make native Codex patch evidence deterministic * fix(qa): recognize native sandbox denial errors * fix(qa): advertise native Codex mock model capabilities * fix(qa): seed native Codex mock model catalog at startup * fix(qa): recognize native Codex patch denial results * fix(ci): repair Codex harness parity checks * fix(qa): distinguish unknown prompt-cache measurements * fix(qa): accept native Codex project patch denials * fix(ci): isolate runtime parity report contracts * fix(codex): eliminate native runtime transport overhead * fix(qa): consume runtime parity report contract * fix(qa): compare matched runtime timing captures * fix(qa): type native response delta context * fix(qa): satisfy native codex websocket lint
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
6bc06abbfb
commit
37cf0c6064
@@ -109,6 +109,9 @@ docs/internal/
|
||||
tmp/
|
||||
IDENTITY.md
|
||||
USER.md
|
||||
# Keep packaged workspace templates visible to Git-aware remote syncs.
|
||||
!docs/reference/templates/IDENTITY.md
|
||||
!docs/reference/templates/USER.md
|
||||
# Exception: oc-path real-world test fixtures need to be tracked even
|
||||
# though the bare names match the local-untracked rule above.
|
||||
!extensions/oc-path/src/oc-path/tests/fixtures/real/IDENTITY.md
|
||||
|
||||
@@ -58,6 +58,7 @@ import {
|
||||
readNumberEnv,
|
||||
resolveArgs,
|
||||
} from "./config-utils.js";
|
||||
import { isForcedPrivateQaCodexRuntime } from "./dynamic-tool-profile.js";
|
||||
import type { CodexSandboxPolicy } from "./protocol.js";
|
||||
|
||||
export function resolveCodexAppServerRuntimeOptions(
|
||||
@@ -190,11 +191,17 @@ export function resolveCodexAppServerRuntimeOptions(
|
||||
? normalizedPolicyMode
|
||||
: (explicitPolicyMode ?? normalizedPolicyMode ?? defaultPolicy?.mode ?? "yolo");
|
||||
const serviceTier = normalizeCodexServiceTier(config.serviceTier);
|
||||
const resolvedSandbox =
|
||||
const configuredRuntimeSandbox =
|
||||
forcedPolicy?.sandbox ??
|
||||
configuredSandbox ??
|
||||
defaultPolicy?.sandbox ??
|
||||
(policyMode === "guardian" ? "workspace-write" : "danger-full-access");
|
||||
// Private QA may bound production yolo, but must never widen a configured
|
||||
// read-only sandbox or override the ordinary policy precedence.
|
||||
const resolvedSandbox =
|
||||
isForcedPrivateQaCodexRuntime(env) && configuredRuntimeSandbox === "danger-full-access"
|
||||
? "workspace-write"
|
||||
: configuredRuntimeSandbox;
|
||||
if (transport === "websocket" && !url) {
|
||||
throw new Error(
|
||||
"plugins.entries.codex.config.appServer.url is required when appServer.transport is websocket",
|
||||
@@ -490,6 +497,7 @@ export function codexAppServerStartOptionsKey(
|
||||
export function codexSandboxPolicyForTurn(
|
||||
mode: CodexAppServerSandboxMode,
|
||||
cwd: string,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): CodexSandboxPolicy {
|
||||
if (mode === "danger-full-access") {
|
||||
return { type: "dangerFullAccess" };
|
||||
@@ -497,12 +505,15 @@ export function codexSandboxPolicyForTurn(
|
||||
if (mode === "read-only") {
|
||||
return { type: "readOnly", networkAccess: false };
|
||||
}
|
||||
// Codex includes /tmp and TMPDIR in workspace-write by default. Private QA
|
||||
// workspaces live there, so retaining either root defeats sibling containment.
|
||||
const excludePrivateQaTempRoots = isForcedPrivateQaCodexRuntime(env);
|
||||
return {
|
||||
type: "workspaceWrite",
|
||||
writableRoots: [cwd],
|
||||
networkAccess: false,
|
||||
excludeTmpdirEnvVar: false,
|
||||
excludeSlashTmp: false,
|
||||
excludeTmpdirEnvVar: excludePrivateQaTempRoots,
|
||||
excludeSlashTmp: excludePrivateQaTempRoots,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
canUseCodexModelBackedApprovalsReviewerForModel,
|
||||
codexAppServerStartOptionsKey,
|
||||
codexSandboxPolicyForTurn,
|
||||
readCodexPluginConfig,
|
||||
resolveCodexAppServerRuntimeOptions,
|
||||
resolveCodexAppServerStartOptionsForAgent,
|
||||
@@ -549,6 +550,91 @@ describe("Codex app-server config", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("confines only explicitly forced private-QA Codex runtime to workspace writes", () => {
|
||||
const privateQaCodexEnv = {
|
||||
OPENCLAW_BUILD_PRIVATE_QA: "1",
|
||||
OPENCLAW_QA_FORCE_RUNTIME: "codex",
|
||||
};
|
||||
const runtime = resolveRuntimeForTest({
|
||||
pluginConfig: {
|
||||
appServer: {
|
||||
mode: "yolo",
|
||||
approvalPolicy: "never",
|
||||
sandbox: "danger-full-access",
|
||||
},
|
||||
},
|
||||
env: privateQaCodexEnv,
|
||||
});
|
||||
|
||||
expectRuntimePolicy(runtime, {
|
||||
approvalPolicy: "never",
|
||||
sandbox: "workspace-write",
|
||||
approvalsReviewer: "user",
|
||||
});
|
||||
expect(codexSandboxPolicyForTurn(runtime.sandbox, "/qa/workspace", privateQaCodexEnv)).toEqual({
|
||||
type: "workspaceWrite",
|
||||
writableRoots: ["/qa/workspace"],
|
||||
networkAccess: false,
|
||||
excludeTmpdirEnvVar: true,
|
||||
excludeSlashTmp: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves an explicitly read-only sandbox for forced private-QA Codex runtime", () => {
|
||||
const privateQaCodexEnv = {
|
||||
OPENCLAW_BUILD_PRIVATE_QA: "1",
|
||||
OPENCLAW_QA_FORCE_RUNTIME: "codex",
|
||||
};
|
||||
const runtime = resolveRuntimeForTest({
|
||||
pluginConfig: {
|
||||
appServer: {
|
||||
mode: "yolo",
|
||||
approvalPolicy: "never",
|
||||
sandbox: "read-only",
|
||||
},
|
||||
},
|
||||
env: privateQaCodexEnv,
|
||||
});
|
||||
|
||||
expectRuntimePolicy(runtime, {
|
||||
approvalPolicy: "never",
|
||||
sandbox: "read-only",
|
||||
approvalsReviewer: "user",
|
||||
});
|
||||
expect(codexSandboxPolicyForTurn(runtime.sandbox, "/qa/workspace", privateQaCodexEnv)).toEqual({
|
||||
type: "readOnly",
|
||||
networkAccess: false,
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ label: "ordinary production", env: {} },
|
||||
{ label: "private build without a forced runtime", env: { OPENCLAW_BUILD_PRIVATE_QA: "1" } },
|
||||
{
|
||||
label: "forced runtime without a private build",
|
||||
env: { OPENCLAW_QA_FORCE_RUNTIME: "codex" },
|
||||
},
|
||||
{
|
||||
label: "forced private-QA OpenClaw runtime",
|
||||
env: { OPENCLAW_BUILD_PRIVATE_QA: "1", OPENCLAW_QA_FORCE_RUNTIME: "openclaw" },
|
||||
},
|
||||
])("preserves production yolo filesystem policy for $label", ({ env }) => {
|
||||
const runtime = resolveRuntimeForTest({ pluginConfig: {}, env });
|
||||
|
||||
expectRuntimePolicy(runtime, {
|
||||
approvalPolicy: "never",
|
||||
sandbox: "danger-full-access",
|
||||
approvalsReviewer: "user",
|
||||
});
|
||||
expect(codexSandboxPolicyForTurn("workspace-write", "/qa/workspace", env)).toEqual({
|
||||
type: "workspaceWrite",
|
||||
writableRoots: ["/qa/workspace"],
|
||||
networkAccess: false,
|
||||
excludeTmpdirEnvVar: false,
|
||||
excludeSlashTmp: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not change ordinary harness connection defaults when supervision is enabled", () => {
|
||||
const runtime = resolveRuntimeForTest({
|
||||
pluginConfig: { supervision: { enabled: true } },
|
||||
|
||||
@@ -5,7 +5,6 @@ export {
|
||||
} from "./config-contracts.js";
|
||||
export type {
|
||||
CodexAppServerApprovalPolicy,
|
||||
CodexAppServerConnectionClass,
|
||||
CodexAppServerHomeScope,
|
||||
CodexAppServerRuntimeOptions,
|
||||
CodexAppServerSandboxMode,
|
||||
|
||||
@@ -565,9 +565,15 @@ describe("Codex app-server dynamic tool build", () => {
|
||||
});
|
||||
|
||||
it("exposes app-server-owned tools directly for forced private QA Codex runtime", () => {
|
||||
const tools = ["read", "write", "get_goal", "image_generate", "message"].map((name) => ({
|
||||
name,
|
||||
}));
|
||||
const tools = [
|
||||
"read",
|
||||
"write",
|
||||
"apply_patch",
|
||||
"apply-patch",
|
||||
"get_goal",
|
||||
"image_generate",
|
||||
"message",
|
||||
].map((name) => ({ name }));
|
||||
const privateQaCodexEnv = {
|
||||
OPENCLAW_BUILD_PRIVATE_QA: "1",
|
||||
OPENCLAW_QA_FORCE_RUNTIME: "codex",
|
||||
|
||||
@@ -5,7 +5,7 @@ import type {
|
||||
CodexAppServerConnectionClass,
|
||||
CodexDynamicToolsLoading,
|
||||
CodexPluginConfig,
|
||||
} from "./config.js";
|
||||
} from "./config-contracts.js";
|
||||
|
||||
/** Tool names owned by Codex app-server and normally excluded from OpenClaw dynamic tools. */
|
||||
const CODEX_APP_SERVER_OWNED_DYNAMIC_TOOL_EXCLUDES = [
|
||||
@@ -142,7 +142,11 @@ function filterCodexDynamicToolsWithOptions<T extends { name: string }>(
|
||||
for (const name of CODEX_NATIVE_GOAL_TOOL_EXCLUDES) {
|
||||
excludes.add(name);
|
||||
}
|
||||
if (!isForcedPrivateQaCodexRuntime(env)) {
|
||||
if (isForcedPrivateQaCodexRuntime(env)) {
|
||||
// Native apply_patch is registered first; advertising a second handler
|
||||
// makes Codex reject the duplicate before either QA patch can execute.
|
||||
excludes.add("apply_patch");
|
||||
} else {
|
||||
for (const name of CODEX_APP_SERVER_OWNED_DYNAMIC_TOOL_EXCLUDES) {
|
||||
if (options.preserveOpenClawShell && CODEX_APP_SERVER_OWNED_SHELL_TOOL_EXCLUDES.has(name)) {
|
||||
continue;
|
||||
|
||||
@@ -156,6 +156,7 @@ describe("resolveCodexProviderWebSearchSupport", () => {
|
||||
const { clientFactory, request } = createClientFactory(false);
|
||||
|
||||
await expect(resolveSupport(clientFactory, " OpenAI ")).resolves.toBe("supported");
|
||||
expect(clientFactory).not.toHaveBeenCalled();
|
||||
expect(request).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -165,6 +166,16 @@ describe("resolveCodexProviderWebSearchSupport", () => {
|
||||
await expect(resolveSupport(clientFactory, "amazon-bedrock")).resolves.toBe("unsupported");
|
||||
await expect(resolveSupport(clientFactory, "custom-provider")).resolves.toBe("unsupported");
|
||||
await expect(resolveSupport(clientFactory, "lmstudio")).resolves.toBe("unsupported");
|
||||
expect(clientFactory).not.toHaveBeenCalled();
|
||||
expect(request).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not wait for app-server startup when the override proves hosted support", async () => {
|
||||
const clientFactory = vi.fn(
|
||||
async () => await new Promise<CodexAppServerClient>(() => {}),
|
||||
) as unknown as CodexAppServerClientFactory;
|
||||
|
||||
await expect(resolveSupport(clientFactory, "openai")).resolves.toBe("supported");
|
||||
expect(clientFactory).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -8,6 +8,18 @@ import {
|
||||
} from "./shared-client.js";
|
||||
import type { CodexNativeWebSearchSupport } from "./web-search.js";
|
||||
|
||||
function resolveOverriddenProviderWebSearchSupport(
|
||||
modelProviderOverride: string | undefined,
|
||||
): CodexNativeWebSearchSupport | undefined {
|
||||
const provider = modelProviderOverride?.trim().toLowerCase();
|
||||
if (!provider) {
|
||||
return undefined;
|
||||
}
|
||||
// The capability RPC describes the configured provider, not a thread
|
||||
// override. OpenAI's hosted support is known; other overrides stay managed.
|
||||
return provider === "openai" ? "supported" : "unsupported";
|
||||
}
|
||||
|
||||
async function readConfiguredProviderWebSearchSupport(params: {
|
||||
client: CodexAppServerClient;
|
||||
timeoutMs: number;
|
||||
@@ -30,15 +42,9 @@ export async function resolveCodexProviderWebSearchSupportForClient(params: {
|
||||
modelProviderOverride: string | undefined;
|
||||
signal: AbortSignal;
|
||||
}): Promise<CodexNativeWebSearchSupport> {
|
||||
const modelProviderOverride = params.modelProviderOverride?.trim().toLowerCase();
|
||||
if (modelProviderOverride === "openai") {
|
||||
return "supported";
|
||||
}
|
||||
if (modelProviderOverride) {
|
||||
// Codex's capability RPC only reports the configured provider, not a
|
||||
// thread-scoped override. Keep managed search for overrides whose hosted
|
||||
// capability cannot be proven from the configured-provider response.
|
||||
return "unsupported";
|
||||
const overrideSupport = resolveOverriddenProviderWebSearchSupport(params.modelProviderOverride);
|
||||
if (overrideSupport) {
|
||||
return overrideSupport;
|
||||
}
|
||||
try {
|
||||
return await readConfiguredProviderWebSearchSupport(params);
|
||||
@@ -57,6 +63,12 @@ export async function resolveCodexProviderWebSearchSupport(params: {
|
||||
modelProviderOverride: string | undefined;
|
||||
signal: AbortSignal;
|
||||
}): Promise<CodexNativeWebSearchSupport> {
|
||||
const overrideSupport = resolveOverriddenProviderWebSearchSupport(params.modelProviderOverride);
|
||||
if (overrideSupport) {
|
||||
// Never serialize the prewarmed app-server startup behind a capability
|
||||
// probe whose answer is already fixed by the selected thread provider.
|
||||
return overrideSupport;
|
||||
}
|
||||
let client: CodexAppServerClient | undefined;
|
||||
try {
|
||||
client = await params.clientFactory({
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { CodexAppServerRuntimeOptions } from "./config.js";
|
||||
import type { CodexAppServerConnectionClass } from "./config-contracts.js";
|
||||
import { normalizeCodexDynamicToolName } from "./dynamic-tool-profile.js";
|
||||
import type { CodexAppServerThreadBinding } from "./session-binding.js";
|
||||
import type {
|
||||
@@ -7,7 +7,7 @@ import type {
|
||||
} from "./thread-lifecycle-types.js";
|
||||
|
||||
export function shouldRotateCodexAppServerBindingForRuntime(params: {
|
||||
connectionClass: CodexAppServerRuntimeOptions["connectionClass"];
|
||||
connectionClass: CodexAppServerConnectionClass;
|
||||
current?: string;
|
||||
binding?: string;
|
||||
}): boolean {
|
||||
|
||||
@@ -1105,6 +1105,44 @@ describe("Codex app-server native code mode config", () => {
|
||||
});
|
||||
|
||||
describe("Codex app-server turn input image sanitizing", () => {
|
||||
it("excludes implicit temporary writable roots from forced private-QA Codex turns", () => {
|
||||
vi.stubEnv("OPENCLAW_BUILD_PRIVATE_QA", "1");
|
||||
vi.stubEnv("OPENCLAW_QA_FORCE_RUNTIME", "codex");
|
||||
try {
|
||||
const request = buildTurnStartParams(createAttemptParams({ provider: "openai" }), {
|
||||
threadId: "thread-1",
|
||||
cwd: "/tmp/qa/workspace",
|
||||
appServer: createAppServerOptions() as never,
|
||||
});
|
||||
|
||||
expect(request.sandboxPolicy).toEqual({
|
||||
type: "workspaceWrite",
|
||||
writableRoots: ["/tmp/qa/workspace"],
|
||||
networkAccess: false,
|
||||
excludeTmpdirEnvVar: true,
|
||||
excludeSlashTmp: true,
|
||||
});
|
||||
} finally {
|
||||
vi.unstubAllEnvs();
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves implicit temporary writable roots for ordinary Codex turns", () => {
|
||||
const request = buildTurnStartParams(createAttemptParams({ provider: "openai" }), {
|
||||
threadId: "thread-1",
|
||||
cwd: "/tmp/qa/workspace",
|
||||
appServer: createAppServerOptions() as never,
|
||||
});
|
||||
|
||||
expect(request.sandboxPolicy).toEqual({
|
||||
type: "workspaceWrite",
|
||||
writableRoots: ["/tmp/qa/workspace"],
|
||||
networkAccess: false,
|
||||
excludeTmpdirEnvVar: false,
|
||||
excludeSlashTmp: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("uses an explicit turn sandbox policy override when provided", () => {
|
||||
const request = buildTurnStartParams(createAttemptParams({ provider: "openai" }), {
|
||||
threadId: "thread-1",
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
"playwright-core": "1.61.1",
|
||||
"pretty-ms": "9.3.0",
|
||||
"semver": "7.8.5",
|
||||
"ws": "8.21.1",
|
||||
"yaml": "2.9.0",
|
||||
"zod": "4.4.3"
|
||||
},
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
import type { RuntimeId, RuntimeParityUsage } from "./runtime-parity.js";
|
||||
|
||||
export type QaRuntimeParityCacheUsage = {
|
||||
totalTokens: number;
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
grossInputTokens: number | null;
|
||||
uncachedInputTokens: number | null;
|
||||
cachedInputTokens: number | null;
|
||||
cacheWriteTokens: number | null;
|
||||
cacheHitPercent: number | null;
|
||||
};
|
||||
|
||||
type QaRuntimeParityCacheScenario = {
|
||||
openclawUsage: QaRuntimeParityCacheUsage | null;
|
||||
codexUsage: QaRuntimeParityCacheUsage | null;
|
||||
};
|
||||
|
||||
export function summarizeRuntimeParityCacheUsage(
|
||||
usage: Pick<
|
||||
RuntimeParityUsage,
|
||||
"inputTokens" | "outputTokens" | "totalTokens" | "cacheRead" | "cacheWrite"
|
||||
>,
|
||||
): QaRuntimeParityCacheUsage {
|
||||
const cachedInputTokens = usage.cacheRead ?? null;
|
||||
const cacheWriteTokens = usage.cacheWrite ?? null;
|
||||
const uncachedInputTokens =
|
||||
cacheWriteTokens === null ? null : usage.inputTokens + cacheWriteTokens;
|
||||
const grossInputTokens =
|
||||
cachedInputTokens === null || uncachedInputTokens === null
|
||||
? null
|
||||
: uncachedInputTokens + cachedInputTokens;
|
||||
return {
|
||||
totalTokens: usage.totalTokens,
|
||||
inputTokens: usage.inputTokens,
|
||||
outputTokens: usage.outputTokens,
|
||||
grossInputTokens,
|
||||
uncachedInputTokens,
|
||||
cachedInputTokens,
|
||||
cacheWriteTokens,
|
||||
cacheHitPercent:
|
||||
grossInputTokens !== null && grossInputTokens > 0 && cachedInputTokens !== null
|
||||
? (cachedInputTokens / grossInputTokens) * 100
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
export function aggregateRuntimeParityCacheUsage(
|
||||
scenarios: readonly QaRuntimeParityCacheScenario[],
|
||||
runtime: RuntimeId,
|
||||
): QaRuntimeParityCacheUsage | null {
|
||||
const captures = scenarios.flatMap((scenario) => {
|
||||
const usage = runtime === "openclaw" ? scenario.openclawUsage : scenario.codexUsage;
|
||||
return usage === null ? [] : [usage];
|
||||
});
|
||||
if (captures.length === 0) {
|
||||
return null;
|
||||
}
|
||||
const measuredCaptures = captures.filter(
|
||||
(capture) => capture.cachedInputTokens !== null && capture.cacheWriteTokens !== null,
|
||||
);
|
||||
const inputTokens = captures.reduce((total, capture) => total + capture.inputTokens, 0);
|
||||
const outputTokens = captures.reduce((total, capture) => total + capture.outputTokens, 0);
|
||||
const totalTokens = captures.reduce((total, capture) => total + capture.totalTokens, 0);
|
||||
if (measuredCaptures.length === 0) {
|
||||
return {
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
totalTokens,
|
||||
grossInputTokens: null,
|
||||
uncachedInputTokens: null,
|
||||
cachedInputTokens: null,
|
||||
cacheWriteTokens: null,
|
||||
cacheHitPercent: null,
|
||||
};
|
||||
}
|
||||
const cachedInputTokens = measuredCaptures.reduce(
|
||||
(total, capture) => total + (capture.cachedInputTokens ?? 0),
|
||||
0,
|
||||
);
|
||||
const cacheWriteTokens = measuredCaptures.reduce(
|
||||
(total, capture) => total + (capture.cacheWriteTokens ?? 0),
|
||||
0,
|
||||
);
|
||||
const uncachedInputTokens = measuredCaptures.reduce(
|
||||
(total, capture) => total + capture.inputTokens + (capture.cacheWriteTokens ?? 0),
|
||||
0,
|
||||
);
|
||||
const grossInputTokens = uncachedInputTokens + cachedInputTokens;
|
||||
return {
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
totalTokens,
|
||||
grossInputTokens,
|
||||
uncachedInputTokens,
|
||||
cachedInputTokens,
|
||||
cacheWriteTokens,
|
||||
cacheHitPercent: grossInputTokens > 0 ? (cachedInputTokens / grossInputTokens) * 100 : null,
|
||||
};
|
||||
}
|
||||
|
||||
export function formatRuntimeCacheHitPercent(value: number | null | undefined): string {
|
||||
return value === null || value === undefined ? "N/A" : `${value.toFixed(1)}%`;
|
||||
}
|
||||
|
||||
export function formatRuntimeCacheCount(value: number | null | undefined): string {
|
||||
return value === null || value === undefined ? "N/A" : String(value);
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
import type { QaRuntimeParitySuiteSummary } from "./agentic-parity-report.js";
|
||||
|
||||
export function makeRuntimeParitySummary(): QaRuntimeParitySuiteSummary {
|
||||
return {
|
||||
scenarios: [
|
||||
{
|
||||
name: "Approval turn tool followthrough",
|
||||
status: "pass",
|
||||
steps: [],
|
||||
runtimeParity: {
|
||||
scenarioId: "approval-turn-tool-followthrough",
|
||||
drift: "none",
|
||||
cells: {
|
||||
openclaw: {
|
||||
runtime: "openclaw",
|
||||
status: "pass",
|
||||
transcriptBytes: '{"role":"assistant"}\n',
|
||||
toolCalls: [{ tool: "read_file", argsHash: "a", resultHash: "r" }],
|
||||
finalText: "done",
|
||||
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
|
||||
wallClockMs: 20,
|
||||
bootStateLines: [],
|
||||
},
|
||||
codex: {
|
||||
runtime: "codex",
|
||||
status: "pass",
|
||||
transcriptBytes: '{"role":"assistant"}\n',
|
||||
toolCalls: [{ tool: "read_file", argsHash: "a", resultHash: "r" }],
|
||||
finalText: "done",
|
||||
usage: { inputTokens: 8, outputTokens: 4, totalTokens: 12 },
|
||||
wallClockMs: 18,
|
||||
bootStateLines: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Compaction retry after mutating tool",
|
||||
status: "pass",
|
||||
steps: [],
|
||||
runtimeParity: {
|
||||
scenarioId: "compaction-retry-after-mutating-tool",
|
||||
drift: "tool-call-shape",
|
||||
driftDetails: "tool call 1 differs",
|
||||
cells: {
|
||||
openclaw: {
|
||||
runtime: "openclaw",
|
||||
status: "pass",
|
||||
transcriptBytes: '{"role":"assistant"}\n',
|
||||
toolCalls: [{ tool: "read_file", argsHash: "a", resultHash: "r" }],
|
||||
finalText: "done",
|
||||
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
|
||||
wallClockMs: 20,
|
||||
bootStateLines: [],
|
||||
},
|
||||
codex: {
|
||||
runtime: "codex",
|
||||
status: "pass",
|
||||
transcriptBytes: '{"role":"assistant"}\n',
|
||||
toolCalls: [{ tool: "read_file", argsHash: "b", resultHash: "r" }],
|
||||
finalText: "done",
|
||||
usage: { inputTokens: 9, outputTokens: 4, totalTokens: 13 },
|
||||
wallClockMs: 19,
|
||||
bootStateLines: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
counts: {
|
||||
total: 2,
|
||||
passed: 2,
|
||||
failed: 0,
|
||||
},
|
||||
run: {
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
runtimePair: ["openclaw", "codex"],
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { makeRuntimeParitySummary } from "./agentic-parity-report-test-helpers.js";
|
||||
import {
|
||||
buildQaRuntimeParityReport,
|
||||
renderQaRuntimeParityMarkdownReport,
|
||||
} from "./agentic-parity-report.js";
|
||||
|
||||
function makeMeasuredRuntimeParitySummary() {
|
||||
const summary = makeRuntimeParitySummary();
|
||||
for (const scenario of summary.scenarios) {
|
||||
if (scenario.runtimeParity) {
|
||||
for (const cell of Object.values(scenario.runtimeParity.cells)) {
|
||||
cell.usage = { ...cell.usage, cacheRead: 0, cacheWrite: 0 };
|
||||
}
|
||||
}
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
describe("qa runtime parity prompt-cache reporting", () => {
|
||||
it("reports cached, uncached, and cache-write input without counting output as cacheable", () => {
|
||||
const summary = makeMeasuredRuntimeParitySummary();
|
||||
const scenario = summary.scenarios[0];
|
||||
if (!scenario?.runtimeParity) {
|
||||
throw new Error("runtime parity fixture missing");
|
||||
}
|
||||
scenario.runtimeParity.cells.openclaw.usage = {
|
||||
inputTokens: 10,
|
||||
outputTokens: 5,
|
||||
totalTokens: 85,
|
||||
cacheRead: 60,
|
||||
cacheWrite: 10,
|
||||
};
|
||||
scenario.runtimeParity.cells.codex.usage = {
|
||||
inputTokens: 8,
|
||||
outputTokens: 4,
|
||||
totalTokens: 20,
|
||||
cacheRead: 8,
|
||||
cacheWrite: 0,
|
||||
};
|
||||
|
||||
const report = buildQaRuntimeParityReport({ summary });
|
||||
|
||||
expect(report.scenarios[0]?.openclawUsage).toEqual({
|
||||
totalTokens: 85,
|
||||
inputTokens: 10,
|
||||
outputTokens: 5,
|
||||
grossInputTokens: 80,
|
||||
uncachedInputTokens: 20,
|
||||
cachedInputTokens: 60,
|
||||
cacheWriteTokens: 10,
|
||||
cacheHitPercent: 75,
|
||||
});
|
||||
expect(report.scenarios[0]?.codexUsage).toEqual({
|
||||
totalTokens: 20,
|
||||
inputTokens: 8,
|
||||
outputTokens: 4,
|
||||
grossInputTokens: 16,
|
||||
uncachedInputTokens: 8,
|
||||
cachedInputTokens: 8,
|
||||
cacheWriteTokens: 0,
|
||||
cacheHitPercent: 50,
|
||||
});
|
||||
expect(report.usage.openclaw).toMatchObject({
|
||||
totalTokens: 100,
|
||||
grossInputTokens: 90,
|
||||
uncachedInputTokens: 30,
|
||||
cachedInputTokens: 60,
|
||||
cacheWriteTokens: 10,
|
||||
});
|
||||
expect(report.usage.openclaw?.cacheHitPercent).toBeCloseTo((60 / 90) * 100);
|
||||
expect(report.usage.codex).toMatchObject({
|
||||
totalTokens: 33,
|
||||
grossInputTokens: 25,
|
||||
uncachedInputTokens: 17,
|
||||
cachedInputTokens: 8,
|
||||
cacheWriteTokens: 0,
|
||||
cacheHitPercent: 32,
|
||||
});
|
||||
const markdown = renderQaRuntimeParityMarkdownReport(report);
|
||||
expect(markdown).toContain("## Prompt Cache");
|
||||
expect(markdown).toContain("| openclaw | 90 | 30 | 60 | 10 | 10 | 100 | 66.7% |");
|
||||
expect(markdown).toContain("| codex | 25 | 17 | 8 | 0 | 8 | 33 | 32.0% |");
|
||||
expect(markdown).toContain(
|
||||
"prompt cache: openclaw 75.0% (60 cached, 20 uncached input); codex 50.0% (8 cached, 8 uncached input)",
|
||||
);
|
||||
});
|
||||
|
||||
it("computes aggregate cache hits from measured input rather than averaging percentages", () => {
|
||||
const summary = makeMeasuredRuntimeParitySummary();
|
||||
const [first, second] = summary.scenarios;
|
||||
if (!first?.runtimeParity || !second?.runtimeParity) {
|
||||
throw new Error("runtime parity fixtures missing");
|
||||
}
|
||||
first.runtimeParity.cells.openclaw.usage = {
|
||||
inputTokens: 10,
|
||||
outputTokens: 5,
|
||||
totalTokens: 85,
|
||||
cacheRead: 60,
|
||||
cacheWrite: 10,
|
||||
};
|
||||
second.runtimeParity.cells.openclaw.usage = {
|
||||
inputTokens: 10,
|
||||
outputTokens: 5,
|
||||
totalTokens: 115,
|
||||
cacheRead: 100,
|
||||
cacheWrite: 0,
|
||||
};
|
||||
|
||||
const report = buildQaRuntimeParityReport({ summary });
|
||||
expect(report.usage.openclaw).toMatchObject({
|
||||
totalTokens: 200,
|
||||
grossInputTokens: 190,
|
||||
uncachedInputTokens: 30,
|
||||
cachedInputTokens: 160,
|
||||
cacheWriteTokens: 10,
|
||||
});
|
||||
expect(report.usage.openclaw?.cacheHitPercent).toBeCloseTo((160 / 190) * 100);
|
||||
});
|
||||
|
||||
it("reports unavailable runtime captures as N/A instead of fabricated zero-token usage", () => {
|
||||
const report = buildQaRuntimeParityReport({
|
||||
summary: {
|
||||
scenarios: [{ name: "Missing runtime capture", status: "fail" }],
|
||||
counts: { total: 1, passed: 0, failed: 1 },
|
||||
run: { providerMode: "live-frontier", runtimePair: ["openclaw", "codex"] },
|
||||
},
|
||||
});
|
||||
expect(report.usage).toEqual({ openclaw: null, codex: null });
|
||||
expect(report.scenarios[0]).toMatchObject({ openclawUsage: null, codexUsage: null });
|
||||
const markdown = renderQaRuntimeParityMarkdownReport(report);
|
||||
expect(markdown).toContain("| openclaw | N/A | N/A | N/A | N/A | N/A | N/A | N/A |");
|
||||
expect(markdown).toContain("| codex | N/A | N/A | N/A | N/A | N/A | N/A | N/A |");
|
||||
});
|
||||
|
||||
it("keeps missing cache telemetry unknown while preserving measured token totals", () => {
|
||||
const report = buildQaRuntimeParityReport({ summary: makeRuntimeParitySummary() });
|
||||
expect(report.scenarios[0]?.openclawUsage).toMatchObject({
|
||||
inputTokens: 10,
|
||||
outputTokens: 5,
|
||||
totalTokens: 15,
|
||||
grossInputTokens: null,
|
||||
uncachedInputTokens: null,
|
||||
cachedInputTokens: null,
|
||||
cacheWriteTokens: null,
|
||||
cacheHitPercent: null,
|
||||
});
|
||||
expect(report.usage.openclaw).toMatchObject({
|
||||
inputTokens: 20,
|
||||
outputTokens: 10,
|
||||
totalTokens: 30,
|
||||
grossInputTokens: null,
|
||||
uncachedInputTokens: null,
|
||||
cachedInputTokens: null,
|
||||
cacheWriteTokens: null,
|
||||
cacheHitPercent: null,
|
||||
});
|
||||
expect(renderQaRuntimeParityMarkdownReport(report)).toContain(
|
||||
"| openclaw | N/A | N/A | N/A | N/A | 10 | 30 | N/A |",
|
||||
);
|
||||
});
|
||||
|
||||
it("excludes unknown cache captures from the weighted cache-hit denominator", () => {
|
||||
const summary = makeRuntimeParitySummary();
|
||||
const first = summary.scenarios[0];
|
||||
if (!first?.runtimeParity) {
|
||||
throw new Error("runtime parity fixture missing");
|
||||
}
|
||||
first.runtimeParity.cells.openclaw.usage = {
|
||||
inputTokens: 10,
|
||||
outputTokens: 5,
|
||||
totalTokens: 85,
|
||||
cacheRead: 60,
|
||||
cacheWrite: 10,
|
||||
};
|
||||
const report = buildQaRuntimeParityReport({ summary });
|
||||
expect(report.usage.openclaw).toMatchObject({
|
||||
inputTokens: 20,
|
||||
outputTokens: 10,
|
||||
totalTokens: 100,
|
||||
grossInputTokens: 80,
|
||||
uncachedInputTokens: 20,
|
||||
cachedInputTokens: 60,
|
||||
cacheWriteTokens: 10,
|
||||
cacheHitPercent: 75,
|
||||
});
|
||||
});
|
||||
|
||||
it("distinguishes a measured zero cache hit from missing cache telemetry", () => {
|
||||
const report = buildQaRuntimeParityReport({ summary: makeMeasuredRuntimeParitySummary() });
|
||||
expect(report.usage.openclaw).toMatchObject({
|
||||
inputTokens: 20,
|
||||
grossInputTokens: 20,
|
||||
uncachedInputTokens: 20,
|
||||
cachedInputTokens: 0,
|
||||
cacheWriteTokens: 0,
|
||||
cacheHitPercent: 0,
|
||||
});
|
||||
expect(renderQaRuntimeParityMarkdownReport(report)).toContain(
|
||||
"| openclaw | 20 | 20 | 0 | 0 | 10 | 30 | 0.0% |",
|
||||
);
|
||||
});
|
||||
|
||||
it("reports measured zero-input cache hits as unavailable rather than infinity", () => {
|
||||
const summary = makeMeasuredRuntimeParitySummary();
|
||||
for (const scenario of summary.scenarios) {
|
||||
if (scenario.runtimeParity) {
|
||||
scenario.runtimeParity.cells.openclaw.usage = {
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
totalTokens: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
const report = buildQaRuntimeParityReport({ summary });
|
||||
expect(report.usage.openclaw).toMatchObject({
|
||||
grossInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
cacheHitPercent: null,
|
||||
});
|
||||
expect(renderQaRuntimeParityMarkdownReport(report)).toContain(
|
||||
"| openclaw | 0 | 0 | 0 | 0 | 0 | 0 | N/A |",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
// Qa Lab tests cover agentic parity report plugin behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { makeRuntimeParitySummary } from "./agentic-parity-report-test-helpers.js";
|
||||
import {
|
||||
buildQaAgenticParityComparison,
|
||||
buildQaRuntimeParityReport,
|
||||
@@ -42,86 +43,6 @@ function withScenarioOverride(name: string, override: Partial<QaParityReportScen
|
||||
);
|
||||
}
|
||||
|
||||
function makeRuntimeParitySummary(): QaRuntimeParitySuiteSummary {
|
||||
return {
|
||||
scenarios: [
|
||||
{
|
||||
name: "Approval turn tool followthrough",
|
||||
status: "pass",
|
||||
steps: [],
|
||||
runtimeParity: {
|
||||
scenarioId: "approval-turn-tool-followthrough",
|
||||
drift: "none",
|
||||
cells: {
|
||||
openclaw: {
|
||||
runtime: "openclaw",
|
||||
status: "pass",
|
||||
transcriptBytes: '{"role":"assistant"}\n',
|
||||
toolCalls: [{ tool: "read_file", argsHash: "a", resultHash: "r" }],
|
||||
finalText: "done",
|
||||
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
|
||||
wallClockMs: 20,
|
||||
bootStateLines: [],
|
||||
},
|
||||
codex: {
|
||||
runtime: "codex",
|
||||
status: "pass",
|
||||
transcriptBytes: '{"role":"assistant"}\n',
|
||||
toolCalls: [{ tool: "read_file", argsHash: "a", resultHash: "r" }],
|
||||
finalText: "done",
|
||||
usage: { inputTokens: 8, outputTokens: 4, totalTokens: 12 },
|
||||
wallClockMs: 18,
|
||||
bootStateLines: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Compaction retry after mutating tool",
|
||||
status: "pass",
|
||||
steps: [],
|
||||
runtimeParity: {
|
||||
scenarioId: "compaction-retry-after-mutating-tool",
|
||||
drift: "tool-call-shape",
|
||||
driftDetails: "tool call 1 differs",
|
||||
cells: {
|
||||
openclaw: {
|
||||
runtime: "openclaw",
|
||||
status: "pass",
|
||||
transcriptBytes: '{"role":"assistant"}\n',
|
||||
toolCalls: [{ tool: "read_file", argsHash: "a", resultHash: "r" }],
|
||||
finalText: "done",
|
||||
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
|
||||
wallClockMs: 20,
|
||||
bootStateLines: [],
|
||||
},
|
||||
codex: {
|
||||
runtime: "codex",
|
||||
status: "pass",
|
||||
transcriptBytes: '{"role":"assistant"}\n',
|
||||
toolCalls: [{ tool: "read_file", argsHash: "b", resultHash: "r" }],
|
||||
finalText: "done",
|
||||
usage: { inputTokens: 9, outputTokens: 4, totalTokens: 13 },
|
||||
wallClockMs: 19,
|
||||
bootStateLines: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
counts: {
|
||||
total: 2,
|
||||
passed: 2,
|
||||
failed: 0,
|
||||
},
|
||||
run: {
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
runtimePair: ["openclaw", "codex"],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function firstRuntimeParityScenario() {
|
||||
const scenario = makeRuntimeParitySummary().scenarios[0];
|
||||
if (!scenario) {
|
||||
@@ -902,6 +823,89 @@ status=done`,
|
||||
expect(report.driftCounts.none).toBe(1);
|
||||
expect(report.driftCounts["tool-call-shape"]).toBe(1);
|
||||
expect(report.failures).toEqual([]);
|
||||
expect(report.scenarios[0]).toMatchObject({
|
||||
openclawWallClockMs: 20,
|
||||
codexWallClockMs: 18,
|
||||
fasterRuntime: "codex",
|
||||
speedupPercent: (2 / 18) * 100,
|
||||
});
|
||||
expect(report.timing).toEqual({
|
||||
openclaw: {
|
||||
totalWallClockMs: 40,
|
||||
p50WallClockMs: 20,
|
||||
p90WallClockMs: 20,
|
||||
},
|
||||
codex: {
|
||||
totalWallClockMs: 37,
|
||||
p50WallClockMs: 18,
|
||||
p90WallClockMs: 19,
|
||||
},
|
||||
fasterRuntime: "codex",
|
||||
speedupPercent: (3 / 37) * 100,
|
||||
});
|
||||
});
|
||||
|
||||
it("reports tied zero-duration captures without an invalid speedup", () => {
|
||||
const summary = makeRuntimeParitySummary();
|
||||
for (const scenario of summary.scenarios) {
|
||||
if (scenario.runtimeParity) {
|
||||
scenario.runtimeParity.cells.openclaw.wallClockMs = 0;
|
||||
scenario.runtimeParity.cells.codex.wallClockMs = 0;
|
||||
}
|
||||
}
|
||||
|
||||
const report = buildQaRuntimeParityReport({ summary });
|
||||
|
||||
expect(report.pass).toBe(true);
|
||||
expect(report.timing).toEqual({
|
||||
openclaw: { totalWallClockMs: 0, p50WallClockMs: 0, p90WallClockMs: 0 },
|
||||
codex: { totalWallClockMs: 0, p50WallClockMs: 0, p90WallClockMs: 0 },
|
||||
fasterRuntime: "tie",
|
||||
speedupPercent: 0,
|
||||
});
|
||||
expect(report.scenarios[0]).toMatchObject({
|
||||
fasterRuntime: "tie",
|
||||
speedupPercent: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("excludes missing runtime captures from wall-clock aggregates", () => {
|
||||
const summary = makeRuntimeParitySummary();
|
||||
summary.scenarios.push({ name: "Missing runtime capture", status: "fail" });
|
||||
|
||||
const report = buildQaRuntimeParityReport({ summary });
|
||||
|
||||
expect(report.pass).toBe(false);
|
||||
expect(report.timing.openclaw.totalWallClockMs).toBe(40);
|
||||
expect(report.timing.codex.totalWallClockMs).toBe(37);
|
||||
expect(report.scenarios[2]).toMatchObject({
|
||||
openclawWallClockMs: null,
|
||||
codexWallClockMs: null,
|
||||
fasterRuntime: null,
|
||||
speedupPercent: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("reports missing runtime timing as unavailable instead of zero", () => {
|
||||
const report = buildQaRuntimeParityReport({
|
||||
summary: {
|
||||
scenarios: [{ name: "Missing runtime capture", status: "fail" }],
|
||||
counts: { total: 1, passed: 0, failed: 1 },
|
||||
run: { providerMode: "live-frontier", runtimePair: ["openclaw", "codex"] },
|
||||
},
|
||||
});
|
||||
|
||||
expect(report.timing).toEqual({
|
||||
openclaw: { totalWallClockMs: null, p50WallClockMs: null, p90WallClockMs: null },
|
||||
codex: { totalWallClockMs: null, p50WallClockMs: null, p90WallClockMs: null },
|
||||
fasterRuntime: null,
|
||||
speedupPercent: null,
|
||||
});
|
||||
|
||||
const markdown = renderQaRuntimeParityMarkdownReport(report);
|
||||
expect(markdown).toContain("| openclaw | N/A | N/A | N/A |");
|
||||
expect(markdown).toContain("| codex | N/A | N/A | N/A |");
|
||||
expect(markdown).toContain("- Faster runtime: N/A");
|
||||
});
|
||||
|
||||
it("fails runtime parity reports when a runtime cell has a hard failure", () => {
|
||||
@@ -1070,7 +1074,12 @@ status=done`,
|
||||
|
||||
expect(report).toContain("# OpenClaw Runtime Parity Report — openclaw vs codex");
|
||||
expect(report).toContain("| Tool-call-shape drift | 1 |");
|
||||
expect(report).toContain("## Runtime Timing");
|
||||
expect(report).toContain("| openclaw | 40 ms | 20 ms | 20 ms |");
|
||||
expect(report).toContain("| codex | 37 ms | 18 ms | 19 ms |");
|
||||
expect(report).toContain("- Faster runtime: codex 8.1% faster");
|
||||
expect(report).toContain("### Compaction retry after mutating tool");
|
||||
expect(report).toContain("- drift: tool-call-shape");
|
||||
expect(report).toContain("- wall time: openclaw 20 ms; codex 19 ms; codex 5.3% faster");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { makeRuntimeParitySummary } from "./agentic-parity-report-test-helpers.js";
|
||||
import {
|
||||
buildQaRuntimeParityReport,
|
||||
renderQaRuntimeParityMarkdownReport,
|
||||
} from "./agentic-parity-report.js";
|
||||
import {
|
||||
measureRuntimeParityCellTiming,
|
||||
summarizeRuntimeParityTiming,
|
||||
} from "./runtime-parity-timing.js";
|
||||
|
||||
describe("qa runtime parity timing reporting", () => {
|
||||
it("excludes gateway bootstrap from measured runtime execution", () => {
|
||||
expect(
|
||||
measureRuntimeParityCellTiming({
|
||||
suiteStartedAt: new Date("2026-07-27T12:00:00.000Z"),
|
||||
scenarioStartedAt: new Date("2026-07-27T12:00:12.000Z"),
|
||||
scenarioFinishedAt: new Date("2026-07-27T12:00:15.500Z"),
|
||||
}),
|
||||
).toEqual({ wallClockMs: 3_500, bootstrapWallClockMs: 12_000 });
|
||||
});
|
||||
|
||||
it("keeps minimum turn timing without inventing negative bootstrap", () => {
|
||||
const startedAt = new Date("2026-07-27T12:00:00.000Z");
|
||||
expect(
|
||||
measureRuntimeParityCellTiming({
|
||||
suiteStartedAt: startedAt,
|
||||
scenarioStartedAt: startedAt,
|
||||
scenarioFinishedAt: startedAt,
|
||||
}),
|
||||
).toEqual({ wallClockMs: 1, bootstrapWallClockMs: 0 });
|
||||
});
|
||||
|
||||
it("excludes failed attempts and retry backoff from both turn and bootstrap timing", () => {
|
||||
expect(
|
||||
measureRuntimeParityCellTiming({
|
||||
suiteStartedAt: new Date("2026-07-27T12:00:00.000Z"),
|
||||
bootstrapFinishedAt: new Date("2026-07-27T12:00:12.000Z"),
|
||||
scenarioStartedAt: new Date("2026-07-27T12:00:20.000Z"),
|
||||
scenarioFinishedAt: new Date("2026-07-27T12:00:22.500Z"),
|
||||
}),
|
||||
).toEqual({ wallClockMs: 2_500, bootstrapWallClockMs: 12_000 });
|
||||
});
|
||||
|
||||
it("reports gateway bootstrap separately without changing runtime comparisons", () => {
|
||||
const summary = makeRuntimeParitySummary();
|
||||
for (const scenario of summary.scenarios) {
|
||||
if (scenario.runtimeParity) {
|
||||
scenario.runtimeParity.cells.openclaw.bootstrapWallClockMs = 4_000;
|
||||
scenario.runtimeParity.cells.codex.bootstrapWallClockMs = 12_000;
|
||||
}
|
||||
}
|
||||
|
||||
const report = buildQaRuntimeParityReport({ summary });
|
||||
|
||||
expect(report.pass).toBe(true);
|
||||
expect(report.timing.openclaw.totalWallClockMs).toBe(40);
|
||||
expect(report.timing.codex.totalWallClockMs).toBe(37);
|
||||
expect(report.timing.bootstrap).toEqual({
|
||||
openclaw: { totalWallClockMs: 8_000, p50WallClockMs: 4_000, p90WallClockMs: 4_000 },
|
||||
codex: { totalWallClockMs: 24_000, p50WallClockMs: 12_000, p90WallClockMs: 12_000 },
|
||||
});
|
||||
expect(report.scenarios[0]).toMatchObject({
|
||||
openclawWallClockMs: 20,
|
||||
codexWallClockMs: 18,
|
||||
openclawBootstrapWallClockMs: 4_000,
|
||||
codexBootstrapWallClockMs: 12_000,
|
||||
fasterRuntime: "codex",
|
||||
});
|
||||
const markdown = renderQaRuntimeParityMarkdownReport(report);
|
||||
expect(markdown).toContain("## Gateway Bootstrap (Excluded From Runtime Timing)");
|
||||
expect(markdown).toContain("| openclaw | 8000 ms | 4000 ms | 4000 ms |");
|
||||
expect(markdown).toContain("| codex | 24000 ms | 12000 ms | 12000 ms |");
|
||||
expect(markdown).toContain("- gateway bootstrap (excluded): openclaw 4000 ms; codex 12000 ms");
|
||||
});
|
||||
|
||||
it("reports when OpenClaw is faster without changing the parity verdict", () => {
|
||||
const summary = makeRuntimeParitySummary();
|
||||
for (const scenario of summary.scenarios) {
|
||||
if (scenario.runtimeParity) {
|
||||
scenario.runtimeParity.cells.codex.wallClockMs = 30;
|
||||
}
|
||||
}
|
||||
const report = buildQaRuntimeParityReport({ summary });
|
||||
expect(report.pass).toBe(true);
|
||||
expect(report.timing.fasterRuntime).toBe("openclaw");
|
||||
expect(report.timing.speedupPercent).toBeCloseTo(50);
|
||||
expect(report.scenarios[0]).toMatchObject({
|
||||
openclawWallClockMs: 20,
|
||||
codexWallClockMs: 30,
|
||||
fasterRuntime: "openclaw",
|
||||
});
|
||||
expect(report.scenarios[0]?.speedupPercent).toBeCloseTo(50);
|
||||
});
|
||||
|
||||
it("does not report an infinite speedup for a zero-duration runtime", () => {
|
||||
const summary = makeRuntimeParitySummary();
|
||||
for (const scenario of summary.scenarios) {
|
||||
if (scenario.runtimeParity) {
|
||||
scenario.runtimeParity.cells.openclaw.wallClockMs = 0;
|
||||
}
|
||||
}
|
||||
const report = buildQaRuntimeParityReport({ summary });
|
||||
expect(report.timing.fasterRuntime).toBe("openclaw");
|
||||
expect(report.timing.speedupPercent).toBeNull();
|
||||
expect(report.scenarios[0]).toMatchObject({
|
||||
fasterRuntime: "openclaw",
|
||||
speedupPercent: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("compares only paired captures while retaining independently measured totals", () => {
|
||||
const timing = summarizeRuntimeParityTiming([
|
||||
{ openclawWallClockMs: 20, codexWallClockMs: 30 },
|
||||
{ openclawWallClockMs: 1_000, codexWallClockMs: null },
|
||||
]);
|
||||
|
||||
expect(timing.openclaw.totalWallClockMs).toBe(1_020);
|
||||
expect(timing.codex.totalWallClockMs).toBe(30);
|
||||
expect(timing.fasterRuntime).toBe("openclaw");
|
||||
expect(timing.speedupPercent).toBeCloseTo(50);
|
||||
});
|
||||
|
||||
it("does not compare independently measured totals without a complete pair", () => {
|
||||
const timing = summarizeRuntimeParityTiming([
|
||||
{ openclawWallClockMs: 20, codexWallClockMs: null },
|
||||
{ openclawWallClockMs: null, codexWallClockMs: 30 },
|
||||
]);
|
||||
|
||||
expect(timing.openclaw.totalWallClockMs).toBe(20);
|
||||
expect(timing.codex.totalWallClockMs).toBe(30);
|
||||
expect(timing.fasterRuntime).toBeNull();
|
||||
expect(timing.speedupPercent).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -1,20 +1,29 @@
|
||||
import {
|
||||
aggregateRuntimeParityCacheUsage,
|
||||
summarizeRuntimeParityCacheUsage,
|
||||
} from "./agentic-parity-cache-usage.js";
|
||||
import type {
|
||||
QaRuntimeParityReport,
|
||||
QaRuntimeParityScenarioReport,
|
||||
} from "./agentic-parity-runtime-report-contract.js";
|
||||
// Qa Lab plugin module implements agentic parity report behavior.
|
||||
import {
|
||||
QA_AGENTIC_PARITY_SCENARIO_TITLES,
|
||||
QA_AGENTIC_PARITY_TOOL_BACKED_SCENARIO_TITLES,
|
||||
} from "./agentic-parity.js";
|
||||
import type {
|
||||
RuntimeId,
|
||||
RuntimeParityDrift,
|
||||
RuntimeParityResult,
|
||||
RuntimeParityUsagePolicy,
|
||||
} from "./runtime-parity.js";
|
||||
import {
|
||||
compareRuntimeWallClockMs,
|
||||
summarizeRuntimeParityTiming,
|
||||
} from "./runtime-parity-timing.js";
|
||||
import type { RuntimeId, RuntimeParityDrift, RuntimeParityResult } from "./runtime-parity.js";
|
||||
import {
|
||||
isRuntimeParityResultPass,
|
||||
resolveRuntimeParityUsagePolicy,
|
||||
runtimeParityCellStatus,
|
||||
} from "./runtime-parity.js";
|
||||
|
||||
export { renderQaRuntimeParityMarkdownReport } from "./agentic-parity-runtime-markdown.js";
|
||||
|
||||
type QaParityReportStep = {
|
||||
name: string;
|
||||
status: "pass" | "fail" | "skip";
|
||||
@@ -63,35 +72,6 @@ export type QaRuntimeParitySuiteSummary = Omit<QaParitySuiteSummary, "scenarios"
|
||||
scenarios: QaRuntimeParitySuiteScenario[];
|
||||
};
|
||||
|
||||
type QaRuntimeParityScenarioReport = {
|
||||
name: string;
|
||||
status: "pass" | "fail";
|
||||
runtimeParityUsage: RuntimeParityUsagePolicy;
|
||||
drift: RuntimeParityDrift | "missing";
|
||||
driftDetails?: string;
|
||||
openclawStatus: "pass" | "fail" | "missing";
|
||||
codexStatus: "pass" | "fail" | "missing";
|
||||
openclawTokens: number;
|
||||
codexTokens: number;
|
||||
openclawToolCalls: number;
|
||||
codexToolCalls: number;
|
||||
};
|
||||
|
||||
type QaRuntimeParityReport = {
|
||||
runtimePair: [RuntimeId, RuntimeId];
|
||||
comparedAt: string;
|
||||
providerMode?: string;
|
||||
primaryModel?: string;
|
||||
totalScenarios: number;
|
||||
passedScenarios: number;
|
||||
failedScenarios: number;
|
||||
driftCounts: Record<RuntimeParityDrift, number>;
|
||||
scenarios: QaRuntimeParityScenarioReport[];
|
||||
pass: boolean;
|
||||
failures: string[];
|
||||
notes: string[];
|
||||
};
|
||||
|
||||
type QaAgenticParityMetrics = {
|
||||
totalScenarios: number;
|
||||
passedScenarios: number;
|
||||
@@ -666,8 +646,14 @@ export function buildQaRuntimeParityReport(params: {
|
||||
codexStatus: "missing",
|
||||
openclawTokens: 0,
|
||||
codexTokens: 0,
|
||||
openclawUsage: null,
|
||||
codexUsage: null,
|
||||
openclawToolCalls: 0,
|
||||
codexToolCalls: 0,
|
||||
openclawWallClockMs: null,
|
||||
codexWallClockMs: null,
|
||||
fasterRuntime: null,
|
||||
speedupPercent: null,
|
||||
} satisfies QaRuntimeParityScenarioReport;
|
||||
}
|
||||
driftCounts[parity.drift] += 1;
|
||||
@@ -687,8 +673,25 @@ export function buildQaRuntimeParityReport(params: {
|
||||
codexStatus,
|
||||
openclawTokens: openclawCell.usage.totalTokens,
|
||||
codexTokens: codexCell.usage.totalTokens,
|
||||
openclawUsage:
|
||||
runtimeParityUsage.expectation === "not-applicable"
|
||||
? null
|
||||
: summarizeRuntimeParityCacheUsage(openclawCell.usage),
|
||||
codexUsage:
|
||||
runtimeParityUsage.expectation === "not-applicable"
|
||||
? null
|
||||
: summarizeRuntimeParityCacheUsage(codexCell.usage),
|
||||
openclawToolCalls: openclawCell.toolCalls.length,
|
||||
codexToolCalls: codexCell.toolCalls.length,
|
||||
openclawWallClockMs: openclawCell.wallClockMs,
|
||||
codexWallClockMs: codexCell.wallClockMs,
|
||||
...(openclawCell.bootstrapWallClockMs === undefined
|
||||
? {}
|
||||
: { openclawBootstrapWallClockMs: openclawCell.bootstrapWallClockMs }),
|
||||
...(codexCell.bootstrapWallClockMs === undefined
|
||||
? {}
|
||||
: { codexBootstrapWallClockMs: codexCell.bootstrapWallClockMs }),
|
||||
...compareRuntimeWallClockMs(openclawCell.wallClockMs, codexCell.wallClockMs),
|
||||
} satisfies QaRuntimeParityScenarioReport;
|
||||
if (parityStatus === "fail") {
|
||||
failures.push(
|
||||
@@ -712,7 +715,6 @@ export function buildQaRuntimeParityReport(params: {
|
||||
if (scenarios.length === 0 || totalScenarios <= 0) {
|
||||
failures.push("Runtime parity report has no executed scenarios.");
|
||||
}
|
||||
|
||||
return {
|
||||
runtimePair,
|
||||
comparedAt: params.comparedAt ?? new Date().toISOString(),
|
||||
@@ -723,76 +725,18 @@ export function buildQaRuntimeParityReport(params: {
|
||||
failedScenarios,
|
||||
driftCounts,
|
||||
scenarios,
|
||||
timing: summarizeRuntimeParityTiming(scenarios),
|
||||
usage: {
|
||||
openclaw: aggregateRuntimeParityCacheUsage(scenarios, "openclaw"),
|
||||
codex: aggregateRuntimeParityCacheUsage(scenarios, "codex"),
|
||||
},
|
||||
pass: failures.length === 0 && failedScenarios === 0,
|
||||
failures,
|
||||
notes: [
|
||||
"Runtime parity fails runtime, transport, and failure-mode drift; structural and tool-shape drift is recorded as advisory when both runtimes complete.",
|
||||
"Token totals here are assistant-message usage captured from the normalized transcript, not provider transport payloads.",
|
||||
"Cache-hit percentages use cached input divided by cached, uncached, and cache-write input; output tokens are excluded from the denominator.",
|
||||
"Wall-clock timings cover each complete QA runtime cell, including gateway, model, and tool execution; they are not provider-reported turn durations.",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export function renderQaRuntimeParityMarkdownReport(report: QaRuntimeParityReport): string {
|
||||
const lines = [
|
||||
`# OpenClaw Runtime Parity Report — ${report.runtimePair[0]} vs ${report.runtimePair[1]}`,
|
||||
"",
|
||||
`- Compared at: ${report.comparedAt}`,
|
||||
`- Provider mode: ${report.providerMode ?? "unknown"}`,
|
||||
`- Primary model: ${report.primaryModel ?? "unknown"}`,
|
||||
`- Verdict: ${report.pass ? "pass" : "fail"}`,
|
||||
"",
|
||||
"## Aggregate Metrics",
|
||||
"",
|
||||
"| Metric | Value |",
|
||||
"| --- | ---: |",
|
||||
`| Total scenarios | ${report.totalScenarios} |`,
|
||||
`| Passed scenarios | ${report.passedScenarios} |`,
|
||||
`| Failed scenarios | ${report.failedScenarios} |`,
|
||||
`| No drift | ${report.driftCounts.none} |`,
|
||||
`| Text-only drift | ${report.driftCounts["text-only"]} |`,
|
||||
`| Tool-call-shape drift | ${report.driftCounts["tool-call-shape"]} |`,
|
||||
`| Tool-result-shape drift | ${report.driftCounts["tool-result-shape"]} |`,
|
||||
`| Structural drift | ${report.driftCounts.structural} |`,
|
||||
`| Failure-mode drift | ${report.driftCounts["failure-mode"]} |`,
|
||||
"",
|
||||
];
|
||||
|
||||
if (report.failures.length > 0) {
|
||||
lines.push("## Gate Failures", "");
|
||||
for (const failure of report.failures) {
|
||||
lines.push(`- ${failure}`);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
lines.push("## Scenario Comparison", "");
|
||||
for (const scenario of report.scenarios) {
|
||||
const usageNotApplicable = scenario.runtimeParityUsage.expectation === "not-applicable";
|
||||
const openclawTokens = usageNotApplicable ? "N/A" : String(scenario.openclawTokens);
|
||||
const codexTokens = usageNotApplicable ? "N/A" : String(scenario.codexTokens);
|
||||
lines.push(`### ${scenario.name}`, "");
|
||||
lines.push(`- status: ${scenario.status}`);
|
||||
lines.push(`- drift: ${scenario.drift}`);
|
||||
lines.push(
|
||||
`- openclaw: ${scenario.openclawStatus} (${scenario.openclawToolCalls} tool calls, ${openclawTokens} tokens)`,
|
||||
);
|
||||
lines.push(
|
||||
`- codex: ${scenario.codexStatus} (${scenario.codexToolCalls} tool calls, ${codexTokens} tokens)`,
|
||||
);
|
||||
if (scenario.runtimeParityUsage.expectation === "not-applicable") {
|
||||
lines.push(`- assistant-message usage: N/A (${scenario.runtimeParityUsage.reason})`);
|
||||
}
|
||||
if (scenario.driftDetails) {
|
||||
lines.push(`- details: ${scenario.driftDetails}`);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
lines.push("## Notes", "");
|
||||
for (const note of report.notes) {
|
||||
lines.push(`- ${note}`);
|
||||
}
|
||||
lines.push("");
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
import {
|
||||
formatRuntimeCacheCount,
|
||||
formatRuntimeCacheHitPercent,
|
||||
} from "./agentic-parity-cache-usage.js";
|
||||
import type { QaRuntimeParityReport } from "./agentic-parity-runtime-report-contract.js";
|
||||
import { formatRuntimeSpeedComparison, formatRuntimeWallClockMs } from "./runtime-parity-timing.js";
|
||||
|
||||
export function renderQaRuntimeParityMarkdownReport(report: QaRuntimeParityReport): string {
|
||||
const lines = [
|
||||
`# OpenClaw Runtime Parity Report — ${report.runtimePair[0]} vs ${report.runtimePair[1]}`,
|
||||
"",
|
||||
`- Compared at: ${report.comparedAt}`,
|
||||
`- Provider mode: ${report.providerMode ?? "unknown"}`,
|
||||
`- Primary model: ${report.primaryModel ?? "unknown"}`,
|
||||
`- Verdict: ${report.pass ? "pass" : "fail"}`,
|
||||
"",
|
||||
"## Aggregate Metrics",
|
||||
"",
|
||||
"| Metric | Value |",
|
||||
"| --- | ---: |",
|
||||
`| Total scenarios | ${report.totalScenarios} |`,
|
||||
`| Passed scenarios | ${report.passedScenarios} |`,
|
||||
`| Failed scenarios | ${report.failedScenarios} |`,
|
||||
`| No drift | ${report.driftCounts.none} |`,
|
||||
`| Text-only drift | ${report.driftCounts["text-only"]} |`,
|
||||
`| Tool-call-shape drift | ${report.driftCounts["tool-call-shape"]} |`,
|
||||
`| Tool-result-shape drift | ${report.driftCounts["tool-result-shape"]} |`,
|
||||
`| Structural drift | ${report.driftCounts.structural} |`,
|
||||
`| Failure-mode drift | ${report.driftCounts["failure-mode"]} |`,
|
||||
"",
|
||||
"## Prompt Cache",
|
||||
"",
|
||||
"| Runtime | Gross input | Uncached input | Cached input | Cache writes | Output | Total tokens | Cache hit |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
|
||||
`| openclaw | ${formatRuntimeCacheCount(report.usage.openclaw?.grossInputTokens)} | ${formatRuntimeCacheCount(report.usage.openclaw?.uncachedInputTokens)} | ${formatRuntimeCacheCount(report.usage.openclaw?.cachedInputTokens)} | ${formatRuntimeCacheCount(report.usage.openclaw?.cacheWriteTokens)} | ${formatRuntimeCacheCount(report.usage.openclaw?.outputTokens)} | ${formatRuntimeCacheCount(report.usage.openclaw?.totalTokens)} | ${formatRuntimeCacheHitPercent(report.usage.openclaw?.cacheHitPercent)} |`,
|
||||
`| codex | ${formatRuntimeCacheCount(report.usage.codex?.grossInputTokens)} | ${formatRuntimeCacheCount(report.usage.codex?.uncachedInputTokens)} | ${formatRuntimeCacheCount(report.usage.codex?.cachedInputTokens)} | ${formatRuntimeCacheCount(report.usage.codex?.cacheWriteTokens)} | ${formatRuntimeCacheCount(report.usage.codex?.outputTokens)} | ${formatRuntimeCacheCount(report.usage.codex?.totalTokens)} | ${formatRuntimeCacheHitPercent(report.usage.codex?.cacheHitPercent)} |`,
|
||||
"",
|
||||
"## Runtime Timing",
|
||||
"",
|
||||
"| Runtime | Total wall time | p50 per scenario | p90 per scenario |",
|
||||
"| --- | ---: | ---: | ---: |",
|
||||
`| openclaw | ${formatRuntimeWallClockMs(report.timing.openclaw.totalWallClockMs)} | ${formatRuntimeWallClockMs(report.timing.openclaw.p50WallClockMs)} | ${formatRuntimeWallClockMs(report.timing.openclaw.p90WallClockMs)} |`,
|
||||
`| codex | ${formatRuntimeWallClockMs(report.timing.codex.totalWallClockMs)} | ${formatRuntimeWallClockMs(report.timing.codex.p50WallClockMs)} | ${formatRuntimeWallClockMs(report.timing.codex.p90WallClockMs)} |`,
|
||||
"",
|
||||
`- Faster runtime: ${formatRuntimeSpeedComparison(report.timing)}`,
|
||||
"",
|
||||
];
|
||||
if (report.timing.bootstrap) {
|
||||
lines.push(
|
||||
"## Gateway Bootstrap (Excluded From Runtime Timing)",
|
||||
"",
|
||||
"| Runtime | Total bootstrap | p50 per scenario | p90 per scenario |",
|
||||
"| --- | ---: | ---: | ---: |",
|
||||
`| openclaw | ${formatRuntimeWallClockMs(report.timing.bootstrap.openclaw.totalWallClockMs)} | ${formatRuntimeWallClockMs(report.timing.bootstrap.openclaw.p50WallClockMs)} | ${formatRuntimeWallClockMs(report.timing.bootstrap.openclaw.p90WallClockMs)} |`,
|
||||
`| codex | ${formatRuntimeWallClockMs(report.timing.bootstrap.codex.totalWallClockMs)} | ${formatRuntimeWallClockMs(report.timing.bootstrap.codex.p50WallClockMs)} | ${formatRuntimeWallClockMs(report.timing.bootstrap.codex.p90WallClockMs)} |`,
|
||||
"",
|
||||
);
|
||||
}
|
||||
if (report.failures.length > 0) {
|
||||
lines.push("## Gate Failures", "");
|
||||
for (const failure of report.failures) {
|
||||
lines.push(`- ${failure}`);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
lines.push("## Scenario Comparison", "");
|
||||
for (const scenario of report.scenarios) {
|
||||
const usageNotApplicable = scenario.runtimeParityUsage.expectation === "not-applicable";
|
||||
const openclawTokens = usageNotApplicable ? "N/A" : String(scenario.openclawTokens);
|
||||
const codexTokens = usageNotApplicable ? "N/A" : String(scenario.codexTokens);
|
||||
lines.push(`### ${scenario.name}`, "");
|
||||
lines.push(`- status: ${scenario.status}`);
|
||||
lines.push(`- drift: ${scenario.drift}`);
|
||||
lines.push(
|
||||
`- openclaw: ${scenario.openclawStatus} (${scenario.openclawToolCalls} tool calls, ${openclawTokens} tokens)`,
|
||||
);
|
||||
lines.push(
|
||||
`- codex: ${scenario.codexStatus} (${scenario.codexToolCalls} tool calls, ${codexTokens} tokens)`,
|
||||
);
|
||||
lines.push(
|
||||
`- wall time: openclaw ${formatRuntimeWallClockMs(scenario.openclawWallClockMs)}; codex ${formatRuntimeWallClockMs(scenario.codexWallClockMs)}; ${formatRuntimeSpeedComparison(scenario)}`,
|
||||
);
|
||||
if (
|
||||
scenario.openclawBootstrapWallClockMs !== undefined ||
|
||||
scenario.codexBootstrapWallClockMs !== undefined
|
||||
) {
|
||||
lines.push(
|
||||
`- gateway bootstrap (excluded): openclaw ${formatRuntimeWallClockMs(scenario.openclawBootstrapWallClockMs ?? null)}; codex ${formatRuntimeWallClockMs(scenario.codexBootstrapWallClockMs ?? null)}`,
|
||||
);
|
||||
}
|
||||
lines.push(
|
||||
`- prompt cache: openclaw ${formatRuntimeCacheHitPercent(scenario.openclawUsage?.cacheHitPercent)} (${formatRuntimeCacheCount(scenario.openclawUsage?.cachedInputTokens)} cached, ${formatRuntimeCacheCount(scenario.openclawUsage?.uncachedInputTokens)} uncached input); codex ${formatRuntimeCacheHitPercent(scenario.codexUsage?.cacheHitPercent)} (${formatRuntimeCacheCount(scenario.codexUsage?.cachedInputTokens)} cached, ${formatRuntimeCacheCount(scenario.codexUsage?.uncachedInputTokens)} uncached input)`,
|
||||
);
|
||||
if (scenario.runtimeParityUsage.expectation === "not-applicable") {
|
||||
lines.push(`- assistant-message usage: N/A (${scenario.runtimeParityUsage.reason})`);
|
||||
}
|
||||
if (scenario.driftDetails) {
|
||||
lines.push(`- details: ${scenario.driftDetails}`);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
lines.push("## Notes", "");
|
||||
for (const note of report.notes) {
|
||||
lines.push(`- ${note}`);
|
||||
}
|
||||
lines.push("");
|
||||
return lines.join("\n");
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import type { QaRuntimeParityCacheUsage } from "./agentic-parity-cache-usage.js";
|
||||
import type { QaRuntimeTiming } from "./runtime-parity-timing.js";
|
||||
import type { RuntimeId, RuntimeParityDrift, RuntimeParityUsagePolicy } from "./runtime-parity.js";
|
||||
|
||||
export type QaRuntimeParityScenarioReport = {
|
||||
name: string;
|
||||
status: "pass" | "fail";
|
||||
runtimeParityUsage: RuntimeParityUsagePolicy;
|
||||
drift: RuntimeParityDrift | "missing";
|
||||
driftDetails?: string;
|
||||
openclawStatus: "pass" | "fail" | "missing";
|
||||
codexStatus: "pass" | "fail" | "missing";
|
||||
openclawTokens: number;
|
||||
codexTokens: number;
|
||||
openclawUsage: QaRuntimeParityCacheUsage | null;
|
||||
codexUsage: QaRuntimeParityCacheUsage | null;
|
||||
openclawToolCalls: number;
|
||||
codexToolCalls: number;
|
||||
openclawWallClockMs: number | null;
|
||||
codexWallClockMs: number | null;
|
||||
openclawBootstrapWallClockMs?: number;
|
||||
codexBootstrapWallClockMs?: number;
|
||||
fasterRuntime: RuntimeId | "tie" | null;
|
||||
speedupPercent: number | null;
|
||||
};
|
||||
|
||||
export type QaRuntimeParityReport = {
|
||||
runtimePair: [RuntimeId, RuntimeId];
|
||||
comparedAt: string;
|
||||
providerMode?: string;
|
||||
primaryModel?: string;
|
||||
totalScenarios: number;
|
||||
passedScenarios: number;
|
||||
failedScenarios: number;
|
||||
driftCounts: Record<RuntimeParityDrift, number>;
|
||||
scenarios: QaRuntimeParityScenarioReport[];
|
||||
timing: QaRuntimeTiming;
|
||||
usage: {
|
||||
openclaw: QaRuntimeParityCacheUsage | null;
|
||||
codex: QaRuntimeParityCacheUsage | null;
|
||||
};
|
||||
pass: boolean;
|
||||
failures: string[];
|
||||
notes: string[];
|
||||
};
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
type QaParitySuiteSummary,
|
||||
type QaRuntimeParitySuiteSummary,
|
||||
} from "./agentic-parity-report.js";
|
||||
import type { QaRuntimeParityReport } from "./agentic-parity-runtime-report-contract.js";
|
||||
import { resolveQaParityPackScenarioIds } from "./agentic-parity.js";
|
||||
import { createQaArtifactRunId } from "./artifact-run-id.js";
|
||||
import { runQaCharacterEval, type QaCharacterModelOptions } from "./character-eval.js";
|
||||
@@ -1115,7 +1116,7 @@ export async function runQaParityReportCommand(opts: {
|
||||
const summary = JSON.parse(
|
||||
await fs.readFile(summaryPath, "utf8"),
|
||||
) as QaRuntimeParitySuiteSummary;
|
||||
const reportPayload = buildQaRuntimeParityReport({ summary });
|
||||
const reportPayload: QaRuntimeParityReport = buildQaRuntimeParityReport({ summary });
|
||||
const report = renderQaRuntimeParityMarkdownReport(reportPayload);
|
||||
const reportPath = path.join(outputDir, "qa-runtime-parity-report.md");
|
||||
const runtimeSummaryPath = path.join(outputDir, "qa-runtime-parity-summary.json");
|
||||
|
||||
@@ -229,6 +229,67 @@ describe("Gateway child fixture helpers", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("stages native Codex model metadata before starting the private mock runtime", async () => {
|
||||
const tempRoot = await tempDirs.makeTempDir("qa-codex-model-catalog-");
|
||||
const modelCatalogPath = await testing.stageQaCodexMockModelCatalog({
|
||||
tempRoot,
|
||||
forcedRuntime: "codex",
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.6-luna",
|
||||
alternateModel: "mock-openai/gpt-5.6-luna-alt",
|
||||
});
|
||||
|
||||
expect(modelCatalogPath).toBe(path.join(tempRoot, "codex-model-catalog.json"));
|
||||
const catalog = JSON.parse(await readFile(modelCatalogPath!, "utf8")) as {
|
||||
models: Array<Record<string, unknown>>;
|
||||
};
|
||||
expect(catalog.models).toEqual([
|
||||
expect.objectContaining({
|
||||
slug: "gpt-5.6-luna",
|
||||
apply_patch_tool_type: "freeform",
|
||||
tool_mode: "direct",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
slug: "gpt-5.6-luna-alt",
|
||||
apply_patch_tool_type: "freeform",
|
||||
tool_mode: "direct",
|
||||
}),
|
||||
]);
|
||||
expect(
|
||||
testing.buildQaForcedRuntimeEnvPatch({
|
||||
forcedRuntime: "codex",
|
||||
providerMode: "mock-openai",
|
||||
providerBaseUrl: "http://127.0.0.1:44080/v1",
|
||||
codexModelCatalogPath: modelCatalogPath,
|
||||
}),
|
||||
).toEqual(
|
||||
expect.objectContaining({
|
||||
OPENCLAW_CODEX_APP_SERVER_ARGS: `app-server -c openai_base_url=http://127.0.0.1:44080/v1 -c ${JSON.stringify(`model_catalog_json=${modelCatalogPath}`)} --listen stdio://`,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not stage a Codex catalog for other runtimes or live providers", async () => {
|
||||
const tempRoot = await tempDirs.makeTempDir("qa-codex-model-catalog-unused-");
|
||||
await expect(
|
||||
testing.stageQaCodexMockModelCatalog({
|
||||
tempRoot,
|
||||
forcedRuntime: "openclaw",
|
||||
providerMode: "mock-openai",
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
await expect(
|
||||
testing.stageQaCodexMockModelCatalog({
|
||||
tempRoot,
|
||||
forcedRuntime: "codex",
|
||||
providerMode: "live-frontier",
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
await expect(
|
||||
readFile(path.join(tempRoot, "codex-model-catalog.json"), "utf8"),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("resolves source and built Gateway CLI commands", async () => {
|
||||
const repoRoot = await tempDirs.makeTempDir("qa-gateway-command-");
|
||||
await mkdir(path.join(repoRoot, "src"), { recursive: true });
|
||||
|
||||
@@ -68,6 +68,7 @@ import {
|
||||
stageQaLiveAnthropicSetupToken,
|
||||
} from "./providers/live-frontier/auth.js";
|
||||
import { stageQaMockAuthProfiles } from "./providers/shared/mock-auth.js";
|
||||
import { listMockCodexModelInfos } from "./providers/shared/mock-model-config.js";
|
||||
import { seedQaAgentWorkspace } from "./qa-agent-workspace.js";
|
||||
import { buildQaGatewayConfig, type QaThinkingLevel } from "./qa-gateway-config.js";
|
||||
import type { QaTransportAdapter } from "./qa-transport.js";
|
||||
@@ -432,10 +433,33 @@ export function buildQaRuntimeEnv(params: {
|
||||
return scrubQaGatewayChildSecretEnv(normalizedEnv);
|
||||
}
|
||||
|
||||
async function stageQaCodexMockModelCatalog(params: {
|
||||
tempRoot: string;
|
||||
forcedRuntime?: RuntimeId;
|
||||
providerMode: QaProviderMode;
|
||||
primaryModel?: string;
|
||||
alternateModel?: string;
|
||||
}): Promise<string | undefined> {
|
||||
if (params.forcedRuntime !== "codex" || params.providerMode !== "mock-openai") {
|
||||
return undefined;
|
||||
}
|
||||
const modelCatalogPath = path.join(params.tempRoot, "codex-model-catalog.json");
|
||||
const selectedModelRefs = [params.primaryModel, params.alternateModel].filter(
|
||||
(model): model is string => typeof model === "string" && model.length > 0,
|
||||
);
|
||||
await fs.writeFile(
|
||||
modelCatalogPath,
|
||||
`${JSON.stringify({ models: listMockCodexModelInfos(selectedModelRefs) }, null, 2)}\n`,
|
||||
{ encoding: "utf8", mode: 0o600 },
|
||||
);
|
||||
return modelCatalogPath;
|
||||
}
|
||||
|
||||
function buildQaForcedRuntimeEnvPatch(params: {
|
||||
forcedRuntime?: RuntimeId;
|
||||
providerMode: QaProviderMode;
|
||||
providerBaseUrl?: string;
|
||||
codexModelCatalogPath?: string;
|
||||
}): NodeJS.ProcessEnv | undefined {
|
||||
if (!params.forcedRuntime) {
|
||||
return undefined;
|
||||
@@ -451,7 +475,11 @@ function buildQaForcedRuntimeEnvPatch(params: {
|
||||
if (!providerBaseUrl) {
|
||||
throw new Error("forced Codex mock QA requires the managed mock provider URL");
|
||||
}
|
||||
patch.OPENCLAW_CODEX_APP_SERVER_ARGS = `app-server -c openai_base_url=${providerBaseUrl} --listen stdio://`;
|
||||
if (!params.codexModelCatalogPath) {
|
||||
throw new Error("forced Codex mock QA requires the staged native model catalog");
|
||||
}
|
||||
const modelCatalogOverride = JSON.stringify(`model_catalog_json=${params.codexModelCatalogPath}`);
|
||||
patch.OPENCLAW_CODEX_APP_SERVER_ARGS = `app-server -c openai_base_url=${providerBaseUrl} -c ${modelCatalogOverride} --listen stdio://`;
|
||||
patch.OPENAI_API_KEY = QA_MOCK_OPENAI_API_KEY;
|
||||
patch.CODEX_API_KEY = QA_MOCK_OPENAI_API_KEY;
|
||||
return patch;
|
||||
@@ -593,6 +621,7 @@ async function waitForQaGatewayRestartBoundary(params: {
|
||||
|
||||
export const testing = {
|
||||
assertQaArtifactDirWithinRepo,
|
||||
buildQaForcedRuntimeEnvPatch,
|
||||
buildQaRuntimeEnv,
|
||||
cleanupQaGatewayTempRoots,
|
||||
fetchLocalGatewayHealth,
|
||||
@@ -609,6 +638,7 @@ export const testing = {
|
||||
stageQaLiveApiKeyProfiles,
|
||||
stageQaLiveAnthropicSetupToken,
|
||||
stageQaMockAuthProfiles,
|
||||
stageQaCodexMockModelCatalog,
|
||||
resolveQaLiveCliAuthEnv,
|
||||
waitForQaGatewayRestartBoundary,
|
||||
resolveQaOwnerPluginIdsForProviderIds,
|
||||
@@ -1047,6 +1077,13 @@ export async function startQaGatewayChild(params: {
|
||||
fs.mkdir(xdgCacheHome, { recursive: true }),
|
||||
]);
|
||||
const providerMode = resolveQaGatewayChildProviderMode(params.providerMode);
|
||||
const codexModelCatalogPath = await stageQaCodexMockModelCatalog({
|
||||
tempRoot,
|
||||
forcedRuntime: params.forcedRuntime,
|
||||
providerMode,
|
||||
primaryModel: params.primaryModel,
|
||||
alternateModel: params.alternateModel,
|
||||
});
|
||||
const resolvedProvider = getQaProvider(providerMode);
|
||||
const liveProviderIds = resolvedProvider.usesModelProviderPlugins
|
||||
? [params.primaryModel, params.alternateModel]
|
||||
@@ -1309,6 +1346,7 @@ export async function startQaGatewayChild(params: {
|
||||
forcedRuntime: params.forcedRuntime,
|
||||
providerMode,
|
||||
providerBaseUrl: params.providerBaseUrl,
|
||||
codexModelCatalogPath,
|
||||
}),
|
||||
},
|
||||
forwardHostHomeForClaudeCli: liveProviderIds.includes("claude-cli"),
|
||||
|
||||
@@ -7,6 +7,7 @@ import { writeJson } from "../shared/http-json.js";
|
||||
export type ResponsesInputItem = Record<string, unknown>;
|
||||
|
||||
export type StreamEvent =
|
||||
| { type: "response.created"; response: { id: string } }
|
||||
| { type: "response.output_item.added"; item: Record<string, unknown> }
|
||||
| {
|
||||
type: "response.output_text.delta";
|
||||
@@ -23,6 +24,12 @@ export type StreamEvent =
|
||||
text: string;
|
||||
}
|
||||
| { type: "response.function_call_arguments.delta"; delta: string }
|
||||
| {
|
||||
type: "response.custom_tool_call_input.delta";
|
||||
item_id: string;
|
||||
call_id: string;
|
||||
delta: string;
|
||||
}
|
||||
| { type: "response.output_item.done"; item: Record<string, unknown> }
|
||||
| {
|
||||
type: "response.completed";
|
||||
|
||||
@@ -206,6 +206,18 @@ export function hasToolDefinition(body: Record<string, unknown>, name: string) {
|
||||
return [...tools, ...dynamicTools].some((tool) => toolDefinitionMentionsName(tool, name));
|
||||
}
|
||||
|
||||
export function hasDeclaredCustomTool(body: Record<string, unknown>, name: string) {
|
||||
const tools = Array.isArray(body.tools) ? body.tools : [];
|
||||
return tools.some(
|
||||
(tool) =>
|
||||
tool !== null &&
|
||||
typeof tool === "object" &&
|
||||
!Array.isArray(tool) &&
|
||||
(tool as Record<string, unknown>).type === "custom" &&
|
||||
(tool as Record<string, unknown>).name === name,
|
||||
);
|
||||
}
|
||||
|
||||
function toolDefinitionMentionsName(value: unknown, name: string, depth = 0): boolean {
|
||||
if (depth > 6 || !value || typeof value !== "object") {
|
||||
return false;
|
||||
|
||||
@@ -98,7 +98,10 @@ export function extractPlannedToolName(events: StreamEvent[]) {
|
||||
continue;
|
||||
}
|
||||
const item = event.item as { type?: unknown; name?: unknown };
|
||||
if (item.type === "function_call" && typeof item.name === "string") {
|
||||
if (
|
||||
(item.type === "function_call" || item.type === "custom_tool_call") &&
|
||||
typeof item.name === "string"
|
||||
) {
|
||||
return item.name;
|
||||
}
|
||||
}
|
||||
@@ -111,7 +114,10 @@ export function extractPlannedToolCallId(events: StreamEvent[]) {
|
||||
continue;
|
||||
}
|
||||
const item = event.item as { type?: unknown; call_id?: unknown };
|
||||
if (item.type === "function_call" && typeof item.call_id === "string") {
|
||||
if (
|
||||
(item.type === "function_call" || item.type === "custom_tool_call") &&
|
||||
typeof item.call_id === "string"
|
||||
) {
|
||||
return item.call_id;
|
||||
}
|
||||
}
|
||||
@@ -123,7 +129,10 @@ export function extractPlannedToolArgs(events: StreamEvent[]) {
|
||||
if (event.type !== "response.output_item.done") {
|
||||
continue;
|
||||
}
|
||||
const item = event.item as { type?: unknown; arguments?: unknown };
|
||||
const item = event.item as { type?: unknown; arguments?: unknown; input?: unknown };
|
||||
if (item.type === "custom_tool_call") {
|
||||
return typeof item.input === "string" ? { input: item.input } : undefined;
|
||||
}
|
||||
if (item.type !== "function_call" || typeof item.arguments !== "string") {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -100,15 +100,19 @@ function stringifyFunctionCallOutput(output: unknown): string {
|
||||
return "";
|
||||
}
|
||||
|
||||
function isResponsesToolCallOutput(item: ResponsesInputItem) {
|
||||
return item.type === "function_call_output" || item.type === "custom_tool_call_output";
|
||||
}
|
||||
|
||||
function extractFunctionCallOutputText(item: ResponsesInputItem) {
|
||||
if (item.type !== "function_call_output") {
|
||||
if (!isResponsesToolCallOutput(item)) {
|
||||
return "";
|
||||
}
|
||||
return stringifyFunctionCallOutput(item.output);
|
||||
}
|
||||
|
||||
function extractFunctionCallOutputCallId(item: ResponsesInputItem) {
|
||||
if (item.type !== "function_call_output") {
|
||||
if (!isResponsesToolCallOutput(item)) {
|
||||
return "";
|
||||
}
|
||||
const record = item as {
|
||||
@@ -124,7 +128,7 @@ function extractFunctionCallOutputCallId(item: ResponsesInputItem) {
|
||||
}
|
||||
|
||||
function functionCallOutputIsStructuredError(item: ResponsesInputItem) {
|
||||
if (item.type !== "function_call_output") {
|
||||
if (!isResponsesToolCallOutput(item)) {
|
||||
return false;
|
||||
}
|
||||
return item.is_error === true || item.isError === true;
|
||||
|
||||
@@ -0,0 +1,375 @@
|
||||
import { once } from "node:events";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { WebSocket, type RawData } from "ws";
|
||||
import { startQaMockOpenAiServer } from "./server.js";
|
||||
|
||||
const cleanups: Array<() => Promise<void>> = [];
|
||||
|
||||
afterEach(async () => {
|
||||
while (cleanups.length > 0) {
|
||||
await cleanups.pop()?.();
|
||||
}
|
||||
});
|
||||
|
||||
async function startServer() {
|
||||
const server = await startQaMockOpenAiServer({ host: "127.0.0.1", port: 0 });
|
||||
cleanups.push(async () => server.stop());
|
||||
return server;
|
||||
}
|
||||
|
||||
async function connectResponsesWebSocket(baseUrl: string) {
|
||||
const socket = new WebSocket(`${baseUrl.replace(/^http/u, "ws")}/v1/responses`);
|
||||
cleanups.push(async () => {
|
||||
if (socket.readyState !== WebSocket.CLOSED) {
|
||||
socket.terminate();
|
||||
}
|
||||
});
|
||||
await once(socket, "open");
|
||||
return socket;
|
||||
}
|
||||
|
||||
async function collectResponseEvents(socket: WebSocket, request: unknown) {
|
||||
return await new Promise<Array<Record<string, unknown>>>((resolve, reject) => {
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
const timeout = setTimeout(() => {
|
||||
cleanup();
|
||||
reject(new Error("timed out waiting for the mock Responses WebSocket"));
|
||||
}, 5_000);
|
||||
const cleanup = () => {
|
||||
clearTimeout(timeout);
|
||||
socket.off("message", onMessage);
|
||||
socket.off("error", onError);
|
||||
};
|
||||
const onError = (error: Error) => {
|
||||
cleanup();
|
||||
reject(error);
|
||||
};
|
||||
const onMessage = (data: RawData) => {
|
||||
const text = Buffer.isBuffer(data)
|
||||
? data.toString("utf8")
|
||||
: Array.isArray(data)
|
||||
? Buffer.concat(data).toString("utf8")
|
||||
: Buffer.from(data).toString("utf8");
|
||||
const event = JSON.parse(text) as Record<string, unknown>;
|
||||
events.push(event);
|
||||
if (event.type === "response.completed" || event.type === "error") {
|
||||
cleanup();
|
||||
resolve(events);
|
||||
}
|
||||
};
|
||||
socket.on("message", onMessage);
|
||||
socket.on("error", onError);
|
||||
socket.send(typeof request === "string" ? request : JSON.stringify(request));
|
||||
});
|
||||
}
|
||||
|
||||
function readCompletedResponse(events: Array<Record<string, unknown>>) {
|
||||
const completion = events.find((event) => event.type === "response.completed");
|
||||
expect(completion).toBeDefined();
|
||||
return completion?.response as Record<string, unknown>;
|
||||
}
|
||||
|
||||
describe("QA mock OpenAI Responses WebSocket", () => {
|
||||
it("streams the native response.create protocol and records normal QA request evidence", async () => {
|
||||
const server = await startServer();
|
||||
const socket = await connectResponsesWebSocket(server.baseUrl);
|
||||
const prompt = "Read README.md for the source and docs discovery report.";
|
||||
|
||||
const events = await collectResponseEvents(socket, {
|
||||
type: "response.create",
|
||||
model: "gpt-5.6-sol",
|
||||
stream: true,
|
||||
tools: [{ type: "function", name: "read" }],
|
||||
input: [{ role: "user", content: [{ type: "input_text", text: prompt }] }],
|
||||
});
|
||||
|
||||
expect(events[0]).toMatchObject({ type: "response.created" });
|
||||
expect(events[0]).toMatchObject({
|
||||
sequence_number: 0,
|
||||
response: {
|
||||
object: "response",
|
||||
model: "gpt-5.6-sol",
|
||||
status: "in_progress",
|
||||
output: [],
|
||||
},
|
||||
});
|
||||
expect(events).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ type: "response.output_item.done" }),
|
||||
expect.objectContaining({ type: "response.completed" }),
|
||||
]),
|
||||
);
|
||||
const lastRequest = await fetch(`${server.baseUrl}/debug/last-request`).then((response) =>
|
||||
response.json(),
|
||||
);
|
||||
expect(lastRequest).toMatchObject({
|
||||
model: "gpt-5.6-sol",
|
||||
prompt,
|
||||
plannedToolName: "read",
|
||||
});
|
||||
});
|
||||
|
||||
it("reconstructs Codex previous-response deltas on the same reusable connection", async () => {
|
||||
const server = await startServer();
|
||||
const socket = await connectResponsesWebSocket(server.baseUrl);
|
||||
const prompt = "Read README.md for the source and docs discovery report.";
|
||||
const firstEvents = await collectResponseEvents(socket, {
|
||||
type: "response.create",
|
||||
model: "gpt-5.6-sol",
|
||||
stream: true,
|
||||
tools: [{ type: "function", name: "read" }],
|
||||
input: [{ role: "user", content: [{ type: "input_text", text: prompt }] }],
|
||||
});
|
||||
const firstResponse = readCompletedResponse(firstEvents);
|
||||
const output = firstResponse.output as Array<Record<string, unknown>>;
|
||||
const toolCall = output.find((item) => item.type === "function_call");
|
||||
expect(toolCall?.call_id).toEqual(expect.any(String));
|
||||
|
||||
const secondEvents = await collectResponseEvents(socket, {
|
||||
type: "response.create",
|
||||
model: "gpt-5.6-sol",
|
||||
stream: true,
|
||||
previous_response_id: firstResponse.id,
|
||||
input: [
|
||||
{
|
||||
type: "function_call_output",
|
||||
call_id: toolCall?.call_id,
|
||||
output: "README: source and docs evidence captured",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(firstEvents.map((event) => event.sequence_number)).toEqual(
|
||||
firstEvents.map((_, index) => index),
|
||||
);
|
||||
expect(secondEvents.map((event) => event.sequence_number)).toEqual(
|
||||
secondEvents.map((_, index) => index),
|
||||
);
|
||||
|
||||
const requests = (await fetch(`${server.baseUrl}/debug/requests`).then((response) =>
|
||||
response.json(),
|
||||
)) as Array<Record<string, unknown>>;
|
||||
expect(requests).toHaveLength(2);
|
||||
expect(requests[1]).toMatchObject({
|
||||
prompt,
|
||||
toolOutput: "README: source and docs evidence captured",
|
||||
toolOutputCallId: toolCall?.call_id,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects a response ID after a newer response replaces the connection cache", async () => {
|
||||
const server = await startServer();
|
||||
const socket = await connectResponsesWebSocket(server.baseUrl);
|
||||
const prompt = "Read README.md for the source and docs discovery report.";
|
||||
const firstEvents = await collectResponseEvents(socket, {
|
||||
type: "response.create",
|
||||
model: "gpt-5.6-sol",
|
||||
stream: true,
|
||||
store: false,
|
||||
tools: [{ type: "function", name: "read" }],
|
||||
input: [{ role: "user", content: [{ type: "input_text", text: prompt }] }],
|
||||
});
|
||||
const firstResponse = readCompletedResponse(firstEvents);
|
||||
const firstOutput = firstResponse.output as Array<Record<string, unknown>>;
|
||||
const toolCall = firstOutput.find((item) => item.type === "function_call");
|
||||
|
||||
const secondEvents = await collectResponseEvents(socket, {
|
||||
type: "response.create",
|
||||
previous_response_id: firstResponse.id,
|
||||
input: [
|
||||
{
|
||||
type: "function_call_output",
|
||||
call_id: toolCall?.call_id,
|
||||
output: "README: source and docs evidence captured",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(readCompletedResponse(secondEvents).id).not.toBe(firstResponse.id);
|
||||
await expect(
|
||||
collectResponseEvents(socket, {
|
||||
type: "response.create",
|
||||
previous_response_id: firstResponse.id,
|
||||
input: [],
|
||||
}),
|
||||
).resolves.toEqual([
|
||||
expect.objectContaining({
|
||||
type: "error",
|
||||
sequence_number: 0,
|
||||
status: 400,
|
||||
error: expect.objectContaining({ code: "previous_response_not_found" }),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("answers connection warmups without inventing a model or tool request", async () => {
|
||||
const server = await startServer();
|
||||
const socket = await connectResponsesWebSocket(server.baseUrl);
|
||||
|
||||
const events = await collectResponseEvents(socket, {
|
||||
type: "response.create",
|
||||
model: "gpt-5.6-sol",
|
||||
stream: true,
|
||||
generate: false,
|
||||
input: [],
|
||||
});
|
||||
|
||||
expect(events).toEqual([
|
||||
expect.objectContaining({ type: "response.created" }),
|
||||
expect.objectContaining({
|
||||
type: "response.completed",
|
||||
response: expect.objectContaining({
|
||||
object: "response",
|
||||
model: "gpt-5.6-sol",
|
||||
status: "completed",
|
||||
output: [],
|
||||
usage: expect.objectContaining({
|
||||
input_tokens_details: { cached_tokens: 0 },
|
||||
output_tokens_details: { reasoning_tokens: 0 },
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
const cursor = await fetch(`${server.baseUrl}/debug/request-cursor`).then((response) =>
|
||||
response.json(),
|
||||
);
|
||||
expect(cursor).toEqual({ cursor: 0 });
|
||||
});
|
||||
|
||||
it("preserves the full warmup prompt when the first real response sends an empty delta", async () => {
|
||||
const server = await startServer();
|
||||
const socket = await connectResponsesWebSocket(server.baseUrl);
|
||||
const prompt = "Read README.md for the source and docs discovery report.";
|
||||
|
||||
const warmup = await collectResponseEvents(socket, {
|
||||
type: "response.create",
|
||||
model: "gpt-5.6-sol",
|
||||
stream: true,
|
||||
generate: false,
|
||||
tools: [{ type: "function", name: "read" }],
|
||||
input: [{ role: "user", content: [{ type: "input_text", text: prompt }] }],
|
||||
});
|
||||
|
||||
const events = await collectResponseEvents(socket, {
|
||||
type: "response.create",
|
||||
stream: true,
|
||||
previous_response_id: readCompletedResponse(warmup).id,
|
||||
input: [],
|
||||
});
|
||||
|
||||
expect(events).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ type: "response.output_item.done" }),
|
||||
expect.objectContaining({ type: "response.completed" }),
|
||||
]),
|
||||
);
|
||||
const lastRequest = await fetch(`${server.baseUrl}/debug/last-request`).then((response) =>
|
||||
response.json(),
|
||||
);
|
||||
expect(lastRequest).toMatchObject({ model: "gpt-5.6-sol", prompt, plannedToolName: "read" });
|
||||
const cursor = await fetch(`${server.baseUrl}/debug/request-cursor`).then((response) =>
|
||||
response.json(),
|
||||
);
|
||||
expect(cursor).toEqual({ cursor: 1 });
|
||||
});
|
||||
|
||||
it("preserves the injected post-tool 503 failure on the WebSocket transport", async () => {
|
||||
const server = await startServer();
|
||||
const socket = await connectResponsesWebSocket(server.baseUrl);
|
||||
const prompt = "Provider HTTP 503 after tool QA check: read QA_KICKOFF_TASK.md, then reply.";
|
||||
|
||||
const firstEvents = await collectResponseEvents(socket, {
|
||||
type: "response.create",
|
||||
model: "gpt-5.6-sol",
|
||||
stream: true,
|
||||
tools: [{ type: "function", name: "read" }],
|
||||
input: [{ role: "user", content: [{ type: "input_text", text: prompt }] }],
|
||||
});
|
||||
const firstResponse = readCompletedResponse(firstEvents);
|
||||
const output = firstResponse.output as Array<Record<string, unknown>>;
|
||||
const toolCall = output.find((item) => item.type === "function_call");
|
||||
|
||||
const failure = await collectResponseEvents(socket, {
|
||||
type: "response.create",
|
||||
model: "gpt-5.6-sol",
|
||||
stream: true,
|
||||
previous_response_id: firstResponse.id,
|
||||
input: [
|
||||
{
|
||||
type: "function_call_output",
|
||||
call_id: toolCall?.call_id,
|
||||
output: "QA mission loaded",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(failure).toEqual([
|
||||
expect.objectContaining({
|
||||
type: "error",
|
||||
status: 503,
|
||||
error: { type: "server_error", message: "Service Unavailable" },
|
||||
}),
|
||||
]);
|
||||
await expect(
|
||||
collectResponseEvents(socket, {
|
||||
type: "response.create",
|
||||
previous_response_id: firstResponse.id,
|
||||
input: [],
|
||||
}),
|
||||
).resolves.toEqual([
|
||||
expect.objectContaining({
|
||||
type: "error",
|
||||
sequence_number: 0,
|
||||
status: 400,
|
||||
error: expect.objectContaining({ code: "previous_response_not_found" }),
|
||||
}),
|
||||
]);
|
||||
const requests = (await fetch(`${server.baseUrl}/debug/requests`).then((response) =>
|
||||
response.json(),
|
||||
)) as Array<Record<string, unknown>>;
|
||||
expect(requests).toHaveLength(2);
|
||||
expect(requests[1]).toMatchObject({ prompt, toolOutput: "QA mission loaded" });
|
||||
});
|
||||
|
||||
it("rejects a response delta whose previous response belongs to another connection", async () => {
|
||||
const server = await startServer();
|
||||
const socket = await connectResponsesWebSocket(server.baseUrl);
|
||||
|
||||
const events = await collectResponseEvents(socket, {
|
||||
type: "response.create",
|
||||
model: "gpt-5.6-sol",
|
||||
previous_response_id: "resp_qa_unknown",
|
||||
input: [],
|
||||
});
|
||||
|
||||
expect(events).toEqual([
|
||||
expect.objectContaining({
|
||||
type: "error",
|
||||
status: 400,
|
||||
error: {
|
||||
type: "invalid_request_error",
|
||||
code: "previous_response_not_found",
|
||||
message: "The previous response was not found on this connection.",
|
||||
},
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("returns a bounded protocol error for malformed WebSocket requests", async () => {
|
||||
const server = await startServer();
|
||||
const socket = await connectResponsesWebSocket(server.baseUrl);
|
||||
|
||||
const events = await collectResponseEvents(socket, "{invalid-json");
|
||||
|
||||
expect(events).toEqual([
|
||||
expect.objectContaining({
|
||||
type: "error",
|
||||
status: 400,
|
||||
error: {
|
||||
type: "invalid_request_error",
|
||||
message: "Expected a JSON response.create request.",
|
||||
},
|
||||
}),
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,261 @@
|
||||
import type { Server } from "node:http";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { WebSocket, WebSocketServer, type RawData } from "ws";
|
||||
import type { ResponsesInputItem, StreamEvent } from "./mock-openai-contracts.js";
|
||||
|
||||
export type QaMockResponsesDispatchResult = {
|
||||
events: StreamEvent[];
|
||||
failure?: {
|
||||
status: number;
|
||||
type: string;
|
||||
message: string;
|
||||
};
|
||||
previewPauseMs?: number;
|
||||
};
|
||||
|
||||
type QaMockResponsesWebSocketDispatch = (params: {
|
||||
body: Record<string, unknown>;
|
||||
raw: string;
|
||||
}) => Promise<QaMockResponsesDispatchResult>;
|
||||
|
||||
type QaMockResponsesWebSocketHistory = {
|
||||
id: string;
|
||||
body: Record<string, unknown>;
|
||||
input: ResponsesInputItem[];
|
||||
};
|
||||
|
||||
function readWebSocketText(data: RawData): string {
|
||||
if (typeof data === "string") {
|
||||
return data;
|
||||
}
|
||||
if (Array.isArray(data)) {
|
||||
return Buffer.concat(data).toString("utf8");
|
||||
}
|
||||
if (data instanceof ArrayBuffer) {
|
||||
return Buffer.from(data).toString("utf8");
|
||||
}
|
||||
return data.toString("utf8");
|
||||
}
|
||||
|
||||
function readWebSocketRequest(raw: string): Record<string, unknown> | undefined {
|
||||
try {
|
||||
const value: unknown = JSON.parse(raw);
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function sendWebSocketEvent(socket: WebSocket, event: unknown): void {
|
||||
if (socket.readyState === WebSocket.OPEN) {
|
||||
socket.send(JSON.stringify(event));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Give Codex the same Responses-over-WebSocket transport its built-in OpenAI
|
||||
* provider uses in production. An SSE-only mock makes each native turn pay
|
||||
* websocket reconnect/fallback retries before the real scenario even starts.
|
||||
*/
|
||||
export function attachQaMockResponsesWebSocketServer(params: {
|
||||
server: Server;
|
||||
dispatch: QaMockResponsesWebSocketDispatch;
|
||||
}) {
|
||||
const webSocketServer = new WebSocketServer({
|
||||
noServer: true,
|
||||
maxPayload: 4 * 1024 * 1024,
|
||||
});
|
||||
|
||||
params.server.on("upgrade", (request, socket, head) => {
|
||||
const url = new URL(request.url ?? "/", "http://127.0.0.1");
|
||||
if (url.pathname !== "/v1/responses") {
|
||||
socket.write("HTTP/1.1 404 Not Found\r\nConnection: close\r\n\r\n");
|
||||
socket.destroy();
|
||||
return;
|
||||
}
|
||||
|
||||
webSocketServer.handleUpgrade(request, socket, head, (connection) => {
|
||||
webSocketServer.emit("connection", connection, request);
|
||||
});
|
||||
});
|
||||
|
||||
webSocketServer.on("connection", (socket: WebSocket) => {
|
||||
let cachedResponse: QaMockResponsesWebSocketHistory | undefined;
|
||||
let warmupOrdinal = 0;
|
||||
let sequenceNumber = 0;
|
||||
let pending = Promise.resolve();
|
||||
const sendEvent = (event: Record<string, unknown>) => {
|
||||
sendWebSocketEvent(socket, { ...event, sequence_number: sequenceNumber++ });
|
||||
};
|
||||
|
||||
socket.on("error", () => {
|
||||
// A disconnected client must not crash or outlive the private mock.
|
||||
});
|
||||
|
||||
socket.on("message", (data: RawData, isBinary: boolean) => {
|
||||
pending = pending
|
||||
.then(async () => {
|
||||
// Responses stream sequence numbers describe one response, not
|
||||
// all responses that happen to reuse the same WebSocket.
|
||||
sequenceNumber = 0;
|
||||
const raw = readWebSocketText(data);
|
||||
const request = isBinary ? undefined : readWebSocketRequest(raw);
|
||||
if (!request || request.type !== "response.create") {
|
||||
sendEvent({
|
||||
type: "error",
|
||||
status: 400,
|
||||
error: {
|
||||
type: "invalid_request_error",
|
||||
message: "Expected a JSON response.create request.",
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const previousResponseId =
|
||||
typeof request.previous_response_id === "string"
|
||||
? request.previous_response_id
|
||||
: undefined;
|
||||
if (previousResponseId && cachedResponse?.id !== previousResponseId) {
|
||||
sendEvent({
|
||||
type: "error",
|
||||
status: 400,
|
||||
error: {
|
||||
type: "invalid_request_error",
|
||||
code: "previous_response_not_found",
|
||||
message: "The previous response was not found on this connection.",
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
const previousResponse = previousResponseId ? cachedResponse : undefined;
|
||||
const previousInput = previousResponse?.input ?? [];
|
||||
const nextInput = Array.isArray(request.input)
|
||||
? (request.input as ResponsesInputItem[])
|
||||
: [];
|
||||
const inheritedRequest = previousResponse ? { ...previousResponse.body } : {};
|
||||
// These control fields belong to one frame; model, instructions,
|
||||
// tool definitions, and other prompt context carry across deltas.
|
||||
delete inheritedRequest.type;
|
||||
delete inheritedRequest.previous_response_id;
|
||||
delete inheritedRequest.generate;
|
||||
delete inheritedRequest.input;
|
||||
const body: Record<string, unknown> & { input: ResponsesInputItem[] } = {
|
||||
...inheritedRequest,
|
||||
...request,
|
||||
input: [...previousInput, ...nextInput],
|
||||
};
|
||||
|
||||
if (request.generate === false) {
|
||||
const id = `resp_qa_ws_warmup_${++warmupOrdinal}`;
|
||||
const createdAt = Math.floor(Date.now() / 1_000);
|
||||
const model = typeof body.model === "string" ? body.model : "";
|
||||
sendEvent({
|
||||
type: "response.created",
|
||||
response: {
|
||||
id,
|
||||
object: "response",
|
||||
created_at: createdAt,
|
||||
model,
|
||||
status: "in_progress",
|
||||
output: [],
|
||||
},
|
||||
});
|
||||
sendEvent({
|
||||
type: "response.completed",
|
||||
response: {
|
||||
id,
|
||||
object: "response",
|
||||
created_at: createdAt,
|
||||
model,
|
||||
status: "completed",
|
||||
output: [],
|
||||
usage: {
|
||||
input_tokens: 0,
|
||||
input_tokens_details: { cached_tokens: 0 },
|
||||
output_tokens: 0,
|
||||
output_tokens_details: { reasoning_tokens: 0 },
|
||||
total_tokens: 0,
|
||||
},
|
||||
},
|
||||
});
|
||||
// Codex's real first request often sends no input at all: its
|
||||
// previous_response_id points at this prompt-bearing warmup.
|
||||
cachedResponse = { id, body, input: body.input };
|
||||
return;
|
||||
}
|
||||
|
||||
// Codex sends only the delta after a cached response; scenario and
|
||||
// debug matching need the complete prompt and inherited tool surface.
|
||||
const dispatched = await params.dispatch({ body, raw: JSON.stringify(body) });
|
||||
if (dispatched.failure) {
|
||||
// A failed continuation invalidates the connection's ephemeral
|
||||
// store=false response; Codex must retry with a full new request.
|
||||
cachedResponse = undefined;
|
||||
sendEvent({
|
||||
type: "error",
|
||||
status: dispatched.failure.status,
|
||||
error: {
|
||||
type: dispatched.failure.type,
|
||||
message: dispatched.failure.message,
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
const { events } = dispatched;
|
||||
const completion = events.find((event) => event.type === "response.completed");
|
||||
if (completion?.type === "response.completed") {
|
||||
if (!events.some((event) => event.type === "response.created")) {
|
||||
sendEvent({
|
||||
type: "response.created",
|
||||
response: {
|
||||
id: completion.response.id,
|
||||
object: "response",
|
||||
created_at: Math.floor(Date.now() / 1_000),
|
||||
model: typeof body.model === "string" ? body.model : "",
|
||||
status: "in_progress",
|
||||
output: [],
|
||||
},
|
||||
});
|
||||
}
|
||||
cachedResponse = {
|
||||
id: completion.response.id,
|
||||
body,
|
||||
input: [...body.input, ...completion.response.output],
|
||||
};
|
||||
}
|
||||
for (const event of events) {
|
||||
if (dispatched.previewPauseMs && event.type === "response.output_text.done") {
|
||||
await sleep(dispatched.previewPauseMs);
|
||||
}
|
||||
sendEvent(event);
|
||||
}
|
||||
})
|
||||
.catch(() => {
|
||||
cachedResponse = undefined;
|
||||
sequenceNumber = 0;
|
||||
sendEvent({
|
||||
type: "error",
|
||||
status: 500,
|
||||
error: {
|
||||
type: "server_error",
|
||||
message: "Mock Responses WebSocket dispatch failed.",
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
return {
|
||||
async close(): Promise<void> {
|
||||
for (const socket of webSocketServer.clients) {
|
||||
socket.terminate();
|
||||
}
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
webSocketServer.close((error) => (error ? reject(error) : resolve()));
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -125,6 +125,45 @@ export function buildToolCallEventsWithArgs(
|
||||
];
|
||||
}
|
||||
|
||||
export function buildCustomToolCallEventsWithInput(name: string, input: string): StreamEvent[] {
|
||||
const call = buildMockFunctionCall(name, { input });
|
||||
const itemId = call.itemId.replace(/^fc_/, "ctc_");
|
||||
const item = {
|
||||
type: "custom_tool_call",
|
||||
id: itemId,
|
||||
call_id: call.callId,
|
||||
name,
|
||||
input,
|
||||
status: "completed",
|
||||
};
|
||||
return [
|
||||
{
|
||||
type: "response.created",
|
||||
response: { id: call.responseId },
|
||||
},
|
||||
{
|
||||
type: "response.output_item.added",
|
||||
item: { ...item, input: "", status: "in_progress" },
|
||||
},
|
||||
{
|
||||
type: "response.custom_tool_call_input.delta",
|
||||
item_id: itemId,
|
||||
call_id: call.callId,
|
||||
delta: input,
|
||||
},
|
||||
{ type: "response.output_item.done", item },
|
||||
{
|
||||
type: "response.completed",
|
||||
response: {
|
||||
id: call.responseId,
|
||||
status: "completed",
|
||||
output: [item],
|
||||
usage: { input_tokens: 64, output_tokens: 16, total_tokens: 80 },
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
export function extractRememberedFact(userTexts: string[]) {
|
||||
for (const text of userTexts) {
|
||||
const qaCanaryMatch = /\bqa canary code is\s+([A-Za-z0-9-]+)/i.exec(text);
|
||||
@@ -171,6 +210,19 @@ export function buildQaToolSearchArgs(
|
||||
if (failureMode && targetTool === "web_search") {
|
||||
return { query: QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY };
|
||||
}
|
||||
if (failureMode && targetTool === "apply_patch") {
|
||||
return {
|
||||
input: [
|
||||
"*** Begin Patch",
|
||||
"*** Update File: ../runtime-tool-fixture-denied.txt",
|
||||
"@@",
|
||||
"-runtime-tool-fixture-denied-original",
|
||||
"+runtime patch outside the workspace",
|
||||
"*** End Patch",
|
||||
"",
|
||||
].join("\n"),
|
||||
};
|
||||
}
|
||||
if (failureMode) {
|
||||
return { __qaFailureMode: "denied-input" };
|
||||
}
|
||||
|
||||
@@ -3971,6 +3971,227 @@ describe("qa mock openai server", () => {
|
||||
expect(String(toolPlanOutput.arguments)).toContain("OPENCLAW_QA_WEB_SEARCH_DENIED_INPUT");
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "workspace-local happy",
|
||||
prompt:
|
||||
"tool search qa check target=apply_patch. Call apply_patch exactly once and then summarize.",
|
||||
operation: "Add File",
|
||||
patchPath: "runtime-tool-fixture-patch.txt",
|
||||
},
|
||||
{
|
||||
label: "workspace-escaping failure",
|
||||
prompt:
|
||||
"tool search qa failure target=apply_patch. Exercise the denied-input path once and then summarize.",
|
||||
operation: "Update File",
|
||||
patchPath: "../runtime-tool-fixture-denied.txt",
|
||||
},
|
||||
])("plans a valid $label apply_patch envelope", async ({ prompt, operation, patchPath }) => {
|
||||
const server = await startMockServer();
|
||||
const response = await postResponses(server, {
|
||||
stream: false,
|
||||
input: [makeUserInput(prompt)],
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
const payload = await response.json();
|
||||
expect(outputItem(payload)).toMatchObject({ type: "function_call", name: "apply_patch" });
|
||||
const args = outputToolArgs(payload);
|
||||
expect(args).not.toHaveProperty("__qaFailureMode");
|
||||
expect(args.input).toBeTypeOf("string");
|
||||
expect(args.input).toContain("*** Begin Patch\n");
|
||||
expect(args.input).toContain(`*** ${operation}: ${patchPath}\n`);
|
||||
if (operation === "Update File") {
|
||||
expect(args.input).toContain("\n@@\n-runtime-tool-fixture-denied-original\n");
|
||||
}
|
||||
expect(args.input).toContain("\n*** End Patch\n");
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "workspace-local happy",
|
||||
prompt:
|
||||
"tool search qa check target=apply_patch. Call apply_patch exactly once and then summarize.",
|
||||
operation: "Add File",
|
||||
patchPath: "runtime-tool-fixture-patch.txt",
|
||||
},
|
||||
{
|
||||
label: "workspace-escaping failure",
|
||||
prompt:
|
||||
"tool search qa failure target=apply_patch. Exercise the denied-input path once and then summarize.",
|
||||
operation: "Update File",
|
||||
patchPath: "../runtime-tool-fixture-denied.txt",
|
||||
},
|
||||
])("plans an actual $label native freeform patch", async (testCase) => {
|
||||
const server = await startMockServer();
|
||||
const response = await postResponses(server, {
|
||||
stream: false,
|
||||
tools: [
|
||||
{
|
||||
type: "custom",
|
||||
name: "apply_patch",
|
||||
format: { type: "grammar", syntax: "lark", definition: "start: /.+/" },
|
||||
},
|
||||
],
|
||||
input: [makeUserInput(testCase.prompt)],
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
const item = outputItem(await response.json());
|
||||
expect(item).toMatchObject({ type: "custom_tool_call", name: "apply_patch" });
|
||||
expect(item).not.toHaveProperty("arguments");
|
||||
expect(item.input).toEqual(
|
||||
expect.stringContaining(`*** ${testCase.operation}: ${testCase.patchPath}\n`),
|
||||
);
|
||||
if (testCase.operation === "Update File") {
|
||||
expect(item.input).toEqual(
|
||||
expect.stringContaining("\n@@\n-runtime-tool-fixture-denied-original\n"),
|
||||
);
|
||||
}
|
||||
|
||||
const debugResponse = await fetch(`${server.baseUrl}/debug/last-request`);
|
||||
expect(debugResponse.status).toBe(200);
|
||||
const debug = requireRecord(await debugResponse.json(), "native patch plan debug request");
|
||||
expect(debug.plannedToolName).toBe("apply_patch");
|
||||
expect(debug.plannedToolCallId).toBe(item.call_id);
|
||||
expect(debug.plannedToolArgs).toEqual({ input: item.input });
|
||||
});
|
||||
|
||||
it("streams native Codex patch input as custom-tool SSE", async () => {
|
||||
const server = await startMockServer();
|
||||
const response = await postResponses(server, {
|
||||
stream: true,
|
||||
tools: [
|
||||
{
|
||||
type: "custom",
|
||||
name: "apply_patch",
|
||||
format: { type: "grammar", syntax: "lark", definition: "start: /.+/" },
|
||||
},
|
||||
],
|
||||
input: [
|
||||
makeUserInput(
|
||||
"tool search qa check target=apply_patch. Call apply_patch exactly once and then summarize.",
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
const body = await response.text();
|
||||
const events = body
|
||||
.split("\n")
|
||||
.filter((line) => line.startsWith("data: {"))
|
||||
.map(
|
||||
(line) =>
|
||||
JSON.parse(line.slice("data: ".length)) as {
|
||||
type: string;
|
||||
response?: { id?: string; output?: Array<Record<string, unknown>> };
|
||||
item?: Record<string, unknown>;
|
||||
item_id?: string;
|
||||
call_id?: string;
|
||||
delta?: string;
|
||||
},
|
||||
);
|
||||
expect(events.map((event) => event.type)).toEqual([
|
||||
"response.created",
|
||||
"response.output_item.added",
|
||||
"response.custom_tool_call_input.delta",
|
||||
"response.output_item.done",
|
||||
"response.completed",
|
||||
]);
|
||||
const [created, added, delta, done, completed] = events;
|
||||
expect(created?.response?.id).toBe(completed?.response?.id);
|
||||
expect(added?.item).toMatchObject({
|
||||
type: "custom_tool_call",
|
||||
name: "apply_patch",
|
||||
input: "",
|
||||
status: "in_progress",
|
||||
});
|
||||
expect(done?.item).toMatchObject({
|
||||
type: "custom_tool_call",
|
||||
name: "apply_patch",
|
||||
status: "completed",
|
||||
});
|
||||
expect(done?.item?.id).toEqual(expect.stringMatching(/^ctc_mock_apply_patch_/));
|
||||
expect(delta?.item_id).toBe(done?.item?.id);
|
||||
expect(delta?.call_id).toBe(done?.item?.call_id);
|
||||
expect(delta?.delta).toBe(done?.item?.input);
|
||||
expect(delta?.delta).toContain("runtime-tool-fixture-patch.txt");
|
||||
expect(completed?.response?.output).toEqual([done?.item]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "successful native patch",
|
||||
prompt:
|
||||
"tool search qa check target=apply_patch. Call apply_patch exactly once and then summarize.",
|
||||
output: "Successfully applied patch",
|
||||
expectedOutput: "Successfully applied patch",
|
||||
structuredError: false,
|
||||
},
|
||||
{
|
||||
label: "denied native patch",
|
||||
prompt:
|
||||
"tool search qa failure target=apply_patch. Exercise the denied-input path once and then summarize.",
|
||||
output: "Error: Path escapes sandbox root",
|
||||
expectedOutput: "Error: Path escapes sandbox root",
|
||||
structuredError: true,
|
||||
},
|
||||
{
|
||||
label: "upstream Codex native patch rejection without a wire error flag",
|
||||
prompt:
|
||||
"tool search qa failure target=apply_patch. Exercise the denied-input path once and then summarize.",
|
||||
output: "patch rejected: writing outside of the project; rejected by user approval settings",
|
||||
expectedOutput:
|
||||
"patch rejected: writing outside of the project; rejected by user approval settings",
|
||||
structuredError: false,
|
||||
},
|
||||
{
|
||||
label: "structured native patch output",
|
||||
prompt:
|
||||
"tool search qa check target=apply_patch. Call apply_patch exactly once and then summarize.",
|
||||
output: [{ type: "input_text", text: "Successfully applied structured patch" }],
|
||||
expectedOutput: "Successfully applied structured patch",
|
||||
structuredError: false,
|
||||
},
|
||||
])("links and completes $label custom-tool outputs", async (testCase) => {
|
||||
const server = await startMockServer();
|
||||
const planResponse = await postResponses(server, {
|
||||
stream: false,
|
||||
input: [makeUserInput(testCase.prompt)],
|
||||
});
|
||||
expect(planResponse.status).toBe(200);
|
||||
const plannedCall = outputItem(await planResponse.json());
|
||||
expect(plannedCall).toMatchObject({ type: "function_call", name: "apply_patch" });
|
||||
const callId = outputToolCallId(plannedCall, "native-patch-call");
|
||||
|
||||
const continuationResponse = await postResponses(server, {
|
||||
stream: false,
|
||||
input: [
|
||||
makeUserInput(testCase.prompt),
|
||||
{
|
||||
type: "custom_tool_call_output",
|
||||
call_id: callId,
|
||||
output: testCase.output,
|
||||
...(testCase.structuredError ? { is_error: true } : {}),
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(continuationResponse.status).toBe(200);
|
||||
expect(outputItem(await continuationResponse.json()).type).toBe("message");
|
||||
|
||||
const debugResponse = await fetch(`${server.baseUrl}/debug/last-request`);
|
||||
expect(debugResponse.status).toBe(200);
|
||||
const debug = requireRecord(await debugResponse.json(), "custom patch debug request");
|
||||
expect(debug.toolOutput).toBe(testCase.expectedOutput);
|
||||
expect(debug.toolOutputCallId).toBe(callId);
|
||||
if (testCase.structuredError) {
|
||||
expect(debug.toolOutputStructuredError).toBe(true);
|
||||
} else {
|
||||
expect(debug).not.toHaveProperty("toolOutputStructuredError");
|
||||
}
|
||||
expect(debug).not.toHaveProperty("plannedToolName");
|
||||
});
|
||||
|
||||
it("plans QA subagent handoff calls even when Codex dynamic tools are not in body.tools", async () => {
|
||||
const server = await startMockServer();
|
||||
|
||||
@@ -4411,6 +4632,49 @@ describe("qa mock openai server", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("advertises directly executable native Codex metadata alongside OpenAI models", async () => {
|
||||
const server = await startMockServer({
|
||||
modelRefs: ["mock-openai/gpt-5.6-luna", "mock-openai/gpt-5.6-luna-alt"],
|
||||
});
|
||||
|
||||
const response = await fetch(`${server.baseUrl}/v1/models?client_version=0.142.0`);
|
||||
expect(response.status).toBe(200);
|
||||
const body = (await response.json()) as {
|
||||
data: Array<{ id: string; object: string }>;
|
||||
models: Array<Record<string, unknown>>;
|
||||
};
|
||||
expect(body.data).toEqual(
|
||||
expect.arrayContaining([
|
||||
{ id: "gpt-5.6-luna", object: "model" },
|
||||
{ id: "gpt-5.6-luna-alt", object: "model" },
|
||||
]),
|
||||
);
|
||||
expect(body.models).toHaveLength(2);
|
||||
expect(body.models).toEqual([
|
||||
expect.objectContaining({
|
||||
slug: "gpt-5.6-luna",
|
||||
display_name: "gpt-5.6-luna",
|
||||
apply_patch_tool_type: "freeform",
|
||||
tool_mode: "direct",
|
||||
shell_type: "shell_command",
|
||||
visibility: "list",
|
||||
supported_in_api: true,
|
||||
base_instructions: expect.any(String),
|
||||
truncation_policy: { mode: "tokens", limit: 10_000 },
|
||||
supported_reasoning_levels: expect.arrayContaining([
|
||||
{ effort: "medium", description: "Balanced QA reasoning" },
|
||||
]),
|
||||
experimental_supported_tools: [],
|
||||
input_modalities: ["text", "image"],
|
||||
}),
|
||||
expect.objectContaining({
|
||||
slug: "gpt-5.6-luna-alt",
|
||||
apply_patch_tool_type: "freeform",
|
||||
tool_mode: "direct",
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("serves deterministic OpenAI-compatible audio transcription responses", async () => {
|
||||
const server = await startMockServer();
|
||||
|
||||
|
||||
@@ -4,7 +4,10 @@ import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { closeQaHttpServer } from "../../bus-server.js";
|
||||
import { parseQaDebugRequestCursor } from "../shared/debug-request-cursor.js";
|
||||
import { writeJson } from "../shared/http-json.js";
|
||||
import { listMockOpenAiServerModelIds } from "../shared/mock-model-config.js";
|
||||
import {
|
||||
listMockCodexModelInfos,
|
||||
listMockOpenAiServerModelIds,
|
||||
} from "../shared/mock-model-config.js";
|
||||
import { buildMessagesPayload } from "./mock-anthropic-messages.js";
|
||||
import { buildAssistantText } from "./mock-openai-assistant-text.js";
|
||||
import {
|
||||
@@ -91,6 +94,7 @@ import {
|
||||
shouldUseWhatsAppContactMarker,
|
||||
shouldUseWhatsAppStickerMarker,
|
||||
extractBlockStreamingMarkerDirectives,
|
||||
hasDeclaredCustomTool,
|
||||
hasDeclaredTool,
|
||||
hasToolDefinition,
|
||||
isQaToolSearchFixture,
|
||||
@@ -135,9 +139,14 @@ import {
|
||||
extractLatestImageUserTurn,
|
||||
parseToolOutputJson,
|
||||
} from "./mock-openai-input.js";
|
||||
import {
|
||||
attachQaMockResponsesWebSocketServer,
|
||||
type QaMockResponsesDispatchResult,
|
||||
} from "./mock-openai-responses-websocket.js";
|
||||
import {
|
||||
readTargetFromPrompt,
|
||||
execCommandFromToolProgressPrompt,
|
||||
buildCustomToolCallEventsWithInput,
|
||||
buildToolCallEventsWithArgs,
|
||||
extractOrbitCode,
|
||||
extractToolSearchTarget,
|
||||
@@ -247,6 +256,13 @@ async function buildResponsesPayload(
|
||||
const plannedArgs = targetTool
|
||||
? buildQaToolSearchArgs(targetTool, QA_TOOL_SEARCH_FAILURE_PROMPT_RE.test(allInputText))
|
||||
: {};
|
||||
if (
|
||||
targetTool === "apply_patch" &&
|
||||
hasDeclaredCustomTool(body, targetTool) &&
|
||||
typeof plannedArgs.input === "string"
|
||||
) {
|
||||
return buildCustomToolCallEventsWithInput(targetTool, plannedArgs.input);
|
||||
}
|
||||
if (targetTool && hasDeclaredTool(body, "tool_search_code")) {
|
||||
return buildToolCallEventsWithArgs("tool_search_code", {
|
||||
code: [
|
||||
@@ -1299,6 +1315,59 @@ export async function startQaMockOpenAiServer(params?: {
|
||||
const inflightRequests = new Map<number, { prompt: string; allInputText: string }>();
|
||||
let nextInflightRequestId = 1;
|
||||
const imageGenerationRequests: Array<Record<string, unknown>> = [];
|
||||
const dispatchResponses = async (request: {
|
||||
body: Record<string, unknown>;
|
||||
raw: string;
|
||||
}): Promise<QaMockResponsesDispatchResult> => {
|
||||
const input = Array.isArray(request.body.input)
|
||||
? (request.body.input as ResponsesInputItem[])
|
||||
: [];
|
||||
if (isRemoteCompactionV2Request(input)) {
|
||||
return { events: buildRemoteCompactionV2Events() };
|
||||
}
|
||||
const prompt = extractLastUserText(input);
|
||||
const allInputText = extractAllRequestTexts(input, request.body);
|
||||
const inflightRequestId = nextInflightRequestId++;
|
||||
inflightRequests.set(inflightRequestId, { prompt, allInputText });
|
||||
let events: StreamEvent[];
|
||||
try {
|
||||
events = await buildResponsesPayload(request.body, scenarioState);
|
||||
} finally {
|
||||
inflightRequests.delete(inflightRequestId);
|
||||
}
|
||||
const resolvedModel = typeof request.body.model === "string" ? request.body.model : "";
|
||||
recordRequest({
|
||||
raw: request.raw,
|
||||
body: request.body,
|
||||
prompt,
|
||||
allInputText,
|
||||
instructions: extractInstructionsText(request.body) || undefined,
|
||||
toolOutput: extractToolOutput(input),
|
||||
model: resolvedModel,
|
||||
providerVariant: resolveProviderVariant(resolvedModel),
|
||||
imageInputCount: countImageInputs(input),
|
||||
plannedToolCallId: extractPlannedToolCallId(events),
|
||||
plannedToolName: extractPlannedToolName(events),
|
||||
plannedToolArgs: extractPlannedToolArgs(events),
|
||||
toolOutputCallId: extractToolOutputCallId(input) || undefined,
|
||||
...(extractToolOutputStructuredError(input) ? { toolOutputStructuredError: true } : {}),
|
||||
});
|
||||
return {
|
||||
events,
|
||||
...(QA_PROVIDER_HTTP_503_AFTER_TOOL_PROMPT_RE.test(allInputText) && extractToolOutput(input)
|
||||
? {
|
||||
failure: {
|
||||
status: 503,
|
||||
type: "server_error",
|
||||
message: "Service Unavailable",
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
...(QA_FINAL_ONLY_MARKER_STREAMING_PROMPT_RE.test(allInputText)
|
||||
? { previewPauseMs: finalOnlyMarkerPauseMs }
|
||||
: {}),
|
||||
};
|
||||
};
|
||||
const server = createServer((req, res) => {
|
||||
void (async () => {
|
||||
const url = new URL(req.url ?? "/", "http://127.0.0.1");
|
||||
@@ -1312,6 +1381,7 @@ export async function startQaMockOpenAiServer(params?: {
|
||||
id,
|
||||
object: "model",
|
||||
})),
|
||||
models: listMockCodexModelInfos(params?.modelRefs),
|
||||
});
|
||||
return;
|
||||
}
|
||||
@@ -1439,45 +1509,17 @@ export async function startQaMockOpenAiServer(params?: {
|
||||
}
|
||||
return;
|
||||
}
|
||||
const prompt = extractLastUserText(input);
|
||||
const allInputText = extractAllRequestTexts(input, body);
|
||||
const inflightRequestId = nextInflightRequestId++;
|
||||
inflightRequests.set(inflightRequestId, { prompt, allInputText });
|
||||
let events: StreamEvent[];
|
||||
try {
|
||||
events = await buildResponsesPayload(body, scenarioState);
|
||||
} finally {
|
||||
inflightRequests.delete(inflightRequestId);
|
||||
}
|
||||
const resolvedModel = typeof body.model === "string" ? body.model : "";
|
||||
recordRequest({
|
||||
raw,
|
||||
body,
|
||||
prompt,
|
||||
allInputText,
|
||||
instructions: extractInstructionsText(body) || undefined,
|
||||
toolOutput: extractToolOutput(input),
|
||||
model: resolvedModel,
|
||||
providerVariant: resolveProviderVariant(resolvedModel),
|
||||
imageInputCount: countImageInputs(input),
|
||||
plannedToolCallId: extractPlannedToolCallId(events),
|
||||
plannedToolName: extractPlannedToolName(events),
|
||||
plannedToolArgs: extractPlannedToolArgs(events),
|
||||
toolOutputCallId: extractToolOutputCallId(input) || undefined,
|
||||
...(extractToolOutputStructuredError(input) ? { toolOutputStructuredError: true } : {}),
|
||||
});
|
||||
if (
|
||||
QA_PROVIDER_HTTP_503_AFTER_TOOL_PROMPT_RE.test(allInputText) &&
|
||||
extractToolOutput(input)
|
||||
) {
|
||||
writeJson(res, 503, {
|
||||
const dispatched = await dispatchResponses({ body, raw });
|
||||
if (dispatched.failure) {
|
||||
writeJson(res, dispatched.failure.status, {
|
||||
error: {
|
||||
type: "server_error",
|
||||
message: "Service Unavailable",
|
||||
type: dispatched.failure.type,
|
||||
message: dispatched.failure.message,
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
const { events } = dispatched;
|
||||
if (body.stream === false) {
|
||||
const completion = events.at(-1);
|
||||
if (!completion || completion.type !== "response.completed") {
|
||||
@@ -1487,8 +1529,8 @@ export async function startQaMockOpenAiServer(params?: {
|
||||
writeJson(res, 200, completion.response);
|
||||
return;
|
||||
}
|
||||
if (QA_FINAL_ONLY_MARKER_STREAMING_PROMPT_RE.test(allInputText)) {
|
||||
await writeSseWithPreviewPause(res, events, finalOnlyMarkerPauseMs);
|
||||
if (dispatched.previewPauseMs !== undefined) {
|
||||
await writeSseWithPreviewPause(res, events, dispatched.previewPauseMs);
|
||||
} else {
|
||||
writeSse(res, events);
|
||||
}
|
||||
@@ -1545,6 +1587,10 @@ export async function startQaMockOpenAiServer(params?: {
|
||||
writeJson(res, 404, { error: "not found" });
|
||||
})();
|
||||
});
|
||||
const responsesWebSocket = attachQaMockResponsesWebSocketServer({
|
||||
server,
|
||||
dispatch: dispatchResponses,
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
@@ -1559,6 +1605,7 @@ export async function startQaMockOpenAiServer(params?: {
|
||||
return {
|
||||
baseUrl: `http://${host}:${address.port}`,
|
||||
async stop() {
|
||||
await responsesWebSocket.close();
|
||||
await closeQaHttpServer(server);
|
||||
},
|
||||
};
|
||||
|
||||
@@ -137,3 +137,43 @@ export function listMockOpenAiServerModelIds(selectedModelRefs: readonly string[
|
||||
"claude-sonnet-4-6",
|
||||
];
|
||||
}
|
||||
|
||||
/** Codex consumes full model metadata, while OpenAI clients consume the sibling `data` list. */
|
||||
export function listMockCodexModelInfos(selectedModelRefs: readonly string[] = []) {
|
||||
return selectedOpenAiModelIds("mock-openai", selectedModelRefs).map((slug, priority) => ({
|
||||
slug,
|
||||
display_name: slug,
|
||||
description: "QA mock OpenAI coding model",
|
||||
default_reasoning_level: "medium",
|
||||
supported_reasoning_levels: [
|
||||
{ effort: "low", description: "Fast QA reasoning" },
|
||||
{ effort: "medium", description: "Balanced QA reasoning" },
|
||||
{ effort: "high", description: "Thorough QA reasoning" },
|
||||
{ effort: "xhigh", description: "Extra-thorough QA reasoning" },
|
||||
],
|
||||
shell_type: "shell_command",
|
||||
visibility: "list",
|
||||
supported_in_api: true,
|
||||
priority,
|
||||
availability_nux: null,
|
||||
upgrade: null,
|
||||
base_instructions: "You are Codex, a coding agent based on GPT-5.",
|
||||
include_skills_usage_instructions: false,
|
||||
supports_reasoning_summaries: true,
|
||||
default_reasoning_summary: "none",
|
||||
support_verbosity: true,
|
||||
default_verbosity: "low",
|
||||
apply_patch_tool_type: "freeform",
|
||||
web_search_tool_type: "text_and_image",
|
||||
truncation_policy: { mode: "tokens", limit: 10_000 },
|
||||
supports_parallel_tool_calls: true,
|
||||
supports_image_detail_original: true,
|
||||
context_window: 128_000,
|
||||
max_context_window: 128_000,
|
||||
effective_context_window_percent: 95,
|
||||
experimental_supported_tools: [],
|
||||
input_modalities: ["text", "image"],
|
||||
supports_search_tool: true,
|
||||
tool_mode: "direct",
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
import type { RuntimeId } from "./runtime-parity.js";
|
||||
|
||||
type QaRuntimeWallClockMetrics = {
|
||||
totalWallClockMs: number | null;
|
||||
p50WallClockMs: number | null;
|
||||
p90WallClockMs: number | null;
|
||||
};
|
||||
|
||||
type QaRuntimeSpeedComparison = {
|
||||
fasterRuntime: RuntimeId | "tie" | null;
|
||||
speedupPercent: number | null;
|
||||
};
|
||||
|
||||
export type QaRuntimeTiming = QaRuntimeSpeedComparison & {
|
||||
openclaw: QaRuntimeWallClockMetrics;
|
||||
codex: QaRuntimeWallClockMetrics;
|
||||
bootstrap?: {
|
||||
openclaw: QaRuntimeWallClockMetrics;
|
||||
codex: QaRuntimeWallClockMetrics;
|
||||
};
|
||||
};
|
||||
|
||||
export type QaRuntimeParityCellTiming = {
|
||||
wallClockMs: number;
|
||||
bootstrapWallClockMs: number;
|
||||
};
|
||||
|
||||
export function measureRuntimeParityCellTiming(params: {
|
||||
suiteStartedAt: Date;
|
||||
bootstrapFinishedAt?: Date;
|
||||
scenarioStartedAt: Date;
|
||||
scenarioFinishedAt: Date;
|
||||
}): QaRuntimeParityCellTiming {
|
||||
return {
|
||||
// Gateway/provider startup is harness bootstrap, not an agent turn. Keep
|
||||
// both measurements so a faster report cannot hide cold-start regressions.
|
||||
wallClockMs: Math.max(
|
||||
1,
|
||||
params.scenarioFinishedAt.getTime() - params.scenarioStartedAt.getTime(),
|
||||
),
|
||||
bootstrapWallClockMs: Math.max(
|
||||
0,
|
||||
(params.bootstrapFinishedAt ?? params.scenarioStartedAt).getTime() -
|
||||
params.suiteStartedAt.getTime(),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export function compareRuntimeWallClockMs(
|
||||
openclawWallClockMs: number | null,
|
||||
codexWallClockMs: number | null,
|
||||
): QaRuntimeSpeedComparison {
|
||||
if (openclawWallClockMs === null || codexWallClockMs === null) {
|
||||
return { fasterRuntime: null, speedupPercent: null };
|
||||
}
|
||||
if (openclawWallClockMs === codexWallClockMs) {
|
||||
return { fasterRuntime: "tie", speedupPercent: 0 };
|
||||
}
|
||||
const fasterRuntime = openclawWallClockMs < codexWallClockMs ? "openclaw" : "codex";
|
||||
const fasterWallClockMs = Math.min(openclawWallClockMs, codexWallClockMs);
|
||||
const slowerWallClockMs = Math.max(openclawWallClockMs, codexWallClockMs);
|
||||
return {
|
||||
fasterRuntime,
|
||||
speedupPercent:
|
||||
fasterWallClockMs === 0
|
||||
? null
|
||||
: ((slowerWallClockMs - fasterWallClockMs) / fasterWallClockMs) * 100,
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeRuntimeWallClock(values: number[]): QaRuntimeWallClockMetrics {
|
||||
if (values.length === 0) {
|
||||
return { totalWallClockMs: null, p50WallClockMs: null, p90WallClockMs: null };
|
||||
}
|
||||
const sorted = values.toSorted((left, right) => left - right);
|
||||
const percentile = (value: number) =>
|
||||
sorted[Math.min(sorted.length - 1, Math.ceil((value / 100) * sorted.length) - 1)] ?? null;
|
||||
return {
|
||||
totalWallClockMs: sorted.reduce((total, value) => total + value, 0),
|
||||
p50WallClockMs: percentile(50),
|
||||
p90WallClockMs: percentile(90),
|
||||
};
|
||||
}
|
||||
|
||||
export function summarizeRuntimeParityTiming(
|
||||
scenarios: readonly {
|
||||
openclawWallClockMs: number | null;
|
||||
codexWallClockMs: number | null;
|
||||
openclawBootstrapWallClockMs?: number | null;
|
||||
codexBootstrapWallClockMs?: number | null;
|
||||
}[],
|
||||
): QaRuntimeTiming {
|
||||
const openclaw = summarizeRuntimeWallClock(
|
||||
scenarios.flatMap(({ openclawWallClockMs }) =>
|
||||
openclawWallClockMs === null ? [] : [openclawWallClockMs],
|
||||
),
|
||||
);
|
||||
const codex = summarizeRuntimeWallClock(
|
||||
scenarios.flatMap(({ codexWallClockMs }) =>
|
||||
codexWallClockMs === null ? [] : [codexWallClockMs],
|
||||
),
|
||||
);
|
||||
const pairedTimingCaptures = scenarios.flatMap(({ openclawWallClockMs, codexWallClockMs }) =>
|
||||
openclawWallClockMs === null || codexWallClockMs === null
|
||||
? []
|
||||
: [{ openclawWallClockMs, codexWallClockMs }],
|
||||
);
|
||||
const openclawBootstrapValues = scenarios.flatMap(({ openclawBootstrapWallClockMs }) =>
|
||||
openclawBootstrapWallClockMs == null ? [] : [openclawBootstrapWallClockMs],
|
||||
);
|
||||
const codexBootstrapValues = scenarios.flatMap(({ codexBootstrapWallClockMs }) =>
|
||||
codexBootstrapWallClockMs == null ? [] : [codexBootstrapWallClockMs],
|
||||
);
|
||||
return {
|
||||
openclaw,
|
||||
codex,
|
||||
...(openclawBootstrapValues.length > 0 || codexBootstrapValues.length > 0
|
||||
? {
|
||||
bootstrap: {
|
||||
openclaw: summarizeRuntimeWallClock(openclawBootstrapValues),
|
||||
codex: summarizeRuntimeWallClock(codexBootstrapValues),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
...compareRuntimeWallClockMs(
|
||||
pairedTimingCaptures.length > 0
|
||||
? pairedTimingCaptures.reduce((total, capture) => total + capture.openclawWallClockMs, 0)
|
||||
: null,
|
||||
pairedTimingCaptures.length > 0
|
||||
? pairedTimingCaptures.reduce((total, capture) => total + capture.codexWallClockMs, 0)
|
||||
: null,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export function formatRuntimeWallClockMs(value: number | null): string {
|
||||
return value === null ? "N/A" : `${value} ms`;
|
||||
}
|
||||
|
||||
export function formatRuntimeSpeedComparison(comparison: QaRuntimeSpeedComparison): string {
|
||||
if (comparison.fasterRuntime === null || comparison.speedupPercent === null) {
|
||||
return "N/A";
|
||||
}
|
||||
if (comparison.fasterRuntime === "tie") {
|
||||
return "tie";
|
||||
}
|
||||
return `${comparison.fasterRuntime} ${comparison.speedupPercent.toFixed(1)}% faster`;
|
||||
}
|
||||
@@ -57,6 +57,7 @@ export type RuntimeParityCell = {
|
||||
finalText: string;
|
||||
usage: RuntimeParityUsage;
|
||||
wallClockMs: number;
|
||||
bootstrapWallClockMs?: number;
|
||||
transportErrorClass?: string;
|
||||
runtimeErrorClass?: string;
|
||||
bootStateLines: string[];
|
||||
@@ -160,6 +161,7 @@ type RuntimeParityCaptureParams = {
|
||||
gateway: QaGatewayLike;
|
||||
scenarioResult: QaSuiteScenarioLike;
|
||||
wallClockMs: number;
|
||||
bootstrapWallClockMs?: number;
|
||||
agentId?: string;
|
||||
mockBaseUrl?: string;
|
||||
};
|
||||
@@ -1470,6 +1472,9 @@ export async function captureRuntimeParityCell(
|
||||
finalText: extractFinalAssistantText(transcriptRecords),
|
||||
usage: aggregateUsage(transcriptRecords),
|
||||
wallClockMs: params.wallClockMs,
|
||||
...(params.bootstrapWallClockMs === undefined
|
||||
? {}
|
||||
: { bootstrapWallClockMs: params.bootstrapWallClockMs }),
|
||||
...(scenarioErrorClass || sentinelErrorClass
|
||||
? { runtimeErrorClass: scenarioErrorClass ?? sentinelErrorClass }
|
||||
: {}),
|
||||
|
||||
@@ -13,11 +13,13 @@ import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js";
|
||||
const tempRoots: string[] = [];
|
||||
|
||||
async function makeEnv(overrides: Partial<QaSuiteRuntimeEnv> = {}): Promise<QaSuiteRuntimeEnv> {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "runtime-tool-fixture-"));
|
||||
tempRoots.push(workspaceDir);
|
||||
const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "runtime-tool-fixture-"));
|
||||
const workspaceDir = path.join(tempRoot, "workspace");
|
||||
await fs.mkdir(workspaceDir);
|
||||
tempRoots.push(tempRoot);
|
||||
return {
|
||||
outputDir: workspaceDir,
|
||||
repoRoot: workspaceDir,
|
||||
outputDir: tempRoot,
|
||||
repoRoot: tempRoot,
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: "openai/gpt-5.6-luna",
|
||||
@@ -26,7 +28,7 @@ async function makeEnv(overrides: Partial<QaSuiteRuntimeEnv> = {}): Promise<QaSu
|
||||
transport: {} as QaSuiteRuntimeEnv["transport"],
|
||||
gateway: {
|
||||
baseUrl: "http://127.0.0.1:1",
|
||||
tempRoot: workspaceDir,
|
||||
tempRoot,
|
||||
workspaceDir,
|
||||
runtimeEnv: {},
|
||||
call: vi.fn(),
|
||||
@@ -104,12 +106,110 @@ async function writeLiveRuntimeToolEvidence(env: QaSuiteRuntimeEnv, toolName = "
|
||||
]);
|
||||
}
|
||||
|
||||
async function writeCodexNativePatchEvidence(
|
||||
env: QaSuiteRuntimeEnv,
|
||||
failureOutput = "apply_patch failed: path escapes sandbox root",
|
||||
options: {
|
||||
happyPath?: string;
|
||||
failurePath?: string;
|
||||
happyKind?: string;
|
||||
failureKind?: string;
|
||||
failureStructuredError?: boolean;
|
||||
} = {},
|
||||
) {
|
||||
const toolName = "apply_patch";
|
||||
await writeQaSessionTranscript(env, `agent:qa:runtime-tool:${toolName}:happy`, [
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "native-patch-happy",
|
||||
name: toolName,
|
||||
arguments: {
|
||||
changes: [
|
||||
{
|
||||
path: options.happyPath ?? "runtime-tool-fixture-patch.txt",
|
||||
kind: { type: options.happyKind ?? "add" },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
role: "toolResult",
|
||||
toolName,
|
||||
toolCallId: "native-patch-happy",
|
||||
isError: false,
|
||||
content: [
|
||||
{
|
||||
type: "toolResult",
|
||||
toolName,
|
||||
toolCallId: "native-patch-happy",
|
||||
content: "apply_patch completed",
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
await writeQaSessionTranscript(env, `agent:qa:runtime-tool:${toolName}:failure`, [
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "native-patch-failure",
|
||||
name: toolName,
|
||||
arguments: {
|
||||
changes: [
|
||||
{
|
||||
path: options.failurePath ?? "../runtime-tool-fixture-denied.txt",
|
||||
kind: { type: options.failureKind ?? "update" },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
role: "toolResult",
|
||||
toolName,
|
||||
toolCallId: "native-patch-failure",
|
||||
isError: options.failureStructuredError ?? true,
|
||||
content: [
|
||||
{
|
||||
type: "toolResult",
|
||||
toolName,
|
||||
toolCallId: "native-patch-failure",
|
||||
content: failureOutput,
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
async function simulateRuntimePatchHappyTurn(
|
||||
env: Pick<QaSuiteRuntimeEnv, "gateway">,
|
||||
params: { sessionKey: string },
|
||||
contents: string | null = "runtime patch\n",
|
||||
) {
|
||||
if (params.sessionKey.endsWith(":happy") && contents !== null) {
|
||||
await fs.writeFile(
|
||||
path.join(env.gateway.workspaceDir, "runtime-tool-fixture-patch.txt"),
|
||||
contents,
|
||||
"utf8",
|
||||
);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
async function runMockRuntimeToolFixtureWithOutputs(params: {
|
||||
toolName: string;
|
||||
happyArgs: Record<string, unknown>;
|
||||
failureArgs: Record<string, unknown>;
|
||||
happyOutput: string;
|
||||
failureOutput: string;
|
||||
happyPatchContents?: string | null;
|
||||
}) {
|
||||
const env = await makeEnv({
|
||||
mock: { baseUrl: "http://127.0.0.1:9999" },
|
||||
@@ -160,7 +260,12 @@ async function runMockRuntimeToolFixtureWithOutputs(params: {
|
||||
{
|
||||
createSession: vi.fn(async (_env, _label, key) => key!),
|
||||
readEffectiveTools: vi.fn(async () => new Set([params.toolName])),
|
||||
runAgentPrompt: vi.fn(async () => ({})),
|
||||
runAgentPrompt: vi.fn(async (runEnv, promptParams) => {
|
||||
if (params.toolName === "apply_patch") {
|
||||
return simulateRuntimePatchHappyTurn(runEnv, promptParams, params.happyPatchContents);
|
||||
}
|
||||
return {};
|
||||
}),
|
||||
fetchJson,
|
||||
ensureImageGenerationConfigured: vi.fn(),
|
||||
},
|
||||
@@ -617,6 +722,573 @@ describe("runtime tool fixture", () => {
|
||||
expect(transcriptToolNames).toEqual([undefined, undefined]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"apply_patch failed: path escapes sandbox root",
|
||||
"Operation not permitted (os error 1)",
|
||||
"patch rejected: writing outside of the project; rejected by user approval settings",
|
||||
])("verifies native Codex patch success and workspace denial: %s", async (failureOutput) => {
|
||||
const env = await makeEnv();
|
||||
env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME = "codex";
|
||||
await writeCodexNativePatchEvidence(env, failureOutput);
|
||||
const promptEvidence: Array<{
|
||||
requireSuccessfulTranscriptToolResult?: boolean;
|
||||
transcriptToolName?: string;
|
||||
}> = [];
|
||||
|
||||
const details = await runRuntimeToolFixture(
|
||||
env,
|
||||
{
|
||||
toolName: "apply_patch",
|
||||
toolCoverage: {
|
||||
bucket: "codex-native-workspace",
|
||||
expectedLayer: "codex-native-workspace",
|
||||
required: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
createSession: vi.fn(async (_env, _label, key) => key!),
|
||||
readEffectiveTools: vi.fn(async () => new Set<string>()),
|
||||
runAgentPrompt: vi.fn(async (_env, params) => {
|
||||
promptEvidence.push({
|
||||
transcriptToolName: params.transcriptToolName,
|
||||
requireSuccessfulTranscriptToolResult: params.requireSuccessfulTranscriptToolResult,
|
||||
});
|
||||
return simulateRuntimePatchHappyTurn(_env, params);
|
||||
}),
|
||||
fetchJson: vi.fn(),
|
||||
ensureImageGenerationConfigured: vi.fn(),
|
||||
},
|
||||
);
|
||||
|
||||
expect(promptEvidence).toEqual([
|
||||
{ transcriptToolName: "apply_patch", requireSuccessfulTranscriptToolResult: true },
|
||||
{ transcriptToolName: "apply_patch", requireSuccessfulTranscriptToolResult: undefined },
|
||||
]);
|
||||
expect(details).toContain("apply_patch live provider happy planned args");
|
||||
expect(details).toContain("runtime-tool-fixture-patch.txt");
|
||||
expect(details).toContain("../runtime-tool-fixture-denied.txt");
|
||||
await expect(
|
||||
fs.access(path.resolve(env.gateway.workspaceDir, "../runtime-tool-fixture-denied.txt")),
|
||||
).rejects.toThrow();
|
||||
await expect(
|
||||
fs.access(path.join(env.gateway.workspaceDir, "runtime-tool-fixture-patch.txt")),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("rejects native patch transcripts that claim success without creating the workspace file", async () => {
|
||||
const env = await makeEnv();
|
||||
env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME = "codex";
|
||||
await writeCodexNativePatchEvidence(env);
|
||||
|
||||
await expect(
|
||||
runRuntimeToolFixture(
|
||||
env,
|
||||
{
|
||||
toolName: "apply_patch",
|
||||
toolCoverage: {
|
||||
bucket: "codex-native-workspace",
|
||||
expectedLayer: "codex-native-workspace",
|
||||
required: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
createSession: vi.fn(async (_env, _label, key) => key!),
|
||||
readEffectiveTools: vi.fn(async () => new Set<string>()),
|
||||
runAgentPrompt: vi.fn(async () => ({})),
|
||||
fetchJson: vi.fn(),
|
||||
ensureImageGenerationConfigured: vi.fn(),
|
||||
},
|
||||
),
|
||||
).rejects.toThrow(
|
||||
"expected apply_patch to create runtime-tool-fixture-patch.txt with exact contents",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects native Codex patch failures that only report missing patch context", async () => {
|
||||
const env = await makeEnv();
|
||||
env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME = "codex";
|
||||
await writeCodexNativePatchEvidence(
|
||||
env,
|
||||
"apply_patch failed: failed to find expected lines in runtime-tool-fixture-denied.txt",
|
||||
);
|
||||
|
||||
await expect(
|
||||
runRuntimeToolFixture(
|
||||
env,
|
||||
{
|
||||
toolName: "apply_patch",
|
||||
toolCoverage: {
|
||||
bucket: "codex-native-workspace",
|
||||
expectedLayer: "codex-native-workspace",
|
||||
required: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
createSession: vi.fn(async (_env, _label, key) => key!),
|
||||
readEffectiveTools: vi.fn(async () => new Set<string>()),
|
||||
runAgentPrompt: vi.fn(simulateRuntimePatchHappyTurn),
|
||||
fetchJson: vi.fn(),
|
||||
ensureImageGenerationConfigured: vi.fn(),
|
||||
},
|
||||
),
|
||||
).rejects.toThrow(
|
||||
"expected live apply_patch failure to explicitly reject the workspace boundary",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects native Codex patch failures without a linked failure result", async () => {
|
||||
const env = await makeEnv();
|
||||
env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME = "codex";
|
||||
await writeCodexNativePatchEvidence(env, "apply_patch completed", {
|
||||
failureStructuredError: false,
|
||||
});
|
||||
|
||||
await expect(
|
||||
runRuntimeToolFixture(
|
||||
env,
|
||||
{
|
||||
toolName: "apply_patch",
|
||||
toolCoverage: {
|
||||
bucket: "codex-native-workspace",
|
||||
expectedLayer: "codex-native-workspace",
|
||||
required: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
createSession: vi.fn(async (_env, _label, key) => key!),
|
||||
readEffectiveTools: vi.fn(async () => new Set<string>()),
|
||||
runAgentPrompt: vi.fn(simulateRuntimePatchHappyTurn),
|
||||
fetchJson: vi.fn(),
|
||||
ensureImageGenerationConfigured: vi.fn(),
|
||||
},
|
||||
),
|
||||
).rejects.toThrow("expected live failure-path tool failure output for apply_patch");
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "happy-path file",
|
||||
options: { happyPath: "runtime-tool-fixture-wrong.txt" },
|
||||
expectedError: "expected linked live apply_patch to add runtime-tool-fixture-patch.txt",
|
||||
},
|
||||
{
|
||||
label: "failure-path file",
|
||||
options: { failurePath: "../runtime-tool-fixture-wrong.txt" },
|
||||
expectedError:
|
||||
"expected linked live apply_patch to update ../runtime-tool-fixture-denied.txt",
|
||||
},
|
||||
{
|
||||
label: "failure-path operation",
|
||||
options: { failureKind: "add" },
|
||||
expectedError:
|
||||
"expected linked live apply_patch to update ../runtime-tool-fixture-denied.txt",
|
||||
},
|
||||
])("rejects linked native Codex patch evidence for the wrong $label", async (testCase) => {
|
||||
const env = await makeEnv();
|
||||
env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME = "codex";
|
||||
await writeCodexNativePatchEvidence(
|
||||
env,
|
||||
"apply_patch failed: path escapes sandbox root",
|
||||
testCase.options,
|
||||
);
|
||||
|
||||
await expect(
|
||||
runRuntimeToolFixture(
|
||||
env,
|
||||
{
|
||||
toolName: "apply_patch",
|
||||
toolCoverage: {
|
||||
bucket: "codex-native-workspace",
|
||||
expectedLayer: "codex-native-workspace",
|
||||
required: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
createSession: vi.fn(async (_env, _label, key) => key!),
|
||||
readEffectiveTools: vi.fn(async () => new Set<string>()),
|
||||
runAgentPrompt: vi.fn(simulateRuntimePatchHappyTurn),
|
||||
fetchJson: vi.fn(),
|
||||
ensureImageGenerationConfigured: vi.fn(),
|
||||
},
|
||||
),
|
||||
).rejects.toThrow(testCase.expectedError);
|
||||
});
|
||||
|
||||
it("validates the native patch call linked to its result instead of the first plan", async () => {
|
||||
const env = await makeEnv();
|
||||
env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME = "codex";
|
||||
await writeQaSessionTranscript(env, "agent:qa:runtime-tool:apply_patch:happy", [
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "native-patch-unlinked-decoy",
|
||||
name: "apply_patch",
|
||||
arguments: {
|
||||
changes: [{ path: "runtime-tool-fixture-wrong.txt", kind: { type: "add" } }],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
await writeCodexNativePatchEvidence(env);
|
||||
|
||||
await expect(
|
||||
runRuntimeToolFixture(
|
||||
env,
|
||||
{
|
||||
toolName: "apply_patch",
|
||||
toolCoverage: {
|
||||
bucket: "codex-native-workspace",
|
||||
expectedLayer: "codex-native-workspace",
|
||||
required: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
createSession: vi.fn(async (_env, _label, key) => key!),
|
||||
readEffectiveTools: vi.fn(async () => new Set<string>()),
|
||||
runAgentPrompt: vi.fn(simulateRuntimePatchHappyTurn),
|
||||
fetchJson: vi.fn(),
|
||||
ensureImageGenerationConfigured: vi.fn(),
|
||||
},
|
||||
),
|
||||
).resolves.toContain("apply_patch live provider happy planned args");
|
||||
});
|
||||
|
||||
it("fails closed and cleans up when a patch changes the outside-workspace sentinel", async () => {
|
||||
const env = await makeEnv();
|
||||
const sentinelPath = path.resolve(
|
||||
env.gateway.workspaceDir,
|
||||
"../runtime-tool-fixture-denied.txt",
|
||||
);
|
||||
|
||||
await expect(
|
||||
runRuntimeToolFixture(
|
||||
env,
|
||||
{
|
||||
toolName: "apply_patch",
|
||||
toolCoverage: {
|
||||
bucket: "openclaw-dynamic-integration",
|
||||
expectedLayer: "openclaw-dynamic",
|
||||
},
|
||||
},
|
||||
{
|
||||
createSession: vi.fn(async (_env, _label, key) => key!),
|
||||
readEffectiveTools: vi.fn(async () => new Set(["apply_patch"])),
|
||||
runAgentPrompt: vi.fn(async (_env, params) => {
|
||||
if (params.sessionKey.endsWith(":failure")) {
|
||||
expect(await fs.readFile(sentinelPath, "utf8")).toBe(
|
||||
"runtime-tool-fixture-denied-original\n",
|
||||
);
|
||||
await fs.writeFile(sentinelPath, "runtime patch outside the workspace\n", "utf8");
|
||||
}
|
||||
return simulateRuntimePatchHappyTurn(_env, params);
|
||||
}),
|
||||
fetchJson: vi.fn(),
|
||||
ensureImageGenerationConfigured: vi.fn(),
|
||||
},
|
||||
),
|
||||
).rejects.toThrow("apply_patch modified or removed the outside-workspace sentinel");
|
||||
|
||||
await expect(fs.access(sentinelPath)).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("fails closed when required native Codex patch execution has no linked transcript", async () => {
|
||||
const env = await makeEnv();
|
||||
env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME = "codex";
|
||||
await writeQaSessionTranscript(env, "agent:qa:runtime-tool:apply_patch:happy", [
|
||||
{ role: "assistant", content: "The patch was applied." },
|
||||
]);
|
||||
await writeQaSessionTranscript(env, "agent:qa:runtime-tool:apply_patch:failure", [
|
||||
{ role: "assistant", content: "The unsafe patch was rejected." },
|
||||
]);
|
||||
|
||||
await expect(
|
||||
runRuntimeToolFixture(
|
||||
env,
|
||||
{
|
||||
toolName: "apply_patch",
|
||||
toolCoverage: {
|
||||
bucket: "codex-native-workspace",
|
||||
expectedLayer: "codex-native-workspace",
|
||||
required: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
createSession: vi.fn(async (_env, _label, key) => key!),
|
||||
readEffectiveTools: vi.fn(async () => new Set<string>()),
|
||||
runAgentPrompt: vi.fn(simulateRuntimePatchHappyTurn),
|
||||
fetchJson: vi.fn(),
|
||||
ensureImageGenerationConfigured: vi.fn(),
|
||||
},
|
||||
),
|
||||
).rejects.toThrow("expected live happy-path tool call for apply_patch");
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ label: "dynamically exposed", dynamicPatchExposed: true },
|
||||
{ label: "native-only", dynamicPatchExposed: false },
|
||||
])("verifies $label private-QA Codex patch calls without skipping them", async (testCase) => {
|
||||
const env = await makeEnv({
|
||||
mock: { baseUrl: "http://127.0.0.1:9999" },
|
||||
});
|
||||
env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME = "codex";
|
||||
const promptEvidence: Array<{
|
||||
requireSuccessfulTranscriptToolResult?: boolean;
|
||||
transcriptToolName?: string;
|
||||
}> = [];
|
||||
const happyCallId = "private-qa-patch-happy";
|
||||
const failureCallId = "private-qa-patch-failure";
|
||||
const fetchJson = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ cursor: 0 })
|
||||
.mockResolvedValueOnce([
|
||||
{
|
||||
allInputText: "target=apply_patch",
|
||||
plannedToolCallId: happyCallId,
|
||||
plannedToolName: "apply_patch",
|
||||
plannedToolArgs: {
|
||||
input:
|
||||
"*** Begin Patch\n*** Add File: runtime-tool-fixture-patch.txt\n+runtime patch\n*** End Patch\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
allInputText: "target=apply_patch",
|
||||
toolOutputCallId: happyCallId,
|
||||
toolOutput: "Successfully applied patch",
|
||||
},
|
||||
{
|
||||
allInputText: "failure target=apply_patch",
|
||||
plannedToolCallId: failureCallId,
|
||||
plannedToolName: "apply_patch",
|
||||
plannedToolArgs: {
|
||||
input:
|
||||
"*** Begin Patch\n*** Update File: ../runtime-tool-fixture-denied.txt\n@@\n-runtime-tool-fixture-denied-original\n+runtime patch outside the workspace\n*** End Patch\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
allInputText: "failure target=apply_patch",
|
||||
toolOutputCallId: failureCallId,
|
||||
toolOutput: "Error: Path escapes sandbox root",
|
||||
},
|
||||
]);
|
||||
|
||||
const details = await runRuntimeToolFixture(
|
||||
env,
|
||||
{
|
||||
toolName: "apply_patch",
|
||||
toolCoverage: {
|
||||
bucket: "codex-native-workspace",
|
||||
expectedLayer: "codex-native-workspace",
|
||||
required: true,
|
||||
},
|
||||
promptSnippet: "target=apply_patch",
|
||||
failurePromptSnippet: "failure target=apply_patch",
|
||||
},
|
||||
{
|
||||
createSession: vi.fn(async (_env, _label, key) => key!),
|
||||
readEffectiveTools: vi.fn(
|
||||
async () => new Set(testCase.dynamicPatchExposed ? ["apply_patch"] : []),
|
||||
),
|
||||
runAgentPrompt: vi.fn(async (_env, params) => {
|
||||
promptEvidence.push({
|
||||
transcriptToolName: params.transcriptToolName,
|
||||
requireSuccessfulTranscriptToolResult: params.requireSuccessfulTranscriptToolResult,
|
||||
});
|
||||
return simulateRuntimePatchHappyTurn(_env, params);
|
||||
}),
|
||||
fetchJson,
|
||||
ensureImageGenerationConfigured: vi.fn(),
|
||||
},
|
||||
);
|
||||
|
||||
expect(promptEvidence).toEqual([
|
||||
{ transcriptToolName: undefined, requireSuccessfulTranscriptToolResult: undefined },
|
||||
{ transcriptToolName: undefined, requireSuccessfulTranscriptToolResult: undefined },
|
||||
]);
|
||||
expect(details).toContain("apply_patch mock provider happy planned args");
|
||||
expect(details).toContain("runtime-tool-fixture-patch.txt");
|
||||
expect(details).toContain("../runtime-tool-fixture-denied.txt");
|
||||
expect(details).not.toContain("codex-native-workspace apply_patch");
|
||||
await expect(
|
||||
fs.access(path.resolve(env.gateway.workspaceDir, "../runtime-tool-fixture-denied.txt")),
|
||||
).rejects.toThrow();
|
||||
await expect(
|
||||
fs.access(path.join(env.gateway.workspaceDir, "runtime-tool-fixture-patch.txt")),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"Operation not permitted",
|
||||
"Operation not permitted (os error 1)",
|
||||
"EPERM: sandbox denied the requested patch",
|
||||
"patch rejected: writing outside of the project; rejected by user approval settings",
|
||||
])("accepts native sandbox denial as a mock patch failure: %s", async (failureOutput) => {
|
||||
await expect(
|
||||
runMockRuntimeToolFixtureWithOutputs({
|
||||
toolName: "apply_patch",
|
||||
happyArgs: {
|
||||
input:
|
||||
"*** Begin Patch\n*** Add File: runtime-tool-fixture-patch.txt\n+runtime patch\n*** End Patch\n",
|
||||
},
|
||||
failureArgs: {
|
||||
input:
|
||||
"*** Begin Patch\n*** Update File: ../runtime-tool-fixture-denied.txt\n@@\n-runtime-tool-fixture-denied-original\n+runtime patch outside the workspace\n*** End Patch\n",
|
||||
},
|
||||
happyOutput: "Successfully applied patch",
|
||||
failureOutput,
|
||||
}),
|
||||
).resolves.toContain("apply_patch mock provider failure planned args");
|
||||
});
|
||||
|
||||
it("rejects mock patch failures that only report missing patch context", async () => {
|
||||
await expect(
|
||||
runMockRuntimeToolFixtureWithOutputs({
|
||||
toolName: "apply_patch",
|
||||
happyArgs: {
|
||||
input:
|
||||
"*** Begin Patch\n*** Add File: runtime-tool-fixture-patch.txt\n+runtime patch\n*** End Patch\n",
|
||||
},
|
||||
failureArgs: {
|
||||
input:
|
||||
"*** Begin Patch\n*** Update File: ../runtime-tool-fixture-denied.txt\n@@\n-runtime-tool-fixture-denied-original\n+runtime patch outside the workspace\n*** End Patch\n",
|
||||
},
|
||||
happyOutput: "Successfully applied patch",
|
||||
failureOutput: "Error: failed to find expected lines in runtime-tool-fixture-denied.txt",
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
"expected mock apply_patch failure to explicitly reject the workspace boundary",
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ label: "no workspace mutation", happyPatchContents: null },
|
||||
{ label: "incorrect workspace contents", happyPatchContents: "a fabricated patch\n" },
|
||||
])("rejects successful linked mock patch claims with $label", async (testCase) => {
|
||||
await expect(
|
||||
runMockRuntimeToolFixtureWithOutputs({
|
||||
toolName: "apply_patch",
|
||||
happyArgs: {
|
||||
input:
|
||||
"*** Begin Patch\n*** Add File: runtime-tool-fixture-patch.txt\n+runtime patch\n*** End Patch\n",
|
||||
},
|
||||
failureArgs: {
|
||||
input:
|
||||
"*** Begin Patch\n*** Update File: ../runtime-tool-fixture-denied.txt\n@@\n-runtime-tool-fixture-denied-original\n+runtime patch outside the workspace\n*** End Patch\n",
|
||||
},
|
||||
happyOutput: "Successfully applied patch",
|
||||
failureOutput: "Error: Path escapes sandbox root",
|
||||
happyPatchContents: testCase.happyPatchContents,
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
"expected apply_patch to create runtime-tool-fixture-patch.txt with exact contents",
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "happy-path file",
|
||||
happyInput:
|
||||
"*** Begin Patch\n*** Add File: runtime-tool-fixture-wrong.txt\n+runtime patch\n*** End Patch\n",
|
||||
failureInput:
|
||||
"*** Begin Patch\n*** Update File: ../runtime-tool-fixture-denied.txt\n@@\n-runtime-tool-fixture-denied-original\n+runtime patch outside the workspace\n*** End Patch\n",
|
||||
expectedError: "expected linked mock apply_patch to add runtime-tool-fixture-patch.txt",
|
||||
},
|
||||
{
|
||||
label: "failure-path file",
|
||||
happyInput:
|
||||
"*** Begin Patch\n*** Add File: runtime-tool-fixture-patch.txt\n+runtime patch\n*** End Patch\n",
|
||||
failureInput:
|
||||
"*** Begin Patch\n*** Update File: ../runtime-tool-fixture-wrong.txt\n@@\n-runtime-tool-fixture-denied-original\n+runtime patch outside the workspace\n*** End Patch\n",
|
||||
expectedError:
|
||||
"expected linked mock apply_patch to update ../runtime-tool-fixture-denied.txt",
|
||||
},
|
||||
{
|
||||
label: "failure-path context",
|
||||
happyInput:
|
||||
"*** Begin Patch\n*** Add File: runtime-tool-fixture-patch.txt\n+runtime patch\n*** End Patch\n",
|
||||
failureInput:
|
||||
"*** Begin Patch\n*** Update File: ../runtime-tool-fixture-denied.txt\n@@\n-context-that-does-not-exist\n+runtime patch outside the workspace\n*** End Patch\n",
|
||||
expectedError:
|
||||
"expected linked mock apply_patch to update ../runtime-tool-fixture-denied.txt",
|
||||
},
|
||||
])("rejects linked mock patch evidence for the wrong $label", async (testCase) => {
|
||||
await expect(
|
||||
runMockRuntimeToolFixtureWithOutputs({
|
||||
toolName: "apply_patch",
|
||||
happyArgs: { input: testCase.happyInput },
|
||||
failureArgs: { input: testCase.failureInput },
|
||||
happyOutput: "Successfully applied patch",
|
||||
failureOutput: "Error: Path escapes sandbox root",
|
||||
}),
|
||||
).rejects.toThrow(testCase.expectedError);
|
||||
});
|
||||
|
||||
it("rejects unlinked private-QA Codex patch results without waiting for a transcript", async () => {
|
||||
const env = await makeEnv({
|
||||
mock: { baseUrl: "http://127.0.0.1:9999" },
|
||||
});
|
||||
env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME = "codex";
|
||||
const promptEvidence: Array<{
|
||||
requireSuccessfulTranscriptToolResult?: boolean;
|
||||
transcriptToolName?: string;
|
||||
}> = [];
|
||||
const fetchJson = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ cursor: 0 })
|
||||
.mockResolvedValueOnce([
|
||||
{
|
||||
allInputText: "target=apply_patch",
|
||||
plannedToolCallId: "private-qa-patch-happy",
|
||||
plannedToolName: "apply_patch",
|
||||
plannedToolArgs: {
|
||||
input:
|
||||
"*** Begin Patch\n*** Add File: runtime-tool-fixture-patch.txt\n+runtime patch\n*** End Patch\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
allInputText: "target=apply_patch",
|
||||
toolOutputCallId: "unrelated-patch-call",
|
||||
toolOutput: "Successfully applied patch",
|
||||
},
|
||||
]);
|
||||
|
||||
await expect(
|
||||
runRuntimeToolFixture(
|
||||
env,
|
||||
{
|
||||
toolName: "apply_patch",
|
||||
toolCoverage: {
|
||||
bucket: "codex-native-workspace",
|
||||
expectedLayer: "codex-native-workspace",
|
||||
required: true,
|
||||
},
|
||||
promptSnippet: "target=apply_patch",
|
||||
failurePromptSnippet: "failure target=apply_patch",
|
||||
},
|
||||
{
|
||||
createSession: vi.fn(async (_env, _label, key) => key!),
|
||||
readEffectiveTools: vi.fn(async () => new Set(["apply_patch"])),
|
||||
runAgentPrompt: vi.fn(async (_env, params) => {
|
||||
promptEvidence.push({
|
||||
transcriptToolName: params.transcriptToolName,
|
||||
requireSuccessfulTranscriptToolResult: params.requireSuccessfulTranscriptToolResult,
|
||||
});
|
||||
return simulateRuntimePatchHappyTurn(_env, params);
|
||||
}),
|
||||
fetchJson,
|
||||
ensureImageGenerationConfigured: vi.fn(),
|
||||
},
|
||||
),
|
||||
).rejects.toThrow("expected mock happy-path tool output for apply_patch");
|
||||
|
||||
expect(promptEvidence).toEqual([
|
||||
{ transcriptToolName: undefined, requireSuccessfulTranscriptToolResult: undefined },
|
||||
{ transcriptToolName: undefined, requireSuccessfulTranscriptToolResult: undefined },
|
||||
]);
|
||||
});
|
||||
|
||||
it("skips Codex-native async planned-only fixtures without treating the plan as proof", async () => {
|
||||
const env = await makeEnv({
|
||||
mock: { baseUrl: "http://127.0.0.1:9999" },
|
||||
|
||||
@@ -33,6 +33,7 @@ type QaRuntimeToolFixtureConfig = Record<string, unknown> & {
|
||||
};
|
||||
|
||||
type QaRuntimeToolFixtureRequest = {
|
||||
body?: unknown;
|
||||
allInputText?: string;
|
||||
plannedToolCallId?: string;
|
||||
plannedToolName?: string;
|
||||
@@ -57,6 +58,12 @@ type QaRuntimeToolFixtureTranscriptToolResult = {
|
||||
};
|
||||
|
||||
const RUNTIME_PARITY_SESSION_KEY_DETAIL_PREFIX = "RUNTIME_PARITY_SESSION_KEY=";
|
||||
const RUNTIME_PATCH_HAPPY_FILENAME = "runtime-tool-fixture-patch.txt";
|
||||
const RUNTIME_PATCH_HAPPY_CONTENTS = "runtime patch\n";
|
||||
const RUNTIME_PATCH_DENIED_FILENAME = "runtime-tool-fixture-denied.txt";
|
||||
const RUNTIME_PATCH_DENIED_CONTENTS = "runtime-tool-fixture-denied-original\n";
|
||||
const RUNTIME_PATCH_WORKSPACE_DENIAL_RE =
|
||||
/(?:path\s+escapes\s+(?:the\s+)?(?:sandbox|workspace)(?:\s+root)?|outside(?:\s+of)?\s+(?:the\s+)?(?:project|sandbox|workspace|allowed\s+(?:sandbox|workspace|root)|writable\s+roots?)(?:\s+root)?|workspace[- ]only|permission\s+denied|operation\s+not\s+permitted|\bos\s+error\s+1\b|\b(?:EACCES|EPERM)\b)/iu;
|
||||
|
||||
function runtimeParitySessionKeyDetails(...sessionKeys: string[]) {
|
||||
return sessionKeys.map(
|
||||
@@ -164,6 +171,148 @@ function requestHasFailureLikeToolOutput(request: QaRuntimeToolFixtureRequest) {
|
||||
);
|
||||
}
|
||||
|
||||
function isWorkspaceBoundaryFailureToolOutput(text: unknown) {
|
||||
return typeof text === "string" && RUNTIME_PATCH_WORKSPACE_DENIAL_RE.test(text);
|
||||
}
|
||||
|
||||
function formatRuntimePatchFailureOutput(request: QaRuntimeToolFixtureRequest): string {
|
||||
const text =
|
||||
typeof request.toolOutput === "string"
|
||||
? request.toolOutput
|
||||
.replace(
|
||||
/\b(?:bearer\s+[a-z\d._~+/-]+=*|(?:api[_-]?key|access[_-]?token|authorization|password|secret)\s*[:=]\s*["']?[^\s"',;]+)/giu,
|
||||
"[REDACTED]",
|
||||
)
|
||||
.replace(
|
||||
/\b(?:sk|sess|ghp|gho|github_pat|xox[baprs])[-_][a-z\d_-]{8,}\b/giu,
|
||||
"[REDACTED]",
|
||||
)
|
||||
.slice(0, 240)
|
||||
: undefined;
|
||||
return JSON.stringify({ text, structuredError: request.toolOutputStructuredError === true });
|
||||
}
|
||||
|
||||
function matchesRuntimePatchInput(input: unknown, operation: "add" | "update"): boolean {
|
||||
if (typeof input !== "string") {
|
||||
return false;
|
||||
}
|
||||
const lines = input.replace(/\r\n?/gu, "\n").split("\n");
|
||||
while (lines.at(-1) === "") {
|
||||
lines.pop();
|
||||
}
|
||||
if (lines[0] !== "*** Begin Patch" || lines.at(-1) !== "*** End Patch") {
|
||||
return false;
|
||||
}
|
||||
const fileHeaders = lines.filter((line) => /^\*\*\* (?:Add|Update|Delete) File: /u.test(line));
|
||||
const expectedHeader =
|
||||
operation === "add"
|
||||
? `*** Add File: ${RUNTIME_PATCH_HAPPY_FILENAME}`
|
||||
: `*** Update File: ../${RUNTIME_PATCH_DENIED_FILENAME}`;
|
||||
if (fileHeaders.length !== 1 || fileHeaders[0] !== expectedHeader) {
|
||||
return false;
|
||||
}
|
||||
return operation === "add"
|
||||
? lines.includes("+runtime patch")
|
||||
: lines.includes("@@") &&
|
||||
lines.includes(`-${RUNTIME_PATCH_DENIED_CONTENTS.trimEnd()}`) &&
|
||||
lines.includes("+runtime patch outside the workspace");
|
||||
}
|
||||
|
||||
function matchesRuntimePatchArguments(params: {
|
||||
args: unknown;
|
||||
workspaceDir: string;
|
||||
operation: "add" | "update";
|
||||
}): boolean {
|
||||
if (!isRecord(params.args)) {
|
||||
return false;
|
||||
}
|
||||
if (typeof params.args.input === "string") {
|
||||
return matchesRuntimePatchInput(params.args.input, params.operation);
|
||||
}
|
||||
const changes = params.args.changes;
|
||||
if (!Array.isArray(changes) || changes.length !== 1 || !isRecord(changes[0])) {
|
||||
return false;
|
||||
}
|
||||
const change = changes[0];
|
||||
if (typeof change.path !== "string") {
|
||||
return false;
|
||||
}
|
||||
const kind = change.kind;
|
||||
const operation = isRecord(kind) ? kind.type : kind;
|
||||
const expectedPath =
|
||||
params.operation === "add"
|
||||
? path.resolve(params.workspaceDir, RUNTIME_PATCH_HAPPY_FILENAME)
|
||||
: path.resolve(params.workspaceDir, "..", RUNTIME_PATCH_DENIED_FILENAME);
|
||||
return (
|
||||
operation === params.operation &&
|
||||
path.resolve(params.workspaceDir, change.path) === expectedPath
|
||||
);
|
||||
}
|
||||
|
||||
async function formatRuntimePatchMutationDiagnostics(params: {
|
||||
env: QaSuiteRuntimeEnv;
|
||||
deps: QaRuntimeToolFixtureDeps;
|
||||
requestCursor: number;
|
||||
}) {
|
||||
const workspaceEntries = await fs
|
||||
.readdir(params.env.gateway.workspaceDir)
|
||||
.then((entries) => entries.toSorted().slice(0, 16))
|
||||
.catch(() => [] as string[]);
|
||||
const tempRootEntries = await fs
|
||||
.readdir(params.env.gateway.tempRoot)
|
||||
.then((entries) => entries.toSorted().slice(0, 16))
|
||||
.catch(() => [] as string[]);
|
||||
const gatewayPatchLogs = (params.env.gateway.logs?.() ?? "")
|
||||
.split(/\r?\n/u)
|
||||
.filter((line) =>
|
||||
/custom tool call output is missing|apply[._ -]?patch|failed to parse responseitem|duplicate|tool registration/iu.test(
|
||||
line,
|
||||
),
|
||||
)
|
||||
.slice(-6)
|
||||
.map((line) =>
|
||||
line
|
||||
.replace(
|
||||
/\b(?:bearer\s+[a-z\d._~+/-]+=*|(?:api[_-]?key|access[_-]?token|authorization|password|secret)\s*[:=]\s*["']?[^\s"',;]+)/giu,
|
||||
"[REDACTED]",
|
||||
)
|
||||
.replace(/\b(?:sk|sess|ghp|gho|github_pat|xox[baprs])[-_][a-z\d_-]{8,}\b/giu, "[REDACTED]")
|
||||
.slice(0, 200),
|
||||
);
|
||||
const mockRequests = params.env.mock
|
||||
? await params.deps
|
||||
.fetchJson(qaMockRequestsAfterUrl(params.env.mock.baseUrl, params.requestCursor))
|
||||
.then(readQaRuntimeToolFixtureRequests)
|
||||
.then((requests) =>
|
||||
requests.slice(-4).map((request) => {
|
||||
const body = isRecord(request.body) ? request.body : undefined;
|
||||
const tools = Array.isArray(body?.tools) ? body.tools : [];
|
||||
return {
|
||||
plannedToolName: request.plannedToolName,
|
||||
plannedToolCallId: request.plannedToolCallId,
|
||||
toolOutputCallId: request.toolOutputCallId,
|
||||
toolOutput: request.toolOutput?.slice(0, 256),
|
||||
patchToolTypes: tools.flatMap((tool) =>
|
||||
isRecord(tool) && tool.name === "apply_patch" && typeof tool.type === "string"
|
||||
? [tool.type]
|
||||
: [],
|
||||
),
|
||||
};
|
||||
}),
|
||||
)
|
||||
.catch(() => [])
|
||||
: [];
|
||||
return [
|
||||
`workspace=${params.env.gateway.workspaceDir}`,
|
||||
`workspaceEntries=${JSON.stringify(workspaceEntries)}`,
|
||||
`tempRootEntries=${JSON.stringify(tempRootEntries)}`,
|
||||
...(gatewayPatchLogs.length > 0
|
||||
? [`gatewayPatchLogs=${JSON.stringify(gatewayPatchLogs)}`]
|
||||
: []),
|
||||
...(params.env.mock ? [`mockPatchRequests=${JSON.stringify(mockRequests)}`] : []),
|
||||
].join("; ");
|
||||
}
|
||||
|
||||
function readNonEmptyString(value: unknown) {
|
||||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
|
||||
}
|
||||
@@ -276,7 +425,7 @@ function readBooleanTrue(value: unknown) {
|
||||
}
|
||||
|
||||
const FAILURE_LIKE_TOOL_RESULT_RE =
|
||||
/\b(?:denied|enoent|error|exception|fail(?:ed|ure)?|forbidden|invalid|missing|not found|permission)\b/iu;
|
||||
/\b(?:denied|enoent|error|exception|fail(?:ed|ure)?|forbidden|invalid|missing|not found|permission|reject(?:ed|ion)?)\b/iu;
|
||||
|
||||
const REQUIRED_FIELD_TOOL_RESULT_RE =
|
||||
/(?:^|[\n:,({[]\s*)["']?[A-Z_][A-Z0-9_.[\]-]*["']?\s+(?:is\s+)?required\b/iu;
|
||||
@@ -290,6 +439,7 @@ function isFailureLikeToolResult(params: {
|
||||
return (
|
||||
isStructuredFailureToolResult(params) ||
|
||||
isHardFailureToolOutputText(params.text) ||
|
||||
isWorkspaceBoundaryFailureToolOutput(params.text) ||
|
||||
FAILURE_LIKE_TOOL_RESULT_RE.test(params.text) ||
|
||||
REQUIRED_FIELD_TOOL_RESULT_RE.test(params.text)
|
||||
);
|
||||
@@ -419,19 +569,22 @@ function readTranscriptToolEvidence(transcriptBytes: string, toolName: string) {
|
||||
// Ignore malformed transcript rows and keep live fixture evidence deterministic.
|
||||
}
|
||||
}
|
||||
const outputResult = calls
|
||||
.map((call) =>
|
||||
results.find((result) =>
|
||||
const linkedEvidence = calls
|
||||
.map((call) => ({
|
||||
call,
|
||||
result: results.find((result) =>
|
||||
transcriptToolResultLinksCall({
|
||||
call,
|
||||
result,
|
||||
targetCallCount: calls.length,
|
||||
}),
|
||||
),
|
||||
)
|
||||
.find((result) => result && result.text.trim().length > 0);
|
||||
}))
|
||||
.find(({ result }) => result && result.text.trim().length > 0);
|
||||
const outputResult = linkedEvidence?.result;
|
||||
return {
|
||||
plannedRequest: calls[0],
|
||||
executedRequest: linkedEvidence?.call,
|
||||
outputRequest: outputResult,
|
||||
failureOutputRequest: outputResult?.failure ? outputResult : undefined,
|
||||
};
|
||||
@@ -677,7 +830,10 @@ export async function runRuntimeToolFixture(
|
||||
});
|
||||
const dynamicExposureIntentionallyExcluded =
|
||||
env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === "codex" &&
|
||||
metadata.expectedLayer === "codex-native-workspace";
|
||||
metadata.expectedLayer === "codex-native-workspace" &&
|
||||
!tools.has(toolName);
|
||||
const requireCodexNativePatchCoverage =
|
||||
dynamicExposureIntentionallyExcluded && metadata.required && toolName === "apply_patch";
|
||||
const expectedAvailable = readBoolean(config.expectedAvailable, true);
|
||||
if (!tools.has(toolName) && !dynamicExposureIntentionallyExcluded) {
|
||||
if (!expectedAvailable) {
|
||||
@@ -710,9 +866,13 @@ export async function runRuntimeToolFixture(
|
||||
`failure target=${toolName}`,
|
||||
);
|
||||
const happyPathOutputRequired = readBoolean(config.happyPathOutputRequired, true);
|
||||
// Private QA can expose an actual dynamic patch tool. Real Codex instead
|
||||
// mirrors native file changes into linked apply_patch transcript evidence.
|
||||
const requireNativePatchTranscriptEvidence = !env.mock && requireCodexNativePatchCoverage;
|
||||
const requireTranscriptEvidence =
|
||||
metadata.required &&
|
||||
!dynamicExposureIntentionallyExcluded &&
|
||||
(!env.mock || toolName !== "apply_patch") &&
|
||||
(!dynamicExposureIntentionallyExcluded || requireNativePatchTranscriptEvidence) &&
|
||||
!isKnownHarnessGap(config.knownHarnessGap);
|
||||
const mockBaseUrl = env.mock?.baseUrl;
|
||||
const requestCursorBefore = mockBaseUrl
|
||||
@@ -721,24 +881,82 @@ export async function runRuntimeToolFixture(
|
||||
)
|
||||
: 0;
|
||||
|
||||
await runFixtureOperation(() =>
|
||||
deps.runAgentPrompt(env, {
|
||||
sessionKey: happySessionKey,
|
||||
message: happyPrompt,
|
||||
timeoutMs: liveTurnTimeoutMs(env, 45_000),
|
||||
...(happyPathOutputRequired && requireTranscriptEvidence
|
||||
? { transcriptToolName: toolName, requireSuccessfulTranscriptToolResult: true }
|
||||
: {}),
|
||||
}),
|
||||
);
|
||||
await runFixtureOperation(() =>
|
||||
deps.runAgentPrompt(env, {
|
||||
sessionKey: failureSessionKey,
|
||||
message: failurePrompt,
|
||||
timeoutMs: liveTurnTimeoutMs(env, 45_000),
|
||||
...(requireTranscriptEvidence ? { transcriptToolName: toolName } : {}),
|
||||
}),
|
||||
);
|
||||
await runFixtureOperation(async () => {
|
||||
const runHappyPrompt = () =>
|
||||
deps.runAgentPrompt(env, {
|
||||
sessionKey: happySessionKey,
|
||||
message: happyPrompt,
|
||||
timeoutMs: liveTurnTimeoutMs(env, 45_000),
|
||||
...(happyPathOutputRequired && requireTranscriptEvidence
|
||||
? { transcriptToolName: toolName, requireSuccessfulTranscriptToolResult: true }
|
||||
: {}),
|
||||
});
|
||||
if (toolName !== "apply_patch" || !metadata.required) {
|
||||
return runHappyPrompt();
|
||||
}
|
||||
const happyPatchPath = path.join(env.gateway.workspaceDir, RUNTIME_PATCH_HAPPY_FILENAME);
|
||||
const readHappyPatchContents = async () =>
|
||||
fs.readFile(happyPatchPath, "utf8").catch((error: unknown) => {
|
||||
if (isRecord(error) && error.code === "ENOENT") {
|
||||
return undefined;
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
if ((await readHappyPatchContents()) !== undefined) {
|
||||
throw new Error(
|
||||
`apply_patch happy-path target already exists: ${RUNTIME_PATCH_HAPPY_FILENAME}`,
|
||||
);
|
||||
}
|
||||
try {
|
||||
const result = await runHappyPrompt();
|
||||
if ((await readHappyPatchContents()) !== RUNTIME_PATCH_HAPPY_CONTENTS) {
|
||||
const diagnostics = await formatRuntimePatchMutationDiagnostics({
|
||||
env,
|
||||
deps,
|
||||
requestCursor: requestCursorBefore,
|
||||
});
|
||||
throw new Error(
|
||||
`expected apply_patch to create ${RUNTIME_PATCH_HAPPY_FILENAME} with exact contents; ${diagnostics}`,
|
||||
);
|
||||
}
|
||||
return result;
|
||||
} finally {
|
||||
await fs.rm(happyPatchPath, { force: true });
|
||||
}
|
||||
});
|
||||
await runFixtureOperation(async () => {
|
||||
const runFailurePrompt = () =>
|
||||
deps.runAgentPrompt(env, {
|
||||
sessionKey: failureSessionKey,
|
||||
message: failurePrompt,
|
||||
timeoutMs: liveTurnTimeoutMs(env, 45_000),
|
||||
...(requireTranscriptEvidence ? { transcriptToolName: toolName } : {}),
|
||||
});
|
||||
if (toolName !== "apply_patch") {
|
||||
return runFailurePrompt();
|
||||
}
|
||||
const deniedPatchPath = path.resolve(
|
||||
env.gateway.workspaceDir,
|
||||
"..",
|
||||
RUNTIME_PATCH_DENIED_FILENAME,
|
||||
);
|
||||
// Matching outside context makes failure evidence prove containment, not
|
||||
// merely that apply_patch could not find a file or match a hunk.
|
||||
await fs.writeFile(deniedPatchPath, RUNTIME_PATCH_DENIED_CONTENTS, {
|
||||
encoding: "utf8",
|
||||
flag: "wx",
|
||||
});
|
||||
try {
|
||||
const result = await runFailurePrompt();
|
||||
const sentinelContents = await fs.readFile(deniedPatchPath, "utf8").catch(() => undefined);
|
||||
if (sentinelContents !== RUNTIME_PATCH_DENIED_CONTENTS) {
|
||||
throw new Error("apply_patch modified or removed the outside-workspace sentinel");
|
||||
}
|
||||
return result;
|
||||
} finally {
|
||||
await fs.rm(deniedPatchPath, { force: true });
|
||||
}
|
||||
});
|
||||
|
||||
if (!env.mock) {
|
||||
const happyRequest = await runFixtureOperation(() =>
|
||||
@@ -775,6 +993,19 @@ export async function runRuntimeToolFixture(
|
||||
new Error(`expected live happy-path successful tool output for ${toolName}`),
|
||||
);
|
||||
}
|
||||
if (
|
||||
toolName === "apply_patch" &&
|
||||
metadata.required &&
|
||||
!matchesRuntimePatchArguments({
|
||||
args: happyRequest.executedRequest?.args,
|
||||
workspaceDir: env.gateway.workspaceDir,
|
||||
operation: "add",
|
||||
})
|
||||
) {
|
||||
throw fixtureError(
|
||||
new Error(`expected linked live apply_patch to add ${RUNTIME_PATCH_HAPPY_FILENAME}`),
|
||||
);
|
||||
}
|
||||
const failureRequest = await runFixtureOperation(() =>
|
||||
readLiveToolEvidence({
|
||||
env,
|
||||
@@ -802,6 +1033,27 @@ export async function runRuntimeToolFixture(
|
||||
new Error(`expected live failure-path tool failure output for ${toolName}`),
|
||||
);
|
||||
}
|
||||
if (
|
||||
toolName === "apply_patch" &&
|
||||
metadata.required &&
|
||||
!matchesRuntimePatchArguments({
|
||||
args: failureRequest.executedRequest?.args,
|
||||
workspaceDir: env.gateway.workspaceDir,
|
||||
operation: "update",
|
||||
})
|
||||
) {
|
||||
throw fixtureError(
|
||||
new Error(`expected linked live apply_patch to update ../${RUNTIME_PATCH_DENIED_FILENAME}`),
|
||||
);
|
||||
}
|
||||
if (
|
||||
toolName === "apply_patch" &&
|
||||
!isWorkspaceBoundaryFailureToolOutput(failureRequest.failureOutputRequest?.text)
|
||||
) {
|
||||
throw fixtureError(
|
||||
new Error("expected live apply_patch failure to explicitly reject the workspace boundary"),
|
||||
);
|
||||
}
|
||||
return withSessionDetails(
|
||||
[
|
||||
`${toolName} live provider happy planned args (diagnostic only): ${JSON.stringify(happyRequest.plannedRequest?.args ?? {})}`,
|
||||
@@ -875,7 +1127,7 @@ export async function runRuntimeToolFixture(
|
||||
);
|
||||
}
|
||||
if (!happyRequest && !happyPlannedOnly) {
|
||||
if (dynamicExposureIntentionallyExcluded) {
|
||||
if (dynamicExposureIntentionallyExcluded && !requireCodexNativePatchCoverage) {
|
||||
skipFixture(
|
||||
formatCodexNativeWorkspaceDetails({
|
||||
toolName,
|
||||
@@ -904,8 +1156,22 @@ export async function runRuntimeToolFixture(
|
||||
new Error(`expected mock happy-path successful tool output for ${toolName}`),
|
||||
);
|
||||
}
|
||||
if (
|
||||
toolName === "apply_patch" &&
|
||||
metadata.required &&
|
||||
happyRequest &&
|
||||
!matchesRuntimePatchArguments({
|
||||
args: happyRequest.plannedRequest.plannedToolArgs,
|
||||
workspaceDir: env.gateway.workspaceDir,
|
||||
operation: "add",
|
||||
})
|
||||
) {
|
||||
throw fixtureError(
|
||||
new Error(`expected linked mock apply_patch to add ${RUNTIME_PATCH_HAPPY_FILENAME}`),
|
||||
);
|
||||
}
|
||||
if (!failureRequest) {
|
||||
if (dynamicExposureIntentionallyExcluded) {
|
||||
if (dynamicExposureIntentionallyExcluded && !requireCodexNativePatchCoverage) {
|
||||
skipFixture(
|
||||
formatCodexNativeWorkspaceDetails({
|
||||
toolName,
|
||||
@@ -931,10 +1197,39 @@ export async function runRuntimeToolFixture(
|
||||
if (isKnownHarnessGap(config.knownHarnessGap)) {
|
||||
skipFixture(formatKnownHarnessGapDetails(toolName, config));
|
||||
}
|
||||
throw fixtureError(new Error(`expected mock failure-path tool failure output for ${toolName}`));
|
||||
const patchFailureDiagnostics =
|
||||
toolName === "apply_patch"
|
||||
? `; received ${formatRuntimePatchFailureOutput(failureRequest.outputRequest)}`
|
||||
: "";
|
||||
throw fixtureError(
|
||||
new Error(
|
||||
`expected mock failure-path tool failure output for ${toolName}${patchFailureDiagnostics}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
if (
|
||||
toolName === "apply_patch" &&
|
||||
metadata.required &&
|
||||
!matchesRuntimePatchArguments({
|
||||
args: failureRequest.plannedRequest.plannedToolArgs,
|
||||
workspaceDir: env.gateway.workspaceDir,
|
||||
operation: "update",
|
||||
})
|
||||
) {
|
||||
throw fixtureError(
|
||||
new Error(`expected linked mock apply_patch to update ../${RUNTIME_PATCH_DENIED_FILENAME}`),
|
||||
);
|
||||
}
|
||||
if (
|
||||
toolName === "apply_patch" &&
|
||||
!isWorkspaceBoundaryFailureToolOutput(failureRequest.outputRequest.toolOutput)
|
||||
) {
|
||||
throw fixtureError(
|
||||
new Error("expected mock apply_patch failure to explicitly reject the workspace boundary"),
|
||||
);
|
||||
}
|
||||
|
||||
if (dynamicExposureIntentionallyExcluded) {
|
||||
if (dynamicExposureIntentionallyExcluded && !requireCodexNativePatchCoverage) {
|
||||
skipFixture(
|
||||
formatCodexNativeWorkspaceDetails({
|
||||
toolName,
|
||||
|
||||
@@ -375,8 +375,25 @@ describe("qa scenario catalog", () => {
|
||||
toolCoverage: {
|
||||
bucket: "codex-native-workspace",
|
||||
expectedLayer: "codex-native-workspace",
|
||||
required: true,
|
||||
},
|
||||
});
|
||||
expect(readQaScenarioExecutionConfig(applyPatch.id)).not.toHaveProperty("knownHarnessGap");
|
||||
expect(readQaScenarioExecutionConfig(applyPatch.id)?.happyPrompt).toContain(
|
||||
"runtime-tool-fixture-patch.txt",
|
||||
);
|
||||
expect(readQaScenarioExecutionConfig(applyPatch.id)?.failurePrompt).toContain(
|
||||
"../runtime-tool-fixture-denied.txt",
|
||||
);
|
||||
expect(readQaScenarioExecutionConfig(applyPatch.id)?.failurePrompt).toContain(
|
||||
"runtime-tool-fixture-denied-original",
|
||||
);
|
||||
expect(readQaScenarioExecutionConfig(applyPatch.id)?.failurePrompt).toContain(
|
||||
"runtime patch outside the workspace",
|
||||
);
|
||||
expect(readQaScenarioExecutionConfig(applyPatch.id)?.failurePrompt).not.toContain(
|
||||
"missing-context",
|
||||
);
|
||||
expect(readQaScenarioExecutionConfig(messageTool.id)).toMatchObject({
|
||||
toolName: "message",
|
||||
expectedAvailable: false,
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
import path from "node:path";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { disposeRegisteredAgentHarnesses } from "openclaw/plugin-sdk/agent-harness";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { startQaGatewayChild } from "./gateway-child.js";
|
||||
import type { QaLabLatestReport, QaLabScenarioOutcome } from "./lab-server.types.js";
|
||||
import { sanitizeQaProgressValue as sanitizeQaSuiteProgressValue } from "./progress-format.js";
|
||||
import { startQaProviderServer } from "./providers/server-runtime.js";
|
||||
import {
|
||||
measureRuntimeParityCellTiming,
|
||||
type QaRuntimeParityCellTiming,
|
||||
} from "./runtime-parity-timing.js";
|
||||
import { captureRuntimeParityCell } from "./runtime-parity.js";
|
||||
import {
|
||||
type QaSuiteGatewayHeapSnapshot,
|
||||
@@ -176,8 +179,8 @@ export async function runQaFlowSuiteStandard(
|
||||
await waitForGatewayHealthy(activeEnv, transportReadyTimeoutMs);
|
||||
await waitForTransportReady(activeEnv, transportReadyTimeoutMs);
|
||||
});
|
||||
await sleep(1_000);
|
||||
const scenarios: QaSuiteScenarioResult[] = [];
|
||||
let runtimeParityCellTiming: QaRuntimeParityCellTiming | undefined;
|
||||
const liveScenarioOutcomes: QaLabScenarioOutcome[] = selectedScenarios.map((scenario) => ({
|
||||
id: scenario.id,
|
||||
name: scenario.title,
|
||||
@@ -240,7 +243,19 @@ export async function runQaFlowSuiteStandard(
|
||||
scenarios: [...liveScenarioOutcomes],
|
||||
});
|
||||
|
||||
const runSelectedScenario = () => runScenarioDefinition(activeEnv, scenario);
|
||||
const scenarioBootstrapFinishedAt = new Date();
|
||||
let scenarioExecutionStartedAt = scenarioBootstrapFinishedAt;
|
||||
let scenarioExecutionFinishedAt = scenarioBootstrapFinishedAt;
|
||||
const runSelectedScenario = async () => {
|
||||
// Retry backoff and unsuccessful attempts are not part of the final
|
||||
// runtime turn, and they must not be relabeled as gateway bootstrap.
|
||||
scenarioExecutionStartedAt = new Date();
|
||||
try {
|
||||
return await runScenarioDefinition(activeEnv, scenario);
|
||||
} finally {
|
||||
scenarioExecutionFinishedAt = new Date();
|
||||
}
|
||||
};
|
||||
const scenarioRetryCount =
|
||||
scenario.execution.kind === "flow" ? scenario.execution.retryCount : undefined;
|
||||
let result: QaSuiteScenarioResult =
|
||||
@@ -273,6 +288,14 @@ export async function runQaFlowSuiteStandard(
|
||||
],
|
||||
};
|
||||
}
|
||||
if (params?.captureRuntimeParityCell && selectedScenarios.length === 1) {
|
||||
runtimeParityCellTiming = measureRuntimeParityCellTiming({
|
||||
suiteStartedAt: startedAt,
|
||||
bootstrapFinishedAt: scenarioBootstrapFinishedAt,
|
||||
scenarioStartedAt: scenarioExecutionStartedAt,
|
||||
scenarioFinishedAt: scenarioExecutionFinishedAt,
|
||||
});
|
||||
}
|
||||
sampleGatewayProcessRss(`scenario:${scenario.id}:finish`);
|
||||
scenarios.push(result);
|
||||
writeQaSuiteProgress(
|
||||
@@ -304,12 +327,13 @@ export async function runQaFlowSuiteStandard(
|
||||
params?.captureRuntimeParityCell &&
|
||||
params.forcedRuntime &&
|
||||
selectedScenarios.length === 1 &&
|
||||
runtimeParityScenario
|
||||
runtimeParityScenario &&
|
||||
runtimeParityCellTiming
|
||||
? await captureRuntimeParityCell({
|
||||
runtime: params.forcedRuntime,
|
||||
gateway: activeGateway,
|
||||
scenarioResult: runtimeParityScenario,
|
||||
wallClockMs: Math.max(1, Date.now() - startedAt.getTime()),
|
||||
...runtimeParityCellTiming,
|
||||
mockBaseUrl: activeMock?.baseUrl,
|
||||
})
|
||||
: undefined;
|
||||
|
||||
@@ -13,6 +13,9 @@ function formatRuntimeParityCellDetails(cell: RuntimeParityCell) {
|
||||
return [
|
||||
`runtime=${cell.runtime}`,
|
||||
`wallMs=${cell.wallClockMs}`,
|
||||
...(cell.bootstrapWallClockMs === undefined
|
||||
? []
|
||||
: [`bootstrapMs=${cell.bootstrapWallClockMs}`]),
|
||||
`toolCalls=${cell.toolCalls.length}`,
|
||||
`finalChars=${cell.finalText.length}`,
|
||||
`tokens=${cell.usage.totalTokens}`,
|
||||
|
||||
@@ -892,15 +892,10 @@ describe("qa tool coverage report", () => {
|
||||
expect.objectContaining({
|
||||
bucket: "codex-native-workspace",
|
||||
expectedLayer: "codex-native-workspace",
|
||||
required: false,
|
||||
}),
|
||||
);
|
||||
expect(applyPatchRow).toEqual(
|
||||
expect.objectContaining({
|
||||
tracking:
|
||||
"#80320 Codex app-server intentionally owns apply_patch natively; this fixture still needs valid patch-shaped fault injection before it can prove product behavior.",
|
||||
required: true,
|
||||
}),
|
||||
);
|
||||
expect(applyPatchRow?.tracking).toBeUndefined();
|
||||
expect(report.rows.find((row) => row.tool === "sessions_spawn")).toEqual(
|
||||
expect.objectContaining({
|
||||
required: true,
|
||||
|
||||
Generated
+3
@@ -1595,6 +1595,9 @@ importers:
|
||||
semver:
|
||||
specifier: 7.8.5
|
||||
version: 7.8.5
|
||||
ws:
|
||||
specifier: 8.21.1
|
||||
version: 8.21.1
|
||||
yaml:
|
||||
specifier: 2.9.0
|
||||
version: 2.9.0
|
||||
|
||||
@@ -7,11 +7,11 @@ scenario:
|
||||
coverage:
|
||||
secondary:
|
||||
- agent-runtime.tool-apply-patch
|
||||
objective: Verify apply_patch behavior is tracked across OpenClaw and Codex while Codex owns patching natively.
|
||||
objective: Verify OpenClaw and Codex execute valid workspace patches and reject workspace-escaping patches.
|
||||
successCriteria:
|
||||
- OpenClaw may expose OpenClaw apply_patch while Codex app-server mode may omit duplicate OpenClaw dynamic apply_patch.
|
||||
- Mock provider apply_patch plans are reported as fixture intent, not as actual runtime tool calls.
|
||||
- The row stays report-only until fault injection uses valid patch-shaped inputs.
|
||||
- OpenClaw executes a valid workspace patch and rejects a matching-context workspace-escaping patch without modifying the outside sentinel.
|
||||
- Private mock QA verifies linked apply_patch calls and results without treating provider plans as execution evidence.
|
||||
- Live Codex verifies projected native file-change calls and results without requiring a duplicate dynamic apply_patch tool.
|
||||
docsRefs:
|
||||
- qa/scenarios/index.yaml
|
||||
codeRefs:
|
||||
@@ -27,15 +27,22 @@ scenario:
|
||||
actualTool: apply_patch
|
||||
bucket: codex-native-workspace
|
||||
expectedLayer: codex-native-workspace
|
||||
required: false
|
||||
tracking: "#80320"
|
||||
required: true
|
||||
codexDefaultImpact: P4
|
||||
qaImpact: P2
|
||||
action: fix fixture fault injection
|
||||
reason: Codex app-server intentionally owns apply_patch natively; this fixture still needs valid patch-shaped fault injection before it can prove product behavior.
|
||||
knownHarnessGap:
|
||||
issue: "#80320"
|
||||
reason: Codex-native apply_patch is intentionally not an OpenClaw dynamic tool; QA fault injection still uses synthetic failure-path inputs.
|
||||
action: require linked successful and workspace-denied patch execution
|
||||
reason: Codex owns workspace patching natively while private QA may expose an OpenClaw dynamic apply_patch tool.
|
||||
happyPrompt: >-
|
||||
tool search qa check target=apply_patch. Call apply_patch exactly once
|
||||
with a complete patch that adds runtime-tool-fixture-patch.txt containing
|
||||
runtime patch, then summarize the actual tool result.
|
||||
failurePrompt: >-
|
||||
tool search qa failure target=apply_patch. Call apply_patch exactly once
|
||||
with a complete patch that attempts to update
|
||||
../runtime-tool-fixture-denied.txt by replacing its existing
|
||||
runtime-tool-fixture-denied-original line with
|
||||
runtime patch outside the workspace, then summarize the actual
|
||||
workspace-escape rejection.
|
||||
promptSnippet: "target=apply_patch"
|
||||
failurePromptSnippet: "failure target=apply_patch"
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
} from "./agent-tools.before-tool-call.js";
|
||||
import { normalizeToolParameters } from "./agent-tools.schema.js";
|
||||
import type { AnyAgentTool } from "./agent-tools.types.js";
|
||||
import { execSchema } from "./bash-tools.schemas.js";
|
||||
import {
|
||||
BEFORE_TOOL_CALL_HOOK_CONTEXT,
|
||||
BEFORE_TOOL_CALL_SOURCE_TOOL,
|
||||
@@ -34,6 +35,26 @@ const TEST_USAGE = {
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
};
|
||||
|
||||
describe("direct exec tool schema", () => {
|
||||
it("keeps model-facing descriptions compact without hiding runtime constraints", () => {
|
||||
const fields = execSchema.properties as Record<string, { description?: string }>;
|
||||
const describeField = (name: string) => fields[name]?.description ?? "";
|
||||
const descriptions = Object.values(fields).map((field) => field.description ?? "");
|
||||
|
||||
expect(descriptions.join("").length).toBeLessThan(550);
|
||||
expect(describeField("workdir")).toContain("Blank/whitespace");
|
||||
expect(describeField("yieldMs")).toContain("Milliseconds");
|
||||
expect(describeField("timeout")).toContain("seconds");
|
||||
expect(describeField("pty")).toContain("PTY");
|
||||
expect(describeField("elevated")).toContain("if allowed");
|
||||
expect(describeField("security")).toContain("tools.exec.security");
|
||||
expect(describeField("security")).toContain("host approvals");
|
||||
expect(describeField("ask")).toContain("tools.exec.ask");
|
||||
expect(describeField("ask")).toContain("channel-origin");
|
||||
expect(describeField("ask")).toContain("ask=off");
|
||||
});
|
||||
});
|
||||
|
||||
describe("normalizeToolParameterSchema", () => {
|
||||
it("reuses normalized schemas for the same schema object and provider options", () => {
|
||||
const schema = {
|
||||
|
||||
@@ -14,31 +14,29 @@ export const execSchema = Type.Object({
|
||||
command: Type.String({ description: "Shell command to execute" }),
|
||||
workdir: Type.Optional(
|
||||
Type.String({
|
||||
description:
|
||||
"Working directory. Blank/whitespace values are invalid; omit to use the default cwd.",
|
||||
description: "Working directory; omit for default. Blank/whitespace is invalid.",
|
||||
}),
|
||||
),
|
||||
env: Type.Optional(Type.Record(Type.String(), Type.String())),
|
||||
yieldMs: Type.Optional(
|
||||
Type.Number({
|
||||
description: "Milliseconds to wait before backgrounding (default 10000)",
|
||||
description: "Milliseconds before backgrounding; default 10000.",
|
||||
}),
|
||||
),
|
||||
background: Type.Optional(Type.Boolean({ description: "Run in background immediately" })),
|
||||
timeout: Type.Optional(
|
||||
Type.Number({
|
||||
description: "Timeout in seconds (optional, kills process on expiry)",
|
||||
description: "Timeout in seconds; kills process on expiry.",
|
||||
}),
|
||||
),
|
||||
pty: Type.Optional(
|
||||
Type.Boolean({
|
||||
description:
|
||||
"Run in a pseudo-terminal (PTY) when available (TTY-required CLIs, coding agents)",
|
||||
description: "Use PTY for TTY-required CLIs and coding agents.",
|
||||
}),
|
||||
),
|
||||
elevated: Type.Optional(
|
||||
Type.Boolean({
|
||||
description: "Run on the host with elevated permissions (if allowed)",
|
||||
description: "Run on host with elevated permissions if allowed.",
|
||||
}),
|
||||
),
|
||||
host: optionalStringEnum(EXEC_TOOL_HOST_VALUES, {
|
||||
@@ -46,14 +44,13 @@ export const execSchema = Type.Object({
|
||||
}),
|
||||
security: Type.Optional(
|
||||
Type.String({
|
||||
description:
|
||||
"Ignored for normal calls; exec security is set by tools.exec.security and host approvals.",
|
||||
description: "Ignored per call; tools.exec.security and host approvals decide.",
|
||||
}),
|
||||
),
|
||||
ask: Type.Optional(
|
||||
Type.String({
|
||||
description:
|
||||
"Baseline ask comes from tools.exec.ask and host approvals; channel-origin calls ignore per-call ask when effective host ask is off.",
|
||||
"Uses tools.exec.ask and host approvals; channel-origin calls cannot override host ask=off.",
|
||||
}),
|
||||
),
|
||||
node: Type.Optional(
|
||||
|
||||
@@ -460,6 +460,31 @@ describe("Code Mode", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps code-mode exec guidance compact without advertising unavailable namespaces", () => {
|
||||
const { config, catalogRef, tools } = createCodeModeHarness();
|
||||
const compacted = applyCodeModeCatalog({
|
||||
tools: [...tools, pluginTool("fake_noop", "Noop")],
|
||||
config,
|
||||
sessionId: "session-code-mode",
|
||||
sessionKey: "agent:main:main",
|
||||
runId: "run-code-mode",
|
||||
catalogRef,
|
||||
});
|
||||
|
||||
const execTool = expectDefined(compacted.tools[0], "exec tool test invariant");
|
||||
const parameters = execTool.parameters as {
|
||||
properties?: Record<string, Record<string, unknown>>;
|
||||
};
|
||||
const codeDescription = parameters.properties?.code?.description;
|
||||
|
||||
expect(execTool.description.length).toBeLessThan(2_400);
|
||||
expect(execTool.description).toContain("parallelize independent work only");
|
||||
expect(codeDescription).toEqual(expect.any(String));
|
||||
expect(String(codeDescription).length).toBeLessThan(320);
|
||||
expect(codeDescription).not.toContain("MCP namespace globals");
|
||||
expect(codeDescription).not.toContain("`API` virtual declaration files");
|
||||
});
|
||||
|
||||
it("primes the exec schema with exact native tool ids and compact contracts", () => {
|
||||
const { config, catalogRef, tools } = createCodeModeHarness();
|
||||
const alpha = pluginTool("alpha_tool", "Another deferred description.");
|
||||
|
||||
@@ -77,10 +77,10 @@ function renderCodeModeCatalogIndex(lines: readonly string[], total: number): st
|
||||
? `${omitted} additional OpenClaw/plugin tools omitted from this prompt index. Use ALL_TOOLS or tools.search inside exec to find them.`
|
||||
: "Use these exact ids with tools.callValue; use ALL_TOOLS or tools.search inside exec when lookup is ambiguous.";
|
||||
return [
|
||||
"OpenClaw/plugin tool quick index (exact ids plus compact input and declared output hints; descriptions are intentionally deferred):",
|
||||
"Each line is `id input -> output`; `-> ?` means the output shape is unknown.",
|
||||
"OUTPUT DECLARED RULE: use the named fields in the first exec; keep dependent reads, checks, and follow-up calls in that exec instead of returning a raw value only to inspect an already-declared shape.",
|
||||
"OUTPUT UNKNOWN RULE: when the needed tool is `-> ?`, including a final dependent call after declared-output calls, return that tool's raw value unchanged. Do not wrap it in the requested answer shape or read guessed fields; filter or map only in a later exec after observing its shape.",
|
||||
"OpenClaw/plugin tool quick index (exact ids; descriptions are intentionally deferred):",
|
||||
"Each line is `id input -> output`; `-> ?` means unknown.",
|
||||
"OUTPUT DECLARED RULE: use declared fields for dependent calls in the first exec.",
|
||||
"OUTPUT UNKNOWN RULE: return the raw tool value unchanged; inspect or map it only in a later exec.",
|
||||
...lines,
|
||||
"",
|
||||
footer,
|
||||
@@ -146,7 +146,7 @@ function createCodeModeExecDescription(
|
||||
: "";
|
||||
const catalogIndex = catalog ? formatCodeModeCatalogIndex(catalog) : "";
|
||||
return (
|
||||
"Run JavaScript or TypeScript in OpenClaw code mode. Use `return` to pass the final value back to the agent; awaited calls without a returned value complete as `null`. Quick-index arrows show trusted declared output hints; `-> ?` means never guess result field names. When the needed tool has an unknown output, including a final dependent call after declared-output calls, the first exec must return the raw tool value unchanged with `return await tools.callValue(id, args);`; do not wrap it in the requested answer shape or read guessed fields; filter or map it only in a later exec after observing its shape. When the arrow declares the fields you need, select, call, and process them in the first exec; do not spend another exec inspecting that declared shape. Within that exec, perform dependent reads, checks, and follow-up calls in order; nested calls still enforce normal tool policy and approvals. Parallelize only independent work. `ALL_TOOLS` is the complete compact catalog with exact ids, input hints, and declared output hints. Select from it directly when practical, use `tools.search(query: string, options?)` when lookup is ambiguous, and use `tools.describe(id: string)` only when the compact input hint is insufficient. Never invent or transform a tool id. `tools.callValue(id: string, args?)` executes a tool and returns its JSON value directly; `tools.call(id: string, args?)` preserves the raw `{ tool, result }` envelope. Example: `const hit = ALL_TOOLS.find((entry) => entry.description.includes('weather')) ?? (await tools.search('weather'))[0]; return await tools.callValue(hit.id, {});`. Node.js modules and `require`/`import` are NOT available; for any shell, file, network, or external action, use enabled catalog tools allowed by policy from inside your code." +
|
||||
"Run JavaScript or TypeScript in OpenClaw code mode. Use `return` to pass the final value back; otherwise the result is `null`. Quick-index arrows show trusted declared output hints; `-> ?` means never guess result field names. For declared fields, process them in the first exec; do not spend another exec inspecting them. Perform dependent reads, checks, and follow-up calls in order; parallelize independent work only. For an unknown output, including a final dependent call after declared-output calls, return the raw tool value unchanged; do not wrap it in the requested answer shape or guess fields; filter or map it only in a later exec. Nested calls enforce normal tool policy and approvals. `ALL_TOOLS` is the complete compact catalog. Select exact ids directly or with `tools.search(query: string, options?)`; use `tools.describe(id: string)` only when needed. Never invent or transform a tool id. `tools.callValue(id: string, args?)` returns its JSON value directly; `tools.call(id: string, args?)` preserves `{ tool, result }`. Example: `const hit = ALL_TOOLS.find((entry) => entry.description.includes('weather')) ?? (await tools.search('weather'))[0]; return await tools.callValue(hit.id, {});`. Node.js modules and `require`/`import` are NOT available; use enabled catalog tools allowed by policy for shell, file, network, or external actions." +
|
||||
apiGuidance +
|
||||
mcpGuidance +
|
||||
swarmGuidance +
|
||||
@@ -167,7 +167,7 @@ export function createCodeModeTools(ctx: CodeModeToolContext): AnyAgentTool[] {
|
||||
code: Type.Optional(
|
||||
Type.String({
|
||||
description:
|
||||
"JavaScript or TypeScript source for one complete workflow. Select exact ids from `ALL_TOOLS` or `tools.search`; never invent ids. `tools.search` takes a query string, not an object. Keep dependent operations in this program, never put dependent calls in Promise.all, and return the final value. `API` virtual declaration files and MCP namespace globals are also available in scope; Node built-in modules are not.",
|
||||
"JavaScript or TypeScript for one complete workflow. Select exact ids from `ALL_TOOLS` or `tools.search`; never invent ids. `tools.search` takes a query string, not an object. Keep dependent calls in order; never put dependent calls in Promise.all. Return the final value. Node built-in modules are not available.",
|
||||
}),
|
||||
),
|
||||
command: Type.Optional(
|
||||
|
||||
@@ -4524,6 +4524,8 @@ wait_for_run plugin-clawhub-new.yml 123 "${expectedSha}" || status=$?
|
||||
".github/release/clawhub-cli/package-lock.json",
|
||||
".gitignore",
|
||||
"apps/android/.gitignore",
|
||||
"docs/reference/templates/IDENTITY.md",
|
||||
"docs/reference/templates/USER.md",
|
||||
"extensions/qa-lab/src/mantis/cli.ts",
|
||||
]) {
|
||||
const result = spawnSync("git", ["check-ignore", "--no-index", path], {
|
||||
|
||||
@@ -20,6 +20,8 @@ function createExtensionCodexAppServerAttemptExtraVitestConfig(
|
||||
{
|
||||
dir: "extensions",
|
||||
env,
|
||||
// Prewarm is owned by the light attempt shard, including narrowed runs.
|
||||
exclude: ["extensions/codex/src/app-server/run-attempt-client-prewarm.test.ts"],
|
||||
fileParallelism: false,
|
||||
name: "extension-codex-app-server-attempt-extra",
|
||||
passWithNoTests: true,
|
||||
|
||||
Reference in New Issue
Block a user