From f14947d256d8238ed2fce3ca3ec81e4dae3fe8c2 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 10:37:25 +0100 Subject: [PATCH] fix(codex): persist automatic compaction history (#110587) * fix(codex): persist automatic compaction counts * test(codex): stress compaction and restart recovery * test(codex): bind compaction waves to fresh counts * chore: keep Codex release note in PR --- docs/help/testing-live.md | 22 + .../src/app-server/event-projector.test.ts | 5 +- .../codex/src/app-server/event-projector.ts | 6 +- .../codex/src/app-server/run-attempt-start.ts | 7 +- scripts/test-live-codex-harness-docker.sh | 12 + ...gateway-codex-harness.live-helpers.test.ts | 25 + .../gateway-codex-harness.live-helpers.ts | 24 + .../gateway-codex-harness.live.test.ts | 477 +++++++++++++++--- .../test-live-codex-harness-docker.test.ts | 18 + 9 files changed, 510 insertions(+), 86 deletions(-) diff --git a/docs/help/testing-live.md b/docs/help/testing-live.md index c4ab7b437911..af947358fdb1 100644 --- a/docs/help/testing-live.md +++ b/docs/help/testing-live.md @@ -334,6 +334,17 @@ Docker notes: history. Override the bounded counts with `OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS` (1-20) and `OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS` (1-10). +- Optional fan-out stress: set `OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE=1` + and `OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT` (1-12). The harness starts + every child concurrently, waits for every terminal run, and verifies each + unique child reply and native thread identity. +- Optional compaction stress: `OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS=1` + generates bounded native tool output, requires automatic compaction events, + verifies the persisted compaction count and hidden-marker recall, restarts + the Gateway and physical Codex app-server, then repeats the output and + compaction wave. Tune the bounded work with + `OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS` (1-8) and + `OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES` (100000-1000000). - Optional loop-relay opt-out probe: `OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY=1` - The requested thinking preference may map to the nearest effort advertised @@ -367,6 +378,17 @@ Restart and history stress: ```bash OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS=1 \ +pnpm test:docker:live-codex-harness +``` + +Fan-out, large-output, compaction, and restart stress: + +```bash +OPENCLAW_LIVE_CODEX_HARNESS_AUTH=api-key \ + OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE=1 \ + OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT=8 \ + OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS=1 \ + OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS=1 \ pnpm test:docker:live-codex-harness ``` diff --git a/extensions/codex/src/app-server/event-projector.test.ts b/extensions/codex/src/app-server/event-projector.test.ts index 3e8ae5f61661..03856fcbcc20 100644 --- a/extensions/codex/src/app-server/event-projector.test.ts +++ b/extensions/codex/src/app-server/event-projector.test.ts @@ -2998,7 +2998,10 @@ describe("CodexAppServerEventProjector", () => { expect(JSON.stringify(result.messagesSnapshot[2])).toContain("Codex plan"); expect(JSON.stringify(result.messagesSnapshot[2])).toContain("next"); expect(JSON.stringify(result.messagesSnapshot[2])).toContain("[in_progress] patch"); - expect(requireRecord(result.itemLifecycle, "item lifecycle").compactionCount).toBe(1); + expect(result.compactionCount).toBe(1); + expect(requireRecord(result.itemLifecycle, "item lifecycle")).not.toHaveProperty( + "compactionCount", + ); expect(onContextCompacted).toHaveBeenCalledOnce(); }); diff --git a/extensions/codex/src/app-server/event-projector.ts b/extensions/codex/src/app-server/event-projector.ts index d894f3d89d78..8feada7e4b42 100644 --- a/extensions/codex/src/app-server/event-projector.ts +++ b/extensions/codex/src/app-server/event-projector.ts @@ -434,6 +434,9 @@ export class CodexAppServerEventProjector { successfulCronAdds: toolTelemetry.successfulCronAdds, cloudCodeAssistFormatError: false, attemptUsage: projectedUsage, + ...(this.completedCompactionCount > 0 + ? { compactionCount: this.completedCompactionCount } + : {}), replayMetadata: { hadPotentialSideEffects, replaySafe: !hadPotentialSideEffects, @@ -442,9 +445,6 @@ export class CodexAppServerEventProjector { startedCount: this.activeItemIds.size + this.completedItemIds.size, completedCount: this.completedItemIds.size, activeCount: this.activeItemIds.size, - ...(this.completedCompactionCount > 0 - ? { compactionCount: this.completedCompactionCount } - : {}), }, yieldDetected: options?.yieldDetected || false, didSendDeterministicApprovalPrompt: diff --git a/extensions/codex/src/app-server/run-attempt-start.ts b/extensions/codex/src/app-server/run-attempt-start.ts index 872c310ab3b5..be79f0d33b43 100644 --- a/extensions/codex/src/app-server/run-attempt-start.ts +++ b/extensions/codex/src/app-server/run-attempt-start.ts @@ -159,7 +159,12 @@ export async function startCodexAttemptRuntime(resources: CodexAttemptResources) state.codexSandboxPolicy = startupResult.sandboxPolicy; void emitCodexAppServerEvent(params, { stream: "codex_app_server.lifecycle", - data: { phase: "thread_ready", threadId: state.thread.threadId }, + data: { + phase: "thread_ready", + threadId: state.thread.threadId, + action: state.thread.lifecycle.action, + clientId: state.client.getInstanceId(), + }, }); } catch (error) { activateNativePreToolUseFailureFallback(); diff --git a/scripts/test-live-codex-harness-docker.sh b/scripts/test-live-codex-harness-docker.sh index 0f06a4ea7ca4..b8f5cfe4a8c8 100644 --- a/scripts/test-live-codex-harness-docker.sh +++ b/scripts/test-live-codex-harness-docker.sh @@ -367,11 +367,17 @@ echo "==> Chat image probe: ${OPENCLAW_LIVE_CODEX_HARNESS_CHAT_IMAGE_PROBE:-0}" echo "==> Image probe: ${OPENCLAW_LIVE_CODEX_HARNESS_IMAGE_PROBE:-1}" echo "==> MCP probe: ${OPENCLAW_LIVE_CODEX_HARNESS_MCP_PROBE:-1}" echo "==> Subagent probe: ${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE:-1}" +echo "==> Subagent count: ${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT:-1}" echo "==> Subagent-only fast path: ${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY:-auto}" echo "==> Guardian probe: ${OPENCLAW_LIVE_CODEX_HARNESS_GUARDIAN_PROBE:-1}" +echo "==> Code-mode-only probe: ${OPENCLAW_LIVE_CODEX_HARNESS_CODE_MODE_ONLY:-0}" +echo "==> Loop relay disabled: ${OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY:-0}" echo "==> Resume stress: ${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS:-0}" echo "==> Resume stress history turns: ${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS:-4}" echo "==> Resume stress restarts: ${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS:-3}" +echo "==> Compaction stress: ${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS:-0}" +echo "==> Compaction stress turns: ${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS:-4}" +echo "==> Large output bytes: ${OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES:-300000}" echo "==> Auth mode: $CODEX_HARNESS_AUTH_MODE" echo "==> Profile file: $PROFILE_STATUS" echo "==> CI-safe Codex config: ${OPENCLAW_LIVE_CODEX_HARNESS_USE_CI_SAFE_CODEX_CONFIG:-1}" @@ -401,9 +407,14 @@ DOCKER_RUN_ARGS+=(--rm -t \ -e OPENCLAW_LIVE_CODEX_HARNESS_AUTH="$CODEX_HARNESS_AUTH_MODE" \ -e OPENCLAW_LIVE_CODEX_HARNESS=1 \ -e OPENCLAW_LIVE_CODEX_HARNESS_CHAT_IMAGE_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_CHAT_IMAGE_PROBE:-0}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_CODE_MODE_ONLY="${OPENCLAW_LIVE_CODEX_HARNESS_CODE_MODE_ONLY:-0}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS="${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS:-0}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS="${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS:-4}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_DEBUG="${OPENCLAW_LIVE_CODEX_HARNESS_DEBUG:-}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY="${OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY:-0}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_GUARDIAN_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_GUARDIAN_PROBE:-1}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_IMAGE_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_IMAGE_PROBE:-1}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES="${OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES:-300000}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_MCP_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_MCP_PROBE:-1}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_MODEL="${OPENCLAW_LIVE_CODEX_HARNESS_MODEL:-openai/gpt-5.6-luna}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_TARGETS="${OPENCLAW_LIVE_CODEX_HARNESS_TARGETS:-}" \ @@ -416,6 +427,7 @@ DOCKER_RUN_ARGS+=(--rm -t \ -e OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS="${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS:-3}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_SETUP_TIMEOUT_SECONDS="$CODEX_HARNESS_SETUP_TIMEOUT_SECONDS" \ -e OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY="${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY:-}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT="${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT:-1}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE:-1}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_USE_CI_SAFE_CODEX_CONFIG="${OPENCLAW_LIVE_CODEX_HARNESS_USE_CI_SAFE_CODEX_CONFIG:-1}" \ -e OPENCLAW_LIVE_CODEX_CLI_PACKAGE_SPEC="$CODEX_CLI_PACKAGE_SPEC" \ diff --git a/src/gateway/gateway-codex-harness.live-helpers.test.ts b/src/gateway/gateway-codex-harness.live-helpers.test.ts index b2aa0136758f..f173074350e6 100644 --- a/src/gateway/gateway-codex-harness.live-helpers.test.ts +++ b/src/gateway/gateway-codex-harness.live-helpers.test.ts @@ -10,6 +10,7 @@ import { isExpectedYieldedAgentTimeout, isRetryableCodexHarnessLiveError, isStrictExpectedCodexModelsCommandText, + shouldUseCodexHarnessSubagentOnlyFastPath, } from "./gateway-codex-harness.live-helpers.js"; const includesExpectedCodexModelsCommandText = (text: string) => @@ -30,6 +31,30 @@ function expectStrictCodexModelsCommandText(text: string): void { } describe("gateway codex harness live helpers", () => { + it("keeps combined stress probes out of the subagent-only fast path", () => { + const base = { + chatImageProbe: false, + codeModeOnly: false, + compactionStress: false, + explicitOptOut: false, + guardianProbe: false, + imageProbe: false, + mcpProbe: false, + resumeStress: false, + subagentProbe: true, + }; + + expect(shouldUseCodexHarnessSubagentOnlyFastPath(base)).toBe(true); + expect(shouldUseCodexHarnessSubagentOnlyFastPath({ ...base, resumeStress: true })).toBe(false); + expect(shouldUseCodexHarnessSubagentOnlyFastPath({ ...base, compactionStress: true })).toBe( + false, + ); + expect(shouldUseCodexHarnessSubagentOnlyFastPath({ ...base, codeModeOnly: true })).toBe(false); + expect(shouldUseCodexHarnessSubagentOnlyFastPath({ ...base, explicitOptOut: true })).toBe( + false, + ); + }); + it("classifies sessions.list timeouts as retryable live Codex errors", () => { const error = new Error("gateway request timeout for sessions.list"); diff --git a/src/gateway/gateway-codex-harness.live-helpers.ts b/src/gateway/gateway-codex-harness.live-helpers.ts index 9b8c83b40146..2fec444a1046 100644 --- a/src/gateway/gateway-codex-harness.live-helpers.ts +++ b/src/gateway/gateway-codex-harness.live-helpers.ts @@ -81,6 +81,30 @@ export const EXPECTED_CODEX_MODELS_COMMAND_TEXT = [ "Current OpenClaw session status reports the active model as:", ] as const; +export function shouldUseCodexHarnessSubagentOnlyFastPath(params: { + chatImageProbe: boolean; + codeModeOnly: boolean; + compactionStress: boolean; + explicitOptOut: boolean; + guardianProbe: boolean; + imageProbe: boolean; + mcpProbe: boolean; + resumeStress: boolean; + subagentProbe: boolean; +}): boolean { + return ( + params.subagentProbe && + !params.chatImageProbe && + !params.codeModeOnly && + !params.compactionStress && + !params.guardianProbe && + !params.imageProbe && + !params.mcpProbe && + !params.resumeStress && + !params.explicitOptOut + ); +} + const HEALTHY_CODEX_MODELS_COMMAND_TEXT = [ "Codex models:", "Available Codex models", diff --git a/src/gateway/gateway-codex-harness.live.test.ts b/src/gateway/gateway-codex-harness.live.test.ts index a781904dbe1c..cf948c623cce 100644 --- a/src/gateway/gateway-codex-harness.live.test.ts +++ b/src/gateway/gateway-codex-harness.live.test.ts @@ -30,6 +30,7 @@ import { isExpectedYieldedAgentTimeout, isRetryableCodexHarnessLiveError, isStrictExpectedCodexModelsCommandText, + shouldUseCodexHarnessSubagentOnlyFastPath, } from "./gateway-codex-harness.live-helpers.js"; import { assertCronJobMatches, @@ -63,13 +64,6 @@ const CODEX_HARNESS_CODE_MODE_ONLY = isTruthyEnvValue( const CODEX_HARNESS_DISABLE_LOOP_RELAY = isTruthyEnvValue( process.env.OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY, ); -const CODEX_HARNESS_SUBAGENT_ONLY = - CODEX_HARNESS_SUBAGENT_PROBE && - !CODEX_HARNESS_CHAT_IMAGE_PROBE && - !CODEX_HARNESS_IMAGE_PROBE && - !CODEX_HARNESS_MCP_PROBE && - !CODEX_HARNESS_GUARDIAN_PROBE && - process.env.OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY !== "0"; const CODEX_HARNESS_REQUIRE_GUARDIAN_EVENTS = isTruthyEnvValue( process.env.OPENCLAW_LIVE_CODEX_HARNESS_REQUIRE_GUARDIAN_EVENTS, ); @@ -88,6 +82,40 @@ const CODEX_HARNESS_RESUME_STRESS_RESTARTS = resolveBoundedPositiveIntEnv( 3, 10, ); +const CODEX_HARNESS_COMPACTION_STRESS = isTruthyEnvValue( + process.env.OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS, +); +const CODEX_HARNESS_COMPACTION_STRESS_TURNS = resolveBoundedPositiveIntEnv( + "OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS", + process.env.OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS, + 4, + 8, +); +const CODEX_HARNESS_LARGE_OUTPUT_BYTES = resolveBoundedPositiveIntEnv( + "OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES", + process.env.OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES, + 300_000, + 1_000_000, + 100_000, +); +const CODEX_HARNESS_SUBAGENT_COUNT = resolveBoundedPositiveIntEnv( + "OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT", + process.env.OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT, + 1, + 12, +); +const CODEX_HARNESS_SUBAGENT_ONLY = shouldUseCodexHarnessSubagentOnlyFastPath({ + chatImageProbe: CODEX_HARNESS_CHAT_IMAGE_PROBE, + codeModeOnly: CODEX_HARNESS_CODE_MODE_ONLY, + compactionStress: CODEX_HARNESS_COMPACTION_STRESS, + explicitOptOut: process.env.OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY === "0", + guardianProbe: CODEX_HARNESS_GUARDIAN_PROBE, + imageProbe: CODEX_HARNESS_IMAGE_PROBE, + mcpProbe: CODEX_HARNESS_MCP_PROBE, + resumeStress: CODEX_HARNESS_RESUME_STRESS, + subagentProbe: CODEX_HARNESS_SUBAGENT_PROBE, +}); +const CODEX_HARNESS_RESTART_STRESS = CODEX_HARNESS_RESUME_STRESS || CODEX_HARNESS_COMPACTION_STRESS; const CODEX_HARNESS_REQUEST_TIMEOUT_MS = resolveLiveTimeoutMs( process.env.OPENCLAW_LIVE_CODEX_HARNESS_REQUEST_TIMEOUT_MS, 300_000, @@ -103,7 +131,7 @@ const CODEX_HARNESS_THINKING = resolveCodexHarnessThinkingLevel( ); const describeLive = LIVE && CODEX_HARNESS_LIVE ? describe : describe.skip; const describeDisabled = LIVE && !CODEX_HARNESS_LIVE ? describe : describe.skip; -const CODEX_HARNESS_TIMEOUT_MS = CODEX_HARNESS_RESUME_STRESS ? 1_800_000 : 900_000; +const CODEX_HARNESS_TIMEOUT_MS = CODEX_HARNESS_RESTART_STRESS ? 3_600_000 : 900_000; const DEFAULT_CODEX_MODEL = "openai/gpt-5.6-luna"; const GATEWAY_CONNECT_TIMEOUT_MS = 60_000; const CODEX_HARNESS_REASONING_EFFORTS = [ @@ -132,6 +160,8 @@ type CapturedAgentEvent = { }; const observedCodexThreadIds = new Map(); +const observedCodexClientIds = new Map(); +const observedCodexThreadActions = new Map(); type GuardianPluginApprovalDecision = "allow-once" | "deny"; type CodexHarnessThinkingLevel = @@ -154,6 +184,7 @@ function resolveBoundedPositiveIntEnv( raw: string | undefined, fallback: number, max: number, + min = 1, ): number { if (raw === undefined || raw.trim() === "") { return fallback; @@ -162,8 +193,8 @@ function resolveBoundedPositiveIntEnv( throw new Error(`invalid ${name}: ${raw}`); } const parsed = Number(raw); - if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > max) { - throw new Error(`${name} must be between 1 and ${max}`); + if (!Number.isSafeInteger(parsed) || parsed < min || parsed > max) { + throw new Error(`${name} must be between ${min} and ${max}`); } return parsed; } @@ -392,6 +423,7 @@ async function assertCodexHarnessTranscriptModelIdentity(params: { async function writeLiveGatewayConfig(params: { codexAppServerMode?: "guardian" | "yolo"; codeModeOnly?: boolean; + compactionStress?: boolean; loopDetectionPreToolUseRelay?: boolean; configPath: string; modelKey: string; @@ -414,6 +446,23 @@ async function writeLiveGatewayConfig(params: { config: { appServer: { mode: params.codexAppServerMode ?? "yolo", + ...(params.compactionStress + ? { + args: [ + "-c", + "model_auto_compact_token_limit_scope=body_after_prefix", + "-c", + // One truncated 300 KB tool result is only a few thousand + // tokens after Codex applies its transcript output cap. + "model_auto_compact_token_limit=4000", + "-c", + "tool_output_token_limit=10000", + "app-server", + "--listen", + "stdio://", + ], + } + : {}), ...(params.codeModeOnly === true ? { codeModeOnly: true } : {}), ...(params.loopDetectionPreToolUseRelay === false ? { loopDetectionPreToolUseRelay: false } @@ -430,6 +479,11 @@ async function writeLiveGatewayConfig(params: { workspace: params.workspace, skipBootstrap: true, timeoutSeconds: CODEX_HARNESS_AGENT_TIMEOUT_SECONDS, + maxConcurrent: Math.max(4, CODEX_HARNESS_SUBAGENT_COUNT + 1), + subagents: { + maxConcurrent: CODEX_HARNESS_SUBAGENT_COUNT, + maxChildrenPerAgent: CODEX_HARNESS_SUBAGENT_COUNT, + }, thinkingDefault: CODEX_HARNESS_THINKING, model: { primary: params.modelKey }, models: { [params.modelKey]: { agentRuntime: { id: "codex" } } }, @@ -454,17 +508,18 @@ async function requestAgentTextWithEvents(params: { acceptYieldedTimeout?: boolean; client: GatewayClient; eventPrefix?: string; + eventPrefixes?: readonly string[]; includeAllSessions?: boolean; message: string; sessionKey: string; -}): Promise<{ text: string; events: CapturedAgentEvent[] }> { +}): Promise<{ text: string; events: CapturedAgentEvent[]; compactionCount: number }> { const { extractPayloadText } = await import("./test-helpers.agent-results.js"); const { onAgentEvent } = await import("../infra/agent-events.js"); const events: CapturedAgentEvent[] = []; - const eventPrefix = params.eventPrefix ?? "codex_app_server.guardian"; + const eventPrefixes = params.eventPrefixes ?? [params.eventPrefix ?? "codex_app_server.guardian"]; const unsubscribe = onAgentEvent((event) => { if ( - !event.stream.startsWith(eventPrefix) || + !eventPrefixes.some((prefix) => event.stream.startsWith(prefix)) || (!params.includeAllSessions && event.sessionKey && event.sessionKey !== params.sessionKey) ) { return; @@ -493,7 +548,14 @@ async function requestAgentTextWithEvents(params: { if (payload?.status !== "ok" && !acceptedYieldedTimeout) { throw new Error(`agent status=${String(payload?.status)} payload=${JSON.stringify(payload)}`); } - return { text: extractPayloadText(payload.result), events }; + const result = payload.result as + | { meta?: { agentMeta?: { compactionCount?: number } } } + | undefined; + return { + text: extractPayloadText(payload.result), + events, + compactionCount: Math.max(0, result?.meta?.agentMeta?.compactionCount ?? 0), + }; } finally { unsubscribe(); } @@ -501,7 +563,7 @@ async function requestAgentTextWithEvents(params: { async function requestAgentText(params: { client: GatewayClient; - expectedToken: string; + expectedReply: string; message: string; sessionKey: string; }): Promise { @@ -511,7 +573,16 @@ async function requestAgentText(params: { message: params.message, sessionKey: params.sessionKey, }); - expect(text).toContain(params.expectedToken); + expect(text).toContain(params.expectedReply); + recordCodexAttemptIdentity({ events, sessionKey: params.sessionKey }); + return text; +} + +function recordCodexAttemptIdentity(params: { + events: CapturedAgentEvent[]; + sessionKey: string; +}): void { + const { events } = params; const turnStarting = events.find( (event) => event.stream === "codex_app_server.lifecycle" && event.data?.phase === "turn_starting", @@ -539,7 +610,15 @@ async function requestAgentText(params: { `expected Codex thread_ready identity for ${params.sessionKey}; events=${JSON.stringify(events)}`, ).toBe(true); observedCodexThreadIds.set(params.sessionKey, threadId as string); - return text; + const clientId = threadReady?.data?.clientId; + expect( + typeof clientId === "string" && clientId.trim().length > 0, + `expected Codex client identity for ${params.sessionKey}; events=${JSON.stringify(events)}`, + ).toBe(true); + observedCodexClientIds.set(params.sessionKey, clientId as string); + const action = threadReady?.data?.action; + expect(["started", "resumed", "forked"]).toContain(action); + observedCodexThreadActions.set(params.sessionKey, action as string); } async function verifyCodexCodeModeOnlyDynamicToolProbe(params: { @@ -726,6 +805,127 @@ function formatAssistantTextPreview(texts: string[], maxChars = 800): string { return combined.length > maxChars ? `${combined.slice(0, maxChars)}...` : combined; } +async function readCodexHarnessCompactionCount(params: { + client: GatewayClient; + events: EventFrame[]; + minimum: number; + sessionKey: string; +}): Promise { + const statusText = await requestCodexCommandText({ + client: params.client, + command: "/status", + events: params.events, + expectedText: "Compactions:", + sessionKey: params.sessionKey, + }); + const count = Number(/Compactions:\s*(\d+)/u.exec(statusText)?.[1]); + expect( + count, + `session ${params.sessionKey} did not report ${params.minimum} persisted compactions`, + ).toBeGreaterThanOrEqual(params.minimum); + return count; +} + +async function verifyCodexCompactionStress(params: { + client: GatewayClient; + events: EventFrame[]; + sessionKey: string; +}): Promise<{ hiddenMarker: string; persistedCount: number }> { + const hiddenMarker = `CODEX-DURABLE-${randomBytes(6).toString("hex").toUpperCase()}`; + await requestAgentText({ + client: params.client, + sessionKey: params.sessionKey, + expectedReply: hiddenMarker, + message: [ + `Remember this as durable slot A: ${hiddenMarker}`, + `Reply exactly ${hiddenMarker} and nothing else.`, + ].join("\n"), + }); + const baselineCount = await readCodexHarnessCompactionCount({ + client: params.client, + events: params.events, + minimum: 0, + sessionKey: params.sessionKey, + }); + + const outputLines = Math.ceil(CODEX_HARNESS_LARGE_OUTPUT_BYTES / 90); + let completedCompactions = 0; + let reportedCompactions = 0; + for (let turn = 1; turn <= CODEX_HARNESS_COMPACTION_STRESS_TURNS; turn += 1) { + const acknowledgement = `CODEX-LARGE-OUTPUT-${turn}-${randomBytes(3) + .toString("hex") + .toUpperCase()}`; + const { text, events, compactionCount } = await requestAgentTextWithEvents({ + client: params.client, + eventPrefixes: ["codex_app_server.", "compaction"], + sessionKey: params.sessionKey, + message: [ + "Large-output compaction probe.", + "Use the native exec_command tool exactly once.", + `Run this exact command: node -e 'for(let i=0;i<${outputLines};i++){console.log(i.toString(36).padStart(8,"0")+"-"+((i*2654435761)>>>0).toString(16).padStart(8,"0")+"-ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789")}'`, + "Set max_output_tokens to 10000.", + `After the tool completes, reply exactly ${acknowledgement} and nothing else.`, + ].join("\n"), + }); + expect(text).toContain(acknowledgement); + recordCodexAttemptIdentity({ events, sessionKey: params.sessionKey }); + completedCompactions += events.filter( + (event) => + event.stream === "compaction" && + event.data?.phase === "end" && + event.data?.completed === true, + ).length; + reportedCompactions += compactionCount; + + const history: { messages?: unknown[] } = await params.client.request("chat.history", { + sessionKey: params.sessionKey, + limit: 100, + }); + const serialized = JSON.stringify(history.messages ?? []); + const originalLengths = Array.from(serialized.matchAll(/original (\d+) chars/gu), (match) => + Number(match[1]), + ); + expect( + originalLengths.some((length) => length > 10_000), + `expected a truncated large native tool result; lengths=${JSON.stringify(originalLengths)}`, + ).toBe(true); + } + + expect(completedCompactions, "expected at least one native automatic compaction").toBeGreaterThan( + 0, + ); + expect(reportedCompactions, "agent result dropped native automatic compactions").toBe( + completedCompactions, + ); + // `/status` stops in the local command handler (`shouldContinue: false`), so + // these snapshots cannot introduce an unobserved native Codex compaction. + const persistedCount = await readCodexHarnessCompactionCount({ + client: params.client, + events: params.events, + minimum: baselineCount + completedCompactions, + sessionKey: params.sessionKey, + }); + expect( + persistedCount - baselineCount, + "persisted session count did not match this wave's native compactions", + ).toBe(completedCompactions); + const recalled = await requestAgentText({ + client: params.client, + sessionKey: params.sessionKey, + expectedReply: hiddenMarker, + message: "Reply with exactly the value stored in durable slot A and nothing else.", + }); + expect(recalled.trim()).toBe(hiddenMarker); + logCodexLiveStep("compaction-stress:complete", { + baselineCount, + completedCompactions, + outputBytes: CODEX_HARNESS_LARGE_OUTPUT_BYTES, + outputTurns: CODEX_HARNESS_COMPACTION_STRESS_TURNS, + persistedCount, + }); + return { hiddenMarker, persistedCount }; +} + async function waitForAssistantText(params: { client: GatewayClient; sessionKey: string; @@ -930,7 +1130,6 @@ async function verifyCodexGuardianProbe(params: { const allowReview = assertGuardianReviewCompleted({ events: allowResult.events, label: "allow probe", - requireEvents: false, }); const allowStatus = findGuardianReviewStatus(allowResult.events); if (allowStatus === "denied") { @@ -977,7 +1176,6 @@ async function verifyCodexGuardianProbe(params: { const review = assertGuardianReviewCompleted({ events: deniedResult.events, label: "ask-back probe", - requireEvents: false, }); // The approve/deny call is Codex policy-owned and may change independently. // OpenClaw's strict projection contract is covered by the allow probe above. @@ -1067,24 +1265,21 @@ async function verifyCodexCronMcpProbe(params: { } } -function hasCodexAppServerLifecycleEvent(params: { - childSessionKey: string; - events: CapturedAgentEvent[]; -}): boolean { - return params.events.some( - (event) => - event.sessionKey === params.childSessionKey && event.stream === "codex_app_server.lifecycle", - ); -} - async function waitForCodexSubagentStarted(params: { childSessionKey: string; events: CapturedAgentEvent[]; -}): Promise { +}): Promise { const deadline = Date.now() + CODEX_HARNESS_REQUEST_TIMEOUT_MS; while (Date.now() < deadline) { - if (hasCodexAppServerLifecycleEvent(params)) { - return; + const threadReady = params.events.find( + (event) => + event.sessionKey === params.childSessionKey && + event.stream === "codex_app_server.lifecycle" && + event.data?.phase === "thread_ready" && + typeof event.data.threadId === "string", + ); + if (threadReady && typeof threadReady.data?.threadId === "string") { + return threadReady.data.threadId; } await delay(2_000); } @@ -1158,36 +1353,84 @@ async function verifyCodexSubagentProbe(params: { } }, }); - const spawnResult = await spawnSubagentDirect( - { - task: `Reply exactly ${expectedToken} and nothing else.`, - agentId: "dev", - thinking: CODEX_HARNESS_THINKING, - mode: "run", - cleanup: "keep", - context: "isolated", - lightContext: true, - expectsCompletionMessage: false, - runTimeoutSeconds: CODEX_HARNESS_AGENT_TIMEOUT_SECONDS, - }, - { - agentSessionKey: params.sessionKey, - }, + const probeReplies = [ + expectedToken, + ...Array.from( + { length: CODEX_HARNESS_SUBAGENT_COUNT - 1 }, + (_, index) => `CODEX-SUBAGENT-${index + 2}-${randomUUID().slice(0, 6).toUpperCase()}`, + ), + ]; + const probes = probeReplies.map((expectedReply, index) => ({ expectedReply, index })); + const spawnResults = await Promise.all( + probes.map(async (probe) => ({ + expectedReply: probe.expectedReply, + index: probe.index, + result: await spawnSubagentDirect( + { + task: `Reply exactly ${probe.expectedReply} and nothing else.`, + agentId: "dev", + thinking: CODEX_HARNESS_THINKING, + mode: "run", + cleanup: "keep", + context: "isolated", + lightContext: true, + expectsCompletionMessage: false, + runTimeoutSeconds: CODEX_HARNESS_AGENT_TIMEOUT_SECONDS, + }, + { + agentSessionKey: params.sessionKey, + }, + ), + })), ); - if (spawnResult.status !== "accepted") { - throw new Error( - `Codex subagent spawn failed: ${JSON.stringify(spawnResult)} trace=${JSON.stringify(gatewayTrace)}`, - ); + for (const probe of spawnResults) { + if (probe.result.status !== "accepted") { + throw new Error( + `Codex subagent ${probe.index + 1} spawn failed: ${JSON.stringify(probe.result)} trace=${JSON.stringify(gatewayTrace)}`, + ); + } + if (!probe.result.childSessionKey?.includes(":subagent:") || !probe.result.runId) { + throw new Error( + `subagent spawn did not return child/run identities: ${JSON.stringify(probe.result)}`, + ); + } } - const childSessionKey = spawnResult.childSessionKey; - if (!childSessionKey?.includes(":subagent:")) { - throw new Error( - `subagent spawn did not return a child session key: ${JSON.stringify(spawnResult)}`, - ); - } - await waitForCodexSubagentStarted({ - childSessionKey, - events, + expect(new Set(spawnResults.map((probe) => probe.result.childSessionKey)).size).toBe( + spawnResults.length, + ); + expect(new Set(spawnResults.map((probe) => probe.result.runId)).size).toBe(spawnResults.length); + + const threadIds = await Promise.all( + spawnResults.map((probe) => + waitForCodexSubagentStarted({ + childSessionKey: probe.result.childSessionKey as string, + events, + }), + ), + ); + expect(new Set(threadIds).size).toBe(threadIds.length); + + await Promise.all( + spawnResults.map((probe) => + waitForChatAgentRunOk(params.client, probe.result.runId as string), + ), + ); + await Promise.all( + spawnResults.map(async (probe) => { + const history: { messages?: unknown[] } = await params.client.request("chat.history", { + sessionKey: probe.result.childSessionKey, + limit: 20, + }); + const replies = extractAssistantTexts(history.messages ?? []); + expect( + replies.some((text) => text.trim() === probe.expectedReply), + `subagent ${probe.index + 1} missing exact reply ${probe.expectedReply}; replies=${formatAssistantTextPreview(replies)}`, + ).toBe(true); + }), + ); + logCodexLiveStep("subagent-fanout:complete", { + count: spawnResults.length, + uniqueThreads: new Set(threadIds).size, }); } finally { const { testing: subagentSpawnTesting } = @@ -1306,6 +1549,7 @@ describeLive("gateway live (Codex harness)", () => { workspace, codexAppServerMode: CODEX_HARNESS_GUARDIAN_PROBE ? "guardian" : "yolo", codeModeOnly: CODEX_HARNESS_CODE_MODE_ONLY, + compactionStress: CODEX_HARNESS_COMPACTION_STRESS, ...(CODEX_HARNESS_DISABLE_LOOP_RELAY ? { loopDetectionPreToolUseRelay: false } : {}), }); const deviceIdentity = await ensurePairedTestGatewayClientIdentity({ @@ -1318,7 +1562,14 @@ describeLive("gateway live (Codex harness)", () => { let guardianPluginApprovalDecision: GuardianPluginApprovalDecision | undefined; let activeApprovalClient: GatewayClient | undefined; let resumeStressState: - | { lastMarker: string; sessionKey: string; threadId: string } + | { + clientId: string; + hiddenMarker?: string; + lastMarker: string; + persistedCompactionCount: number; + sessionKey: string; + threadId: string; + } | undefined; const maybeResolveGuardianPluginApproval = (event: EventFrame): void => { const decision = guardianPluginApprovalDecision; @@ -1345,6 +1596,8 @@ describeLive("gateway live (Codex harness)", () => { }; logCodexLiveStep("config-written", { configPath, modelKey, port }); observedCodexThreadIds.clear(); + observedCodexClientIds.clear(); + observedCodexThreadActions.clear(); try { server = await startGatewayServer(port, { @@ -1411,7 +1664,7 @@ describeLive("gateway live (Codex harness)", () => { const firstText = await requestAgentText({ client: activeClient, sessionKey, - expectedToken: firstToken, + expectedReply: firstToken, message: `Reply with exactly ${firstToken} and nothing else.`, }); expect(firstText).toContain(firstToken); @@ -1422,7 +1675,7 @@ describeLive("gateway live (Codex harness)", () => { const secondText = await requestAgentText({ client: activeClient, sessionKey, - expectedToken: secondToken, + expectedReply: secondToken, message: `Reply with exactly ${secondToken} and nothing else. Do not repeat ${firstToken}.`, }); expect(secondText).toContain(secondToken); @@ -1529,31 +1782,51 @@ describeLive("gateway live (Codex harness)", () => { }); logCodexLiveStep("guardian-probe:done"); } - if (CODEX_HARNESS_RESUME_STRESS) { + const compactionStressState = CODEX_HARNESS_COMPACTION_STRESS + ? await verifyCodexCompactionStress({ + client: activeClient, + events: gatewayEvents, + sessionKey, + }) + : undefined; + if (CODEX_HARNESS_RESTART_STRESS) { const threadId = observedCodexThreadIds.get(sessionKey); if (!threadId) { throw new Error("Codex resume stress did not observe a thread identity"); } + const clientId = observedCodexClientIds.get(sessionKey); + if (!clientId) { + throw new Error("Codex resume stress did not observe a client identity"); + } let lastMarker = ""; - for ( - let historyTurn = 1; - historyTurn <= CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS; - historyTurn += 1 - ) { + const historyTurns = CODEX_HARNESS_RESUME_STRESS + ? CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS + : 1; + for (let historyTurn = 1; historyTurn <= historyTurns; historyTurn += 1) { lastMarker = `CODEX-HISTORY-${historyTurn}-${randomBytes(3) .toString("hex") .toUpperCase()}`; await requestAgentText({ client: activeClient, sessionKey, - expectedToken: lastMarker, - message: `Reply with exactly ${lastMarker} and nothing else.`, + expectedReply: lastMarker, + message: [ + `Replace durable resume slot B with ${lastMarker}.`, + `Reply with exactly ${lastMarker} and nothing else.`, + ].join(" "), }); expect(observedCodexThreadIds.get(sessionKey)).toBe(threadId); } - resumeStressState = { lastMarker, sessionKey, threadId }; + resumeStressState = { + clientId, + hiddenMarker: compactionStressState?.hiddenMarker, + lastMarker, + persistedCompactionCount: compactionStressState?.persistedCount ?? 0, + sessionKey, + threadId, + }; logCodexLiveStep("resume-stress:history-ready", { - historyTurns: CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS + 2, + historyTurns: historyTurns + 2, threadId, }); } @@ -1583,7 +1856,7 @@ describeLive("gateway live (Codex harness)", () => { } } } - if (CODEX_HARNESS_RESUME_STRESS) { + if (CODEX_HARNESS_RESTART_STRESS) { if (!resumeStressState) { throw new Error("Codex resume stress did not seed a thread"); } @@ -1622,25 +1895,67 @@ describeLive("gateway live (Codex harness)", () => { const nextMarker = `CODEX-RESTART-${restart}-${randomBytes(3) .toString("hex") .toUpperCase()}`; + const priorClientId = resumeStressState.clientId; + const replyInstructions = [ + "Read durable resume slot B before replacing it.", + "Reply with exactly the requested lines and nothing else.", + "First line: the current value of durable resume slot B.", + `Second line: ${nextMarker}`, + `Then replace durable resume slot B with ${nextMarker}.`, + ...(resumeStressState.hiddenMarker + ? ["Third line: the value stored in durable slot A."] + : []), + ]; const resumedText = await requestAgentText({ client, sessionKey: resumeStressState.sessionKey, - expectedToken: nextMarker, - message: [ - "Reply with exactly two lines and nothing else.", - "First line: the final marker from the immediately preceding assistant message.", - `Second line: ${nextMarker}`, - ].join(" "), + expectedReply: nextMarker, + message: replyInstructions.join(" "), }); - expect(resumedText.trim()).toBe(`${resumeStressState.lastMarker}\n${nextMarker}`); + const expectedResumeText = [ + resumeStressState.lastMarker, + nextMarker, + ...(resumeStressState.hiddenMarker ? [resumeStressState.hiddenMarker] : []), + ].join("\n"); + expect(resumedText.trim()).toBe(expectedResumeText); const resumedThreadId = observedCodexThreadIds.get(resumeStressState.sessionKey); expect(resumedThreadId).toBe(resumeStressState.threadId); + expect(observedCodexThreadActions.get(resumeStressState.sessionKey)).toBe("resumed"); + const resumedClientId = observedCodexClientIds.get(resumeStressState.sessionKey); + expect(resumedClientId).toBeTruthy(); + expect(resumedClientId).not.toBe(priorClientId); + resumeStressState.clientId = resumedClientId as string; resumeStressState.lastMarker = nextMarker; + if (resumeStressState.persistedCompactionCount > 0) { + await readCodexHarnessCompactionCount({ + client, + events: gatewayEvents, + minimum: resumeStressState.persistedCompactionCount, + sessionKey: resumeStressState.sessionKey, + }); + } logCodexLiveStep("resume-stress:restart-complete", { restart, + action: observedCodexThreadActions.get(resumeStressState.sessionKey), threadId: resumedThreadId, }); } + if (CODEX_HARNESS_SUBAGENT_PROBE && !CODEX_HARNESS_SUBAGENT_ONLY) { + await verifyCodexSubagentProbe({ + client, + sessionKey: resumeStressState.sessionKey, + }); + } + if (CODEX_HARNESS_COMPACTION_STRESS) { + const continued = await verifyCodexCompactionStress({ + client, + events: gatewayEvents, + sessionKey: resumeStressState.sessionKey, + }); + expect(continued.persistedCount).toBeGreaterThan( + resumeStressState.persistedCompactionCount, + ); + } } } finally { try { diff --git a/test/scripts/test-live-codex-harness-docker.test.ts b/test/scripts/test-live-codex-harness-docker.test.ts index 9e3a45758ed3..f56f6101abda 100644 --- a/test/scripts/test-live-codex-harness-docker.test.ts +++ b/test/scripts/test-live-codex-harness-docker.test.ts @@ -110,6 +110,24 @@ describe("scripts/test-live-codex-harness-docker.sh", () => { expect(script).toContain( '-e OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS="${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS:-3}"', ); + expect(script).toContain( + '-e OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT="${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT:-1}"', + ); + expect(script).toContain( + '-e OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS="${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS:-0}"', + ); + expect(script).toContain( + '-e OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS="${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS:-4}"', + ); + expect(script).toContain( + '-e OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES="${OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES:-300000}"', + ); + expect(script).toContain( + '-e OPENCLAW_LIVE_CODEX_HARNESS_CODE_MODE_ONLY="${OPENCLAW_LIVE_CODEX_HARNESS_CODE_MODE_ONLY:-0}"', + ); + expect(script).toContain( + '-e OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY="${OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY:-0}"', + ); }); it("installs the plugin-pinned Codex CLI package for app-server proof", () => {