From 46c3dba537261109e0465da092d28cc8cdb02f35 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 16 Jul 2026 09:01:00 -0700 Subject: [PATCH] chore: gate releases on Codex progress follow-through (#108828) * test(release): gate Codex progress follow-through * test(release): register Codex follow-through entry * test(release): harden Codex follow-through proof * test(release): preserve legacy Codex follow-through proof --- config/knip.config.ts | 1 + docs/ci.md | 4 +- docs/help/testing.md | 6 + docs/reference/full-release-validation.md | 24 +- scripts/e2e/codex-npm-plugin-live-docker.sh | 69 +++ .../lib/codex-npm-plugin-live/assertions.mjs | 396 +++++++++++++-- .../followthrough-turn.mjs | 60 +++ scripts/test-projects.test-support.mjs | 4 + test/scripts/codex-install-assertions.test.ts | 475 +++++++++++++++++- test/scripts/docker-build-helper.test.ts | 74 +++ 10 files changed, 1036 insertions(+), 77 deletions(-) create mode 100644 scripts/e2e/lib/codex-npm-plugin-live/followthrough-turn.mjs diff --git a/config/knip.config.ts b/config/knip.config.ts index a798831629fe..ae01a54c8dd3 100644 --- a/config/knip.config.ts +++ b/config/knip.config.ts @@ -25,6 +25,7 @@ const repositoryScriptEntries = [ "scripts/e2e/lib/codex-media-path/client.mjs!", "scripts/e2e/lib/codex-media-path/fake-codex-app-server.mjs!", "scripts/e2e/lib/codex-media-path/write-config.mjs!", + "scripts/e2e/lib/codex-npm-plugin-live/followthrough-turn.mjs!", "scripts/e2e/lib/config-reload/assert-log.mjs!", "scripts/e2e/lib/config-reload/mutate-metadata.mjs!", "scripts/e2e/lib/docker-artifact-proof/write-identities.ts!", diff --git a/docs/ci.md b/docs/ci.md index 6d4a65d950b8..33c88c8b2f42 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -256,7 +256,7 @@ Every lane uploads its complete GitHub artifact, including CPU, heap, trace, and ## Full Release Validation -`Full Release Validation` is the manual umbrella workflow for "run everything before release." It accepts a branch, tag, or full commit SHA, dispatches the manual `CI` workflow with that target (including Android), dispatches `Plugin Prerelease` for release-only plugin/package/static/Docker proof, dispatches `OpenClaw Performance` against the target SHA, and dispatches `OpenClaw Release Checks` for install smoke, package acceptance, cross-OS package checks, QA Lab parity, Matrix, Telegram, and gated Discord, WhatsApp, and Slack lanes (advisory maturity scorecard rendering is opt-in via `run_maturity_scorecard`). Stable and full profiles always include exhaustive live/E2E and Docker release-path soak coverage; the beta profile can opt in with `run_release_soak=true`. The canonical package Telegram E2E runs inside Package Acceptance, so a full candidate does not start a duplicate live poller. After publishing, pass `release_package_spec` to reuse the shipped npm package across release checks, Package Acceptance, Docker, cross-OS, and Telegram without rebuilding. Use `npm_telegram_package_spec` only for a focused published-package Telegram rerun. The Codex plugin live package lane uses the same selected state by default: published `release_package_spec=openclaw@` derives `codex_plugin_spec=npm:@openclaw/codex@`, while SHA/artifact runs pack `extensions/codex` from the selected ref. Set `codex_plugin_spec` explicitly for custom plugin sources such as `npm:`, `npm-pack:`, or `git:` specs. +`Full Release Validation` is the manual umbrella workflow for "run everything before release." It accepts a branch, tag, or full commit SHA, dispatches the manual `CI` workflow with that target (including Android), dispatches `Plugin Prerelease` for release-only plugin/package/static/Docker proof, dispatches `OpenClaw Performance` against the target SHA, and dispatches `OpenClaw Release Checks` for install smoke, package acceptance, cross-OS package checks, QA Lab parity, Matrix, Telegram, and gated Discord, WhatsApp, and Slack lanes (advisory maturity scorecard rendering is opt-in via `run_maturity_scorecard`). Stable and full profiles always include exhaustive live/E2E and Docker release-path soak coverage; the beta profile can opt in with `run_release_soak=true`. The canonical package Telegram E2E runs inside Package Acceptance, so a full candidate does not start a duplicate live poller. After publishing, pass `release_package_spec` to reuse the shipped npm package across release checks, Package Acceptance, Docker, cross-OS, and Telegram without rebuilding. Use `npm_telegram_package_spec` only for a focused published-package Telegram rerun. The Codex plugin live package lane uses the same selected state by default: published `release_package_spec=openclaw@` derives `codex_plugin_spec=npm:@openclaw/codex@`, while SHA/artifact runs pack `extensions/codex` from the selected ref. Set `codex_plugin_spec` explicitly for custom plugin sources such as `npm:`, `npm-pack:`, or `git:` specs. Its live agent proof sends visible progress, continues through randomized workspace reads and an exact artifact write, then sends completion. See [Full release validation](/reference/full-release-validation) for the stage matrix, exact workflow job names, profile differences, artifacts, and @@ -513,7 +513,7 @@ Release Docker coverage runs smaller chunked jobs with `OPENCLAW_SKIP_DOCKER_BUI - `OPENCLAW_DOCKER_ALL_PROFILE=release-path` - `OPENCLAW_DOCKER_ALL_CHUNK=core | package-update-openai | package-update-anthropic | package-update-core | plugins-runtime-plugins | plugins-runtime-services | plugins-runtime-install-a..h | openwebui` -Current release Docker chunks are `core`, `package-update-openai`, `package-update-anthropic`, `package-update-core`, `plugins-runtime-plugins`, `plugins-runtime-services`, `plugins-runtime-install-a` through `plugins-runtime-install-h`, and `openwebui`. `package-update-openai` includes the live Codex plugin package lane, which installs the candidate OpenClaw package, installs the Codex plugin from `codex_plugin_spec` or a same-ref tarball with explicit Codex CLI install approval, runs Codex CLI preflight, then runs multiple same-session OpenClaw agent turns against OpenAI. `plugins-runtime-core`, `plugins-runtime`, and `plugins-integrations` remain aggregate plugin/runtime aliases. The `install-e2e` lane alias remains the aggregate manual rerun alias for both provider installer lanes. +Current release Docker chunks are `core`, `package-update-openai`, `package-update-anthropic`, `package-update-core`, `plugins-runtime-plugins`, `plugins-runtime-services`, `plugins-runtime-install-a` through `plugins-runtime-install-h`, and `openwebui`. `package-update-openai` includes the live Codex plugin package lane, which installs the candidate OpenClaw package, installs the Codex plugin from `codex_plugin_spec` or a same-ref tarball with explicit Codex CLI install approval, runs Codex CLI preflight and same-session agent turns, then runs a zero-retry medium-thinking turn that sends progress, reads randomized workspace inputs, writes their exact artifact, and sends completion. `plugins-runtime-core`, `plugins-runtime`, and `plugins-integrations` remain aggregate plugin/runtime aliases. The `install-e2e` lane alias remains the aggregate manual rerun alias for both provider installer lanes. OpenWebUI runs as a standalone `openwebui` chunk on a dedicated large-disk Blacksmith runner whenever stable or full release-path coverage requests it, even when the reusable workflow routes supported jobs to GitHub-hosted runners. Keeping the external image pull separate prevents the large image from competing with the shared package and plugin images in `plugins-runtime-services`; legacy aggregate plugin/runtime chunks still include OpenWebUI for compatible manual reruns. Bundled-channel update lanes retry once for transient npm network failures. diff --git a/docs/help/testing.md b/docs/help/testing.md index f38e2c2a53fd..8933191b8727 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -125,6 +125,12 @@ When debugging real providers/models (requires real creds): - Installs the packaged OpenClaw tarball in Docker, runs OpenAI API-key onboarding, and verifies the Codex plugin plus `@openai/codex` dependency were downloaded into the managed npm project root on demand. +- Codex npm-plugin live package smoke: `pnpm test:docker:live-codex-npm-plugin` + - Installs the candidate OpenClaw package and exact Codex plugin into Docker, + then uses a real OpenAI key for CLI preflight and same-session turns. + - Its zero-retry medium-thinking follow-through turn must send progress, keep + working through randomized workspace reads and an exact artifact write, + then send completion. A progress-only terminal turn fails the lane. - Live plugin tool dependency smoke: `pnpm test:docker:live-plugin-tool` - Packs a fixture plugin with a real `slugify` dependency, installs it through `npm-pack:`, verifies the dependency under the managed npm diff --git a/docs/reference/full-release-validation.md b/docs/reference/full-release-validation.md index 67de3d9fe2c5..4e601e1ee42a 100644 --- a/docs/reference/full-release-validation.md +++ b/docs/reference/full-release-validation.md @@ -56,6 +56,10 @@ SHA/artifact runs pack `extensions/codex` from the selected ref; and operators can set `codex_plugin_spec` directly for `npm:`, `npm-pack:`, or `git:` plugin sources. The lane grants the explicit Codex CLI install approval required by that plugin, then runs Codex CLI preflight and same-session OpenAI agent turns. +Its final zero-retry, medium-thinking turn sends visible progress with omitted +Codex `final`, reads randomized workspace inputs, writes their exact artifact, +and sends explicit completion. This catches the v2026.7.1 regression where an +ordinary progress send terminated the turn. ## Top-level stages @@ -144,16 +148,16 @@ or Docker-facing stages need it. The Docker release-path stage runs these chunks when `live_suite_filter` is empty: -| Chunk | Coverage | -| --------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | -| `core` | Core Docker release-path smoke lanes. | -| `package-update-openai` | OpenAI package install/update behavior, Codex on-demand install, Codex plugin live turns, and Chat Completions tool calls. | -| `package-update-anthropic` | Anthropic package install and update behavior. | -| `package-update-core` | Provider-neutral package and update behavior. | -| `plugins-runtime-plugins` | Plugin runtime lanes that exercise plugin behavior. | -| `plugins-runtime-services` | Service-backed and live plugin runtime lanes. | -| `plugins-runtime-install-a` through `plugins-runtime-install-h` | Plugin install/runtime batches split for parallel release validation. | -| `openwebui` | OpenWebUI compatibility smoke isolated on a dedicated large-disk runner when requested. | +| Chunk | Coverage | +| --------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | +| `core` | Core Docker release-path smoke lanes. | +| `package-update-openai` | OpenAI package install/update behavior, Codex on-demand install, Codex plugin live progress follow-through, and Chat Completions tool calls. | +| `package-update-anthropic` | Anthropic package install and update behavior. | +| `package-update-core` | Provider-neutral package and update behavior. | +| `plugins-runtime-plugins` | Plugin runtime lanes that exercise plugin behavior. | +| `plugins-runtime-services` | Service-backed and live plugin runtime lanes. | +| `plugins-runtime-install-a` through `plugins-runtime-install-h` | Plugin install/runtime batches split for parallel release validation. | +| `openwebui` | OpenWebUI compatibility smoke isolated on a dedicated large-disk runner when requested. | Use targeted `docker_lanes=` on the reusable live/E2E workflow when only one Docker lane failed. The release artifacts include per-lane rerun diff --git a/scripts/e2e/codex-npm-plugin-live-docker.sh b/scripts/e2e/codex-npm-plugin-live-docker.sh index fa3320d55c5f..597fcd1bcee4 100644 --- a/scripts/e2e/codex-npm-plugin-live-docker.sh +++ b/scripts/e2e/codex-npm-plugin-live-docker.sh @@ -47,6 +47,7 @@ if [[ -z "$SESSION_STORE_CONTRACT" ]]; then SESSION_STORE_CONTRACT="legacy-json" fi fi +BINDING_STORE_CONTRACT="${OPENCLAW_CODEX_NPM_PLUGIN_BINDING_STORE_CONTRACT:-plugin-kv}" run_log="" cleanup() { @@ -155,6 +156,7 @@ if ! docker_e2e_run_with_harness \ -e OPENCLAW_CODEX_NPM_PLUGIN_FORCE_UNSAFE_INSTALL="${OPENCLAW_CODEX_NPM_PLUGIN_FORCE_UNSAFE_INSTALL:-1}" \ -e OPENCLAW_CODEX_NPM_PLUGIN_MODEL="${OPENCLAW_CODEX_NPM_PLUGIN_MODEL:-openai/gpt-5.4}" \ -e OPENCLAW_CODEX_NPM_PLUGIN_SPEC="$CODEX_PLUGIN_SPEC" \ + -e OPENCLAW_CODEX_NPM_PLUGIN_BINDING_STORE_CONTRACT="$BINDING_STORE_CONTRACT" \ -e OPENCLAW_CODEX_NPM_PLUGIN_SESSION_STORE_CONTRACT="$SESSION_STORE_CONTRACT" \ -e "OPENCLAW_CODEX_NPM_PLUGIN_ASSERT_MAX_TEXT_FILE_BYTES=$ASSERT_MAX_TEXT_FILE_BYTES" \ -e "OPENCLAW_CODEX_NPM_PLUGIN_ASSERT_MAX_ERROR_TAIL_BYTES=$ASSERT_MAX_ERROR_TAIL_BYTES" \ @@ -226,6 +228,9 @@ dump_debug_logs() { /tmp/openclaw-codex-agent-turn1.err \ /tmp/openclaw-codex-agent-turn2.json \ /tmp/openclaw-codex-agent-turn2.err \ + /tmp/openclaw-codex-followthrough.json \ + /tmp/openclaw-codex-followthrough.log \ + /tmp/openclaw-codex-followthrough.err \ /tmp/openclaw-codex-plugin-uninstall.log \ /tmp/openclaw-codex-plugins-list-after-uninstall.json \ /tmp/openclaw-codex-agent-after-uninstall.json \ @@ -340,6 +345,70 @@ run_agent_turn \ /tmp/openclaw-codex-agent.err node scripts/e2e/lib/codex-npm-plugin-live/assertions.mjs assert-agent-turn "$SUCCESS_MARKER" "$SESSION_ID" "$MODEL_REF" + +FOLLOWTHROUGH_SESSION_ID="${SESSION_ID}-followthrough" +FOLLOWTHROUGH_PROGRESS_MARKER="${SUCCESS_MARKER}-FOLLOWTHROUGH-PROGRESS" +FOLLOWTHROUGH_COMPLETE_MARKER="${SUCCESS_MARKER}-FOLLOWTHROUGH-COMPLETE" +FOLLOWTHROUGH_WORKSPACE="${OPENCLAW_STATE_DIR:?missing OPENCLAW_STATE_DIR}/workspace" +FOLLOWTHROUGH_ARTIFACT="$FOLLOWTHROUGH_WORKSPACE/codex-progress-followthrough.txt" +FOLLOWTHROUGH_SUFFIX="$(node -e 'process.stdout.write(crypto.randomUUID().replaceAll("-", "").slice(0, 12))')" +mkdir -p "$FOLLOWTHROUGH_WORKSPACE" +printf 'qa_alpha=amber-%s\n' "$FOLLOWTHROUGH_SUFFIX" >"$FOLLOWTHROUGH_WORKSPACE/FOLLOWTHROUGH_ALPHA.md" +printf 'qa_beta=violet-%s\n' "$FOLLOWTHROUGH_SUFFIX" >"$FOLLOWTHROUGH_WORKSPACE/FOLLOWTHROUGH_BETA.md" +printf 'qa_gamma=silver-%s\n' "$FOLLOWTHROUGH_SUFFIX" >"$FOLLOWTHROUGH_WORKSPACE/FOLLOWTHROUGH_GAMMA.md" +rm -f "$FOLLOWTHROUGH_ARTIFACT" + +FOLLOWTHROUGH_PROMPT="$(cat </tmp/openclaw-codex-followthrough.log \ + 2>/tmp/openclaw-codex-followthrough.err /dev/null || printf 0) stderr_bytes=$(wc -c /dev/null || printf 0)" +if [ "$followthrough_status" -ne 0 ]; then + dump_debug_logs "$followthrough_status" + exit "$followthrough_status" +fi +node scripts/e2e/lib/codex-npm-plugin-live/assertions.mjs \ + assert-followthrough \ + "$FOLLOWTHROUGH_PROGRESS_MARKER" \ + "$FOLLOWTHROUGH_COMPLETE_MARKER" \ + "$FOLLOWTHROUGH_SESSION_ID" \ + "$MODEL_REF" \ + "$FOLLOWTHROUGH_ARTIFACT" \ + "$FOLLOWTHROUGH_WORKSPACE/FOLLOWTHROUGH_ALPHA.md" \ + "$FOLLOWTHROUGH_WORKSPACE/FOLLOWTHROUGH_BETA.md" \ + "$FOLLOWTHROUGH_WORKSPACE/FOLLOWTHROUGH_GAMMA.md" +echo "followthrough_reply: ${FOLLOWTHROUGH_PROGRESS_MARKER} -> ${FOLLOWTHROUGH_COMPLETE_MARKER}" echo "TRANSCRIPT_END" echo "Uninstalling Codex plugin and verifying the configured harness now fails..." diff --git a/scripts/e2e/lib/codex-npm-plugin-live/assertions.mjs b/scripts/e2e/lib/codex-npm-plugin-live/assertions.mjs index 1de2cbc45696..3380efab98c0 100644 --- a/scripts/e2e/lib/codex-npm-plugin-live/assertions.mjs +++ b/scripts/e2e/lib/codex-npm-plugin-live/assertions.mjs @@ -21,6 +21,8 @@ const allowBetaCompatDiagnostics = process.env.OPENCLAW_CODEX_NPM_PLUGIN_ALLOW_BETA_COMPAT_DIAGNOSTICS === "1"; const sessionStoreContract = process.env.OPENCLAW_CODEX_NPM_PLUGIN_SESSION_STORE_CONTRACT || "sqlite"; +const bindingStoreContract = + process.env.OPENCLAW_CODEX_NPM_PLUGIN_BINDING_STORE_CONTRACT || "plugin-kv"; const MAX_TEXT_FILE_BYTES = readPositiveIntEnv( "OPENCLAW_CODEX_NPM_PLUGIN_ASSERT_MAX_TEXT_FILE_BYTES", 1024 * 1024, @@ -86,7 +88,7 @@ function readTextFileTail(filePath, label, maxBytes = MAX_ERROR_TAIL_BYTES) { } function readCodexBinding(sessionId, sessionKey, entry) { - if (sessionStoreContract === "legacy-json") { + if (bindingStoreContract === "legacy-sidecar") { const sessionFile = typeof entry?.sessionFile === "string" ? entry.sessionFile : ""; if (!sessionFile) { throw new Error(`missing legacy Codex session file for ${sessionId}`); @@ -98,6 +100,11 @@ function readCodexBinding(sessionId, sessionKey, entry) { } return binding; } + if (bindingStoreContract !== "plugin-kv") { + throw new Error( + `OPENCLAW_CODEX_NPM_PLUGIN_BINDING_STORE_CONTRACT must be plugin-kv or legacy-sidecar; got ${bindingStoreContract}`, + ); + } const dbPath = path.join(stateDir(), "state", "openclaw.sqlite"); if (!fs.existsSync(dbPath)) { @@ -148,13 +155,31 @@ function readLegacySessionEntry(sessionId) { if (typeof entry.sessionFile !== "string" || !fs.existsSync(entry.sessionFile)) { throw new Error(`missing OpenClaw session file: ${entry.sessionFile}`); } - const transcriptEventCount = readTextFileBounded( + const transcriptLines = readTextFileBounded( entry.sessionFile, "OpenClaw legacy session transcript", + Math.min(MAX_TEXT_FILE_BYTES, MAX_TRANSCRIPT_SCAN_BYTES), ) .split("\n") - .filter((line) => line.trim()).length; - return { entry, sessionKey, transcriptEventCount }; + .filter((line) => line.trim()); + if (transcriptLines.length > MAX_TRANSCRIPT_WALK_ENTRIES) { + throw new Error( + `OpenClaw transcript exceeded ${MAX_TRANSCRIPT_WALK_ENTRIES} events for ${sessionId}`, + ); + } + const transcriptEvents = transcriptLines.map((line, index) => { + try { + return JSON.parse(line); + } catch { + throw new Error(`invalid OpenClaw legacy transcript event ${index + 1} for ${sessionId}`); + } + }); + return { + entry, + sessionKey, + transcriptEventCount: transcriptEvents.length, + transcriptEvents, + }; } function readSessionEntry(sessionId) { @@ -172,33 +197,84 @@ function readSessionEntry(sessionId) { } const db = new DatabaseSync(dbPath, { readOnly: true }); try { - const row = db - .prepare( - `SELECT se.session_key, se.entry_json, s.agent_harness_id, - (SELECT COUNT(*) - FROM transcript_events AS te - WHERE te.session_id = s.session_id) AS transcript_event_count - FROM sessions AS s - INNER JOIN session_entries AS se ON se.session_id = s.session_id - WHERE s.session_id = ? - ORDER BY se.updated_at DESC, se.session_key - LIMIT 1`, - ) - .get(sessionId); - if (!row || typeof row.session_key !== "string" || typeof row.entry_json !== "string") { - throw new Error(`missing session store entry for ${sessionId}`); + // Keep the aggregate check and row materialization on one SQLite snapshot. + // The release runner never loads transcript JSON until SQL proves the whole set is bounded. + db.exec("BEGIN"); + try { + const row = db + .prepare( + `SELECT se.session_key, se.entry_json, s.agent_harness_id + FROM sessions AS s + INNER JOIN session_entries AS se ON se.session_id = s.session_id + WHERE s.session_id = ? + ORDER BY se.updated_at DESC, se.session_key + LIMIT 1`, + ) + .get(sessionId); + if (!row || typeof row.session_key !== "string" || typeof row.entry_json !== "string") { + throw new Error(`missing session store entry for ${sessionId}`); + } + const transcriptSummary = db + .prepare( + `SELECT COUNT(*) AS event_count, + COALESCE(SUM(length(CAST(event_json AS BLOB))), 0) AS transcript_bytes + FROM transcript_events + WHERE session_id = ?`, + ) + .get(sessionId); + if ( + !transcriptSummary || + !Number.isSafeInteger(transcriptSummary.event_count) || + !Number.isSafeInteger(transcriptSummary.transcript_bytes) + ) { + throw new Error(`invalid OpenClaw transcript summary for ${sessionId}`); + } + if (transcriptSummary.event_count > MAX_TRANSCRIPT_WALK_ENTRIES) { + throw new Error( + `OpenClaw transcript exceeded ${MAX_TRANSCRIPT_WALK_ENTRIES} events for ${sessionId}`, + ); + } + if (transcriptSummary.transcript_bytes > MAX_TRANSCRIPT_SCAN_BYTES) { + throw new Error( + `OpenClaw transcript exceeded ${MAX_TRANSCRIPT_SCAN_BYTES} bytes for ${sessionId}`, + ); + } + const transcriptRows = db + .prepare( + `SELECT event_json + FROM transcript_events + WHERE session_id = ? + ORDER BY seq`, + ) + .all(sessionId); + let transcriptBytes = 0; + const transcriptEvents = transcriptRows.map((transcriptRow) => { + if (typeof transcriptRow.event_json !== "string") { + throw new Error(`invalid OpenClaw transcript event for ${sessionId}`); + } + transcriptBytes += Buffer.byteLength(transcriptRow.event_json); + if (transcriptBytes > MAX_TRANSCRIPT_SCAN_BYTES) { + throw new Error( + `OpenClaw transcript exceeded ${MAX_TRANSCRIPT_SCAN_BYTES} bytes for ${sessionId}`, + ); + } + return JSON.parse(transcriptRow.event_json); + }); + const entry = JSON.parse(row.entry_json); + return { + entry: { + ...entry, + agentHarnessId: + typeof row.agent_harness_id === "string" ? row.agent_harness_id : entry.agentHarnessId, + sessionId, + }, + sessionKey: row.session_key, + transcriptEventCount: transcriptEvents.length, + transcriptEvents, + }; + } finally { + db.exec("ROLLBACK"); } - const entry = JSON.parse(row.entry_json); - return { - entry: { - ...entry, - agentHarnessId: - typeof row.agent_harness_id === "string" ? row.agent_harness_id : entry.agentHarnessId, - sessionId, - }, - sessionKey: row.session_key, - transcriptEventCount: Number(row.transcript_event_count), - }; } finally { db.close(); } @@ -302,12 +378,6 @@ function assertPlugin() { if (inspect.plugin?.id !== "codex" || inspect.plugin?.status !== "loaded") { throw new Error(`unexpected inspect plugin state: ${JSON.stringify(inspect.plugin)}`); } - if ( - !Array.isArray(inspect.plugin?.providerIds) || - !inspect.plugin.providerIds.includes("codex") - ) { - throw new Error(`codex provider was not registered: ${JSON.stringify(inspect.plugin)}`); - } const hasCodexHarness = (Array.isArray(inspect.plugin?.agentHarnessIds) && inspect.plugin.agentHarnessIds.includes("codex")) || @@ -488,29 +558,193 @@ function assertNativeCodexSessionEvidence(params) { ); } let scannedBytes = 0; - const matchingFile = files.find(({ filePath, stat }) => { + let matchingEvidence; + for (const { filePath, stat } of files) { const readableBytes = Math.min(stat.size, MAX_TEXT_FILE_BYTES); if (scannedBytes + readableBytes > MAX_TRANSCRIPT_SCAN_BYTES) { - return false; + continue; } scannedBytes += readableBytes; const content = readTextFileTail(filePath, "native Codex session transcript", readableBytes); - return content.includes(params.marker) || content.includes(params.threadId); - })?.filePath; - if (!matchingFile) { + if (content.includes(params.marker) || content.includes(params.threadId)) { + matchingEvidence = { content, filePath }; + break; + } + } + if (!matchingEvidence) { throw new Error( `native Codex session transcripts did not contain ${params.marker} or ${params.threadId}; scanned ${scannedBytes} bytes across ${files.length} newest files: ${files.map((entry) => entry.filePath).join(", ")}`, ); } - assertPathInside(params.codexHome, matchingFile, "native Codex session transcript"); + assertPathInside(params.codexHome, matchingEvidence.filePath, "native Codex session transcript"); + return matchingEvidence; } -function assertAgentTurn() { - const marker = process.argv[3]; - const sessionId = process.argv[4]; - const modelRef = process.argv[5]; - const stdout = readTextFileBounded("/tmp/openclaw-codex-agent.json", "OpenClaw agent JSON"); - const stderr = readTextFileTail("/tmp/openclaw-codex-agent.err", "OpenClaw agent stderr"); +function extractOpenClawToolTimeline(transcriptEvents) { + const timeline = []; + for (const event of transcriptEvents) { + const message = event?.type === "message" ? event.message : undefined; + if (!message || typeof message !== "object") { + continue; + } + if (message.role === "user") { + timeline.push({ type: "turn-start", index: timeline.length }); + } else if (message.role === "assistant" && Array.isArray(message.content)) { + for (const item of message.content) { + if ( + item?.type === "toolCall" && + typeof item.id === "string" && + typeof item.name === "string" + ) { + timeline.push({ + type: "call", + id: item.id, + name: item.name, + args: item.arguments ?? item.input, + index: timeline.length, + }); + } + } + } else if (message.role === "toolResult" && typeof message.toolCallId === "string") { + timeline.push({ + type: "result", + id: message.toolCallId, + name: typeof message.toolName === "string" ? message.toolName : undefined, + isError: message.isError === true, + index: timeline.length, + }); + } + } + return timeline; +} + +function findSuccessfulToolResult(timeline, call, beforeIndex = Number.POSITIVE_INFINITY) { + return timeline.find( + (entry) => + entry.type === "result" && + entry.id === call.id && + entry.index > call.index && + entry.index < beforeIndex && + entry.isError !== true, + ); +} + +function findToolResult(timeline, call, beforeIndex = Number.POSITIVE_INFINITY) { + return timeline.find( + (entry) => + entry.type === "result" && + entry.id === call.id && + entry.index > call.index && + entry.index < beforeIndex, + ); +} + +function assertFollowthroughTranscript({ transcriptEvents, progressMarker, completeMarker }) { + const timeline = extractOpenClawToolTimeline(transcriptEvents); + const markerCalls = timeline.flatMap((entry) => { + if (entry.type !== "call" || entry.name !== "message") { + return []; + } + const args = entry.args; + if (!args || typeof args !== "object") { + return []; + } + const text = [args.message, args.text, args.body, args.content].find( + (value) => typeof value === "string", + ); + return text === progressMarker || text === completeMarker ? [{ ...entry, args, text }] : []; + }); + const expected = [ + { text: progressMarker, final: undefined }, + { text: completeMarker, final: true }, + ]; + if ( + markerCalls.length !== expected.length || + markerCalls.some( + (call, index) => + call.text !== expected[index].text || + call.args.action !== "send" || + call.args.final !== expected[index].final, + ) + ) { + throw new Error( + `expected exact message final controls ${JSON.stringify(expected)}, got ${JSON.stringify(markerCalls.map((call) => ({ text: call.text, action: call.args.action, final: call.args.final })))}`, + ); + } + const [progressCall, completeCall] = markerCalls; + const progressResult = findSuccessfulToolResult(timeline, progressCall, completeCall.index); + if (!progressResult) { + throw new Error("missing successful progress message result before completion"); + } + const completeResult = findSuccessfulToolResult(timeline, completeCall); + if (!completeResult) { + throw new Error("missing successful completion message result"); + } + + let turnStartIndex = -1; + for (const entry of timeline) { + if (entry.type === "turn-start" && entry.index < progressCall.index) { + turnStartIndex = entry.index; + } + } + const prematureCalls = timeline.filter( + (entry) => + entry.type === "call" && + entry.id !== progressCall.id && + entry.index > turnStartIndex && + entry.index < progressResult.index, + ); + if (prematureCalls.length > 0) { + throw new Error( + `expected progress to be the first completed tool call in its turn; premature calls: ${prematureCalls.map((entry) => entry.name).join(", ")}`, + ); + } + + const workCalls = timeline.filter( + (entry) => + entry.type === "call" && + entry.name !== "message" && + entry.index > progressResult.index && + entry.index < completeCall.index, + ); + if (workCalls.length === 0) { + throw new Error("expected successful workspace work between progress and completion"); + } + const successfulWorkCalls = workCalls.filter((entry) => + findSuccessfulToolResult(timeline, entry, completeCall.index), + ); + if (successfulWorkCalls.length === 0) { + throw new Error("expected successful workspace work between progress and completion"); + } + const unsettledWorkCalls = workCalls.filter( + (entry) => !findToolResult(timeline, entry, completeCall.index), + ); + if (unsettledWorkCalls.length > 0) { + throw new Error( + `expected all workspace work to settle before completion; unsettled calls: ${unsettledWorkCalls.map((entry) => entry.name).join(", ")}`, + ); + } + + const nextTurnStart = timeline.find( + (entry) => entry.type === "turn-start" && entry.index > completeResult.index, + ); + const postCompletionWorkCalls = timeline.filter( + (entry) => + entry.type === "call" && + entry.name !== "message" && + entry.index > completeCall.index && + entry.index < (nextTurnStart?.index ?? Number.POSITIVE_INFINITY), + ); + if (postCompletionWorkCalls.length > 0) { + throw new Error( + `unexpected workspace work after completion: ${postCompletionWorkCalls.map((entry) => entry.name).join(", ")}`, + ); + } +} + +function assertAgentTurnEvidence({ marker, sessionId, modelRef, stdoutPath, stderrPath }) { + const stdout = readTextFileBounded(stdoutPath, "OpenClaw agent JSON"); + const stderr = readTextFileTail(stderrPath, "OpenClaw agent stderr"); const response = JSON.parse(stdout); const text = extractAgentReplyTexts(JSON.stringify(response)).join("\n"); if (!text.includes(marker)) { @@ -526,7 +760,7 @@ function assertAgentTurn() { ); } - const { entry, sessionKey, transcriptEventCount } = readSessionEntry(sessionId); + const { entry, sessionKey, transcriptEventCount, transcriptEvents } = readSessionEntry(sessionId); if (entry.agentHarnessId !== "codex") { throw new Error(`expected codex harness in session entry, got ${entry.agentHarnessId}`); } @@ -560,12 +794,73 @@ function assertAgentTurn() { } const codexSessionRoot = path.join(codexHome, "sessions"); const nativeSessionRoot = path.join(codexHome, "home", ".codex", "sessions"); - assertNativeCodexSessionEvidence({ + const nativeSessionEvidence = assertNativeCodexSessionEvidence({ codexHome, marker, roots: [codexSessionRoot, nativeSessionRoot], threadId: binding.threadId, }); + return { nativeSessionEvidence, transcriptEvents }; +} + +function assertAgentTurn() { + assertAgentTurnEvidence({ + marker: process.argv[3], + sessionId: process.argv[4], + modelRef: process.argv[5], + stdoutPath: "/tmp/openclaw-codex-agent.json", + stderrPath: "/tmp/openclaw-codex-agent.err", + }); +} + +function assertFollowthrough() { + const progressMarker = process.argv[3]; + const completeMarker = process.argv[4]; + const sessionId = process.argv[5]; + const modelRef = process.argv[6]; + const artifactPath = process.argv[7]; + const inputPaths = process.argv.slice(8); + if (inputPaths.length !== 3) { + throw new Error(`expected three follow-through input paths, got ${inputPaths.length}`); + } + + const stdoutPath = "/tmp/openclaw-codex-followthrough.json"; + const stderrPath = "/tmp/openclaw-codex-followthrough.err"; + const response = JSON.parse(readTextFileBounded(stdoutPath, "OpenClaw follow-through JSON")); + const replyTexts = (response.payloads || []) + .map((payload) => (payload && typeof payload.text === "string" ? payload.text.trim() : "")) + .filter(Boolean); + const expectedReplies = [progressMarker, completeMarker]; + if (JSON.stringify(replyTexts) !== JSON.stringify(expectedReplies)) { + throw new Error( + `expected exact progress and completion replies ${JSON.stringify(expectedReplies)}, got ${JSON.stringify(replyTexts)}`, + ); + } + + const expectedArtifact = `${inputPaths + .map((inputPath, index) => + readTextFileBounded(inputPath, `follow-through input ${index + 1}`).trim(), + ) + .join("\n")}\n`; + const artifact = readTextFileBounded(artifactPath, "Codex follow-through artifact"); + if (artifact !== expectedArtifact) { + throw new Error( + `unexpected Codex follow-through artifact; expected ${JSON.stringify(expectedArtifact)}, got ${JSON.stringify(artifact)}`, + ); + } + + const evidence = assertAgentTurnEvidence({ + marker: completeMarker, + sessionId, + modelRef, + stdoutPath, + stderrPath, + }); + assertFollowthroughTranscript({ + transcriptEvents: evidence.transcriptEvents, + progressMarker, + completeMarker, + }); } function assertUninstalled() { @@ -624,6 +919,7 @@ const commands = { "print-codex-bin": printCodexBin, "assert-preflight": assertPreflight, "assert-agent-turn": assertAgentTurn, + "assert-followthrough": assertFollowthrough, "assert-uninstalled": assertUninstalled, "assert-agent-error": assertAgentError, }; diff --git a/scripts/e2e/lib/codex-npm-plugin-live/followthrough-turn.mjs b/scripts/e2e/lib/codex-npm-plugin-live/followthrough-turn.mjs new file mode 100644 index 000000000000..11b56f15e71b --- /dev/null +++ b/scripts/e2e/lib/codex-npm-plugin-live/followthrough-turn.mjs @@ -0,0 +1,60 @@ +// Runs one package-installed OpenClaw turn with explicit message-tool-only +// source delivery so Codex exposes the progress/final control under test. +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +const [packageRoot, sessionId, modelRef, timeoutSeconds, outputPath, prompt] = + process.argv.slice(2); +if (!packageRoot || !sessionId || !modelRef || !timeoutSeconds || !outputPath || !prompt) { + throw new Error( + "usage: followthrough-turn.mjs ", + ); +} +if (!/^\d+$/u.test(timeoutSeconds) || Number(timeoutSeconds) < 1) { + throw new Error(`invalid timeout seconds: ${timeoutSeconds}`); +} + +const agentRuntimePath = path.join( + path.resolve(packageRoot), + "dist", + "plugin-sdk", + "agent-runtime.js", +); +const { agentCommandFromIngress } = await import(pathToFileURL(agentRuntimePath).href); +if (typeof agentCommandFromIngress !== "function") { + throw new Error( + `package agent runtime did not export agentCommandFromIngress: ${agentRuntimePath}`, + ); +} + +const quietRuntime = { + log: () => {}, + error: (...args) => console.error(...args), + exit: (code) => { + throw new Error(`agent runtime exited with code ${code}`); + }, +}; +const result = await agentCommandFromIngress( + { + agentId: "main", + sessionId, + message: prompt, + model: modelRef, + thinking: "medium", + timeout: timeoutSeconds, + json: true, + messageChannel: "webchat", + channel: "webchat", + sourceReplyDeliveryMode: "message_tool_only", + senderIsOwner: true, + allowModelOverride: true, + // The embedded one-shot path retires bundled runtime resources; the Codex + // harness uses this signal to close its shared app-server client and child. + cleanupBundleMcpOnRunEnd: true, + cleanupCliLiveSessionOnRunEnd: true, + oneShotCliRun: true, + }, + quietRuntime, +); +fs.writeFileSync(outputPath, `${JSON.stringify(result)}\n`); diff --git a/scripts/test-projects.test-support.mjs b/scripts/test-projects.test-support.mjs index cdc13b174ace..bdd6ce43499e 100644 --- a/scripts/test-projects.test-support.mjs +++ b/scripts/test-projects.test-support.mjs @@ -1781,6 +1781,10 @@ const TOOLING_SOURCE_TEST_TARGETS = new Map([ "scripts/e2e/lib/codex-npm-plugin-live/assertions.mjs", ["test/scripts/codex-install-assertions.test.ts", "test/scripts/docker-build-helper.test.ts"], ], + [ + "scripts/e2e/lib/codex-npm-plugin-live/followthrough-turn.mjs", + ["test/scripts/docker-build-helper.test.ts"], + ], ["scripts/e2e/lib/codex-install-utils.mjs", ["test/scripts/codex-install-assertions.test.ts"]], [ "scripts/e2e/lib/codex-on-demand/assertions.mjs", diff --git a/test/scripts/codex-install-assertions.test.ts b/test/scripts/codex-install-assertions.test.ts index e5999afd19ab..80abb77fa174 100644 --- a/test/scripts/codex-install-assertions.test.ts +++ b/test/scripts/codex-install-assertions.test.ts @@ -11,6 +11,7 @@ import { findPackageJson, npmProjectRootForInstalledPackage, } from "../../scripts/e2e/lib/codex-install-utils.mjs"; +import { writePluginInstallIndexForE2E } from "../../scripts/e2e/lib/plugin-index-sqlite.mjs"; import { cleanupTempDirs, makeTempDir } from "../helpers/temp-dir.js"; const CODEX_ON_DEMAND_ASSERTIONS_SCRIPT = "scripts/e2e/lib/codex-on-demand/assertions.mjs"; @@ -21,7 +22,11 @@ const tempDirs: string[] = []; const tmpFixtureFiles = [ "/tmp/openclaw-codex-agent.err", "/tmp/openclaw-codex-agent.json", + "/tmp/openclaw-codex-followthrough.err", + "/tmp/openclaw-codex-followthrough.json", "/tmp/openclaw-codex-inspect.json", + "/tmp/openclaw-codex-plugin-inspect.json", + "/tmp/openclaw-codex-plugins-list.json", "/tmp/openclaw-plugins-list.json", ]; @@ -97,6 +102,7 @@ function runCodexNpmPluginLiveAssertions(params: { marker: string; sessionId: string; modelRef: string; + bindingStoreContract?: "legacy-sidecar" | "plugin-kv"; sessionStoreContract?: "legacy-json" | "sqlite"; }) { return spawnSync( @@ -115,12 +121,54 @@ function runCodexNpmPluginLiveAssertions(params: { HOME: path.join(params.root, "home"), NODE_OPTIONS: nodeOptionsWithoutExperimentalWarnings(), OPENCLAW_STATE_DIR: path.join(params.root, "state"), + OPENCLAW_CODEX_NPM_PLUGIN_BINDING_STORE_CONTRACT: + params.bindingStoreContract ?? "plugin-kv", OPENCLAW_CODEX_NPM_PLUGIN_SESSION_STORE_CONTRACT: params.sessionStoreContract ?? "sqlite", }, }, ); } +function runCodexNpmPluginLiveFollowthroughAssertions(params: { + root: string; + progressMarker: string; + completeMarker: string; + sessionId: string; + modelRef: string; + artifactPath: string; + inputPaths: string[]; + bindingStoreContract?: "legacy-sidecar" | "plugin-kv"; + sessionStoreContract?: "legacy-json" | "sqlite"; + assertionEnv?: Record; +}) { + return spawnSync( + process.execPath, + [ + CODEX_NPM_PLUGIN_LIVE_ASSERTIONS_SCRIPT, + "assert-followthrough", + params.progressMarker, + params.completeMarker, + params.sessionId, + params.modelRef, + params.artifactPath, + ...params.inputPaths, + ], + { + encoding: "utf8", + env: { + ...process.env, + HOME: path.join(params.root, "home"), + NODE_OPTIONS: nodeOptionsWithoutExperimentalWarnings(), + OPENCLAW_STATE_DIR: path.join(params.root, "state"), + OPENCLAW_CODEX_NPM_PLUGIN_BINDING_STORE_CONTRACT: + params.bindingStoreContract ?? "plugin-kv", + OPENCLAW_CODEX_NPM_PLUGIN_SESSION_STORE_CONTRACT: params.sessionStoreContract ?? "sqlite", + ...params.assertionEnv, + }, + }, + ); +} + function runCodexNpmPluginLiveConfigure(root: string) { return spawnSync(process.execPath, [CODEX_NPM_PLUGIN_LIVE_ASSERTIONS_SCRIPT, "configure"], { encoding: "utf8", @@ -134,6 +182,23 @@ function runCodexNpmPluginLiveConfigure(root: string) { }); } +function runCodexNpmPluginLivePluginAssertions(root: string) { + return spawnSync( + process.execPath, + [CODEX_NPM_PLUGIN_LIVE_ASSERTIONS_SCRIPT, "assert-plugin", "npm:@openclaw/codex"], + { + encoding: "utf8", + env: { + ...process.env, + HOME: path.join(root, "home"), + NODE_OPTIONS: nodeOptionsWithoutExperimentalWarnings(), + OPENCLAW_CONFIG_PATH: path.join(root, "state", "openclaw.json"), + OPENCLAW_STATE_DIR: path.join(root, "state"), + }, + }, + ); +} + function writeCodexBindingStateSqlite(params: { stateDir: string; sessionKey: string; @@ -244,6 +309,51 @@ function writeSessionStoreSqlite(params: { } } +function replaceSessionTranscriptMessages(params: { + stateDir: string; + sessionId: string; + messages: unknown[]; +}) { + const dbPath = path.join(params.stateDir, "agents", "main", "agent", "openclaw-agent.sqlite"); + const db = new DatabaseSync(dbPath); + try { + const now = Date.now(); + db.prepare("DELETE FROM transcript_events WHERE session_id = ?").run(params.sessionId); + const insert = db.prepare( + `INSERT INTO transcript_events (session_id, seq, event_json, created_at) + VALUES (?, ?, ?, ?)`, + ); + insert.run(params.sessionId, 0, '{"type":"session"}', now); + params.messages.forEach((message, index) => { + insert.run( + params.sessionId, + index + 1, + JSON.stringify({ type: "message", message }), + now + index + 1, + ); + }); + } finally { + db.close(); + } +} + +function transcriptToolCall(id: string, name: string, args: Record) { + return { + role: "assistant", + content: [{ type: "toolCall", id, name, arguments: args, input: args }], + }; +} + +function transcriptToolResult(id: string, name: string, isError = false) { + return { + role: "toolResult", + toolCallId: id, + toolName: name, + isError, + content: [{ type: "toolResult", id, name, content: "ok" }], + }; +} + function createCodexNpmPluginLiveFixture(root: string, storedSessionId?: string) { const stateDir = path.join(root, "state"); const sessionKey = "agent:main:codex-npm-plugin-live"; @@ -274,20 +384,133 @@ function createCodexNpmPluginLiveFixture(root: string, storedSessionId?: string) return { root, marker, sessionId, modelRef }; } -function createLegacyCodexNpmPluginLiveFixture(root: string) { - const fixture = createCodexNpmPluginLiveFixture(root); - const stateDir = path.join(root, "state"); - rmSync(path.join(stateDir, "agents", "main", "agent", "openclaw-agent.sqlite")); - rmSync(path.join(stateDir, "state", "openclaw.sqlite")); - const sessionFile = path.join(stateDir, "agents", "main", "sessions", "session.jsonl"); - writeJson(sessionFile, { type: "message" }); - writeJson(`${sessionFile}.codex-app-server.json`, { - schemaVersion: 2, - threadId: "thread-codex-npm-live", - cwd: stateDir, - model: "gpt-5.4", - modelProvider: "codex", +function createCodexNpmPluginLiveFollowthroughFixture(params: { + root: string; + replyTexts?: string[]; + artifactText?: string; + messageFinals?: Array; + readFails?: boolean; + bindingStoreContract?: "legacy-sidecar" | "plugin-kv"; + sessionStoreContract?: "legacy-json" | "sqlite"; + workPlacement?: + | "between" + | "before-progress" + | "before-progress-result" + | "write-result-after-completion"; +}) { + const fixture = createCodexNpmPluginLiveFixture(params.root); + const progressMarker = `${fixture.marker}-FOLLOWTHROUGH-PROGRESS`; + const completeMarker = `${fixture.marker}-FOLLOWTHROUGH-COMPLETE`; + const workspaceDir = path.join(params.root, "state", "workspace"); + mkdirSync(workspaceDir, { recursive: true }); + const inputPaths = ["ALPHA.md", "BETA.md", "GAMMA.md"].map((name, index) => { + const inputPath = path.join(workspaceDir, name); + writeFileSync(inputPath, `hidden-${index + 1}\n`, "utf8"); + return inputPath; }); + const artifactPath = path.join(workspaceDir, "codex-progress-followthrough.txt"); + writeFileSync(artifactPath, params.artifactText ?? "hidden-1\nhidden-2\nhidden-3\n", "utf8"); + const replyTexts = params.replyTexts ?? [progressMarker, completeMarker]; + writeJson("/tmp/openclaw-codex-followthrough.json", { + payloads: replyTexts.map((text) => ({ text })), + meta: { executionTrace: { winnerProvider: "openai" } }, + }); + const messageFinals = params.messageFinals ?? [undefined, true]; + const messageCalls = [progressMarker, completeMarker].map((text, index) => { + const args = { + action: "send", + message: text, + ...(messageFinals[index] === undefined ? {} : { final: messageFinals[index] }), + }; + const id = `message-${index + 1}`; + return [transcriptToolCall(id, "message", args), transcriptToolResult(id, "message")]; + }); + const [progressCalls, completionCalls] = messageCalls; + if (!progressCalls || !completionCalls) { + throw new Error("expected progress and completion message fixtures"); + } + const readId = "workspace-read"; + const readMessages = [ + transcriptToolCall(readId, "bash", { + command: "cat *.md", + }), + transcriptToolResult(readId, "bash", params.readFails), + ]; + const writeId = "workspace-write"; + const writeMessages = [ + transcriptToolCall(writeId, "bash", { + command: "cat *.md > codex-progress-followthrough.txt", + }), + transcriptToolResult(writeId, "bash"), + ]; + const workMessages = [...readMessages, ...writeMessages]; + let transcriptMessages; + if (params.workPlacement === "before-progress") { + transcriptMessages = [...workMessages, ...progressCalls, ...completionCalls]; + } else if (params.workPlacement === "before-progress-result") { + transcriptMessages = [progressCalls[0], ...workMessages, progressCalls[1], ...completionCalls]; + } else if (params.workPlacement === "write-result-after-completion") { + transcriptMessages = [ + ...progressCalls, + ...readMessages, + ...writeMessages.slice(0, 1), + ...completionCalls, + ...writeMessages.slice(1), + ]; + } else { + transcriptMessages = [...progressCalls, ...workMessages, ...completionCalls]; + } + const transcriptEvents = [ + { type: "session" }, + ...transcriptMessages.map((message) => ({ type: "message", message })), + ]; + replaceSessionTranscriptMessages({ + stateDir: path.join(params.root, "state"), + sessionId: fixture.sessionId, + messages: transcriptMessages, + }); + const result = { + ...fixture, + progressMarker, + completeMarker, + artifactPath, + inputPaths, + }; + return params.sessionStoreContract === "legacy-json" + ? convertCodexNpmPluginLiveFixtureToLegacy( + result, + transcriptEvents, + params.bindingStoreContract ?? "plugin-kv", + ) + : result; +} + +function convertCodexNpmPluginLiveFixtureToLegacy< + Fixture extends { root: string; sessionId: string }, +>( + fixture: Fixture, + transcriptEvents: unknown[] = [{ type: "message" }], + bindingStoreContract: "legacy-sidecar" | "plugin-kv" = "legacy-sidecar", +) { + const stateDir = path.join(fixture.root, "state"); + rmSync(path.join(stateDir, "agents", "main", "agent", "openclaw-agent.sqlite")); + const sessionFile = path.join(stateDir, "agents", "main", "sessions", "session.jsonl"); + mkdirSync(path.dirname(sessionFile), { recursive: true }); + writeFileSync( + sessionFile, + `${transcriptEvents.map((event) => JSON.stringify(event)).join("\n")}\n`, + "utf8", + ); + if (bindingStoreContract === "legacy-sidecar") { + rmSync(path.join(stateDir, "state", "openclaw.sqlite")); + writeJson(`${sessionFile}.codex-app-server.json`, { + schemaVersion: 2, + threadId: "thread-codex-npm-live", + cwd: stateDir, + model: "gpt-5.4", + modelProvider: "codex", + }); + } writeJson(path.join(stateDir, "agents", "main", "sessions", "sessions.json"), { "agent:main:codex-npm-plugin-live": { sessionId: fixture.sessionId, @@ -295,7 +518,15 @@ function createLegacyCodexNpmPluginLiveFixture(root: string) { sessionFile, }, }); - return { ...fixture, sessionStoreContract: "legacy-json" as const }; + return { + ...fixture, + bindingStoreContract, + sessionStoreContract: "legacy-json" as const, + }; +} + +function createLegacyCodexNpmPluginLiveFixture(root: string) { + return convertCodexNpmPluginLiveFixtureToLegacy(createCodexNpmPluginLiveFixture(root)); } function createCodexInstallFixture(root: string) { @@ -341,7 +572,7 @@ describe("Codex install helpers", () => { const result = runCodexNpmPluginLiveConfigure(root); - expect(result.status).toBe(0); + expect(result.status, result.stderr).toBe(0); expect(result.stderr).toBe(""); const config = JSON.parse(readFileSync(path.join(root, "state", "openclaw.json"), "utf8")) as { agents: { @@ -362,6 +593,54 @@ describe("Codex install helpers", () => { expect(config.agents.defaults.models).not.toHaveProperty("codex/gpt-5.4"); }); + it("accepts the canonical harness-only Codex plugin registration", () => { + const root = makeTempDir(tempDirs, "openclaw-codex-harness-registration-"); + createCodexInstallFixture(root); + const installPath = path.join( + root, + "state", + "npm", + "projects", + "codex", + "node_modules", + "@openclaw", + "codex", + ); + writePluginInstallIndexForE2E( + { + installRecords: { + codex: { + installPath, + source: "npm", + spec: "@openclaw/codex", + resolvedVersion: "2026.7.2", + resolvedSpec: "@openclaw/codex@2026.7.2", + }, + }, + }, + { stateDir: path.join(root, "state") }, + ); + writeJson("/tmp/openclaw-codex-plugins-list.json", { + plugins: [{ id: "codex", enabled: true, status: "loaded" }], + diagnostics: [], + }); + writeJson("/tmp/openclaw-codex-plugin-inspect.json", { + plugin: { + id: "codex", + status: "loaded", + providerIds: [], + agentHarnessIds: ["codex"], + }, + capabilities: [{ kind: "agent-harness", ids: ["codex"] }], + diagnostics: [], + }); + + const result = runCodexNpmPluginLivePluginAssertions(root); + + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).toBe(""); + }); + it("resolves package roots and package manifests inside managed npm installs", () => { const root = makeTempDir(tempDirs, "openclaw-codex-install-utils-"); const packageRoot = path.join( @@ -414,6 +693,172 @@ describe("Codex install helpers", () => { expect(result.stderr).toBe(""); }); + it.each(["sqlite", "legacy-json"] as const)( + "accepts progress, artifact work, and completion with the %s session contract", + (sessionStoreContract) => { + const root = makeTempDir(tempDirs, "openclaw-codex-npm-followthrough-"); + const fixture = createCodexNpmPluginLiveFollowthroughFixture({ root, sessionStoreContract }); + + const result = runCodexNpmPluginLiveFollowthroughAssertions(fixture); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + }, + ); + + it("accepts settled failed work before a later successful artifact write", () => { + const root = makeTempDir(tempDirs, "openclaw-codex-npm-followthrough-recovered-work-"); + const fixture = createCodexNpmPluginLiveFollowthroughFixture({ root, readFails: true }); + + const result = runCodexNpmPluginLiveFollowthroughAssertions(fixture); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + }); + + it.each( + (["sqlite", "legacy-json"] as const).flatMap( + (sessionStoreContract) => + [ + [ + sessionStoreContract, + "event count", + "OPENCLAW_CODEX_NPM_PLUGIN_ASSERT_MAX_TRANSCRIPT_WALK_ENTRIES", + "2", + "exceeded 2 events", + ], + [ + sessionStoreContract, + "aggregate bytes", + "OPENCLAW_CODEX_NPM_PLUGIN_ASSERT_MAX_TRANSCRIPT_SCAN_BYTES", + "128", + "exceeded 128 bytes", + ], + ] as const, + ), + )( + "rejects an oversized %s transcript by %s before assertions", + (sessionStoreContract, _label, envName, limit, errorText) => { + const root = makeTempDir(tempDirs, "openclaw-codex-npm-followthrough-bounded-"); + const fixture = createCodexNpmPluginLiveFollowthroughFixture({ + root, + sessionStoreContract, + }); + + const result = runCodexNpmPluginLiveFollowthroughAssertions({ + ...fixture, + assertionEnv: { [envName]: limit }, + }); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain(errorText); + }, + ); + + it("rejects a Codex live turn that stops after its progress message", () => { + const root = makeTempDir(tempDirs, "openclaw-codex-npm-followthrough-progress-only-"); + const fixture = createCodexNpmPluginLiveFollowthroughFixture({ + root, + replyTexts: ["OPENCLAW-CODEX-NPM-PLUGIN-LIVE-OK-FOLLOWTHROUGH-PROGRESS"], + }); + + const result = runCodexNpmPluginLiveFollowthroughAssertions(fixture); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("expected exact progress and completion replies"); + }); + + it("rejects a Codex live turn whose follow-through artifact is incomplete", () => { + const root = makeTempDir(tempDirs, "openclaw-codex-npm-followthrough-incomplete-"); + const fixture = createCodexNpmPluginLiveFollowthroughFixture({ + root, + artifactText: "hidden-1\n", + }); + + const result = runCodexNpmPluginLiveFollowthroughAssertions(fixture); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("unexpected Codex follow-through artifact"); + }); + + it("rejects workspace work outside the progress and completion messages", () => { + const root = makeTempDir(tempDirs, "openclaw-codex-npm-followthrough-work-order-"); + const fixture = createCodexNpmPluginLiveFollowthroughFixture({ + root, + workPlacement: "before-progress", + }); + + const result = runCodexNpmPluginLiveFollowthroughAssertions(fixture); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain( + "expected progress to be the first completed tool call in its turn", + ); + }); + + it.each(["sqlite", "legacy-json"] as const)( + "rejects workspace work issued before progress delivery completes with the %s session contract", + (sessionStoreContract) => { + const root = makeTempDir(tempDirs, "openclaw-codex-npm-followthrough-batched-work-"); + const fixture = createCodexNpmPluginLiveFollowthroughFixture({ + root, + sessionStoreContract, + workPlacement: "before-progress-result", + }); + + const result = runCodexNpmPluginLiveFollowthroughAssertions(fixture); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain( + "expected progress to be the first completed tool call in its turn", + ); + }, + ); + + it("rejects a malformed legacy follow-through transcript", () => { + const root = makeTempDir(tempDirs, "openclaw-codex-npm-followthrough-legacy-malformed-"); + const fixture = createCodexNpmPluginLiveFollowthroughFixture({ + root, + sessionStoreContract: "legacy-json", + }); + const sessionFile = path.join(root, "state", "agents", "main", "sessions", "session.jsonl"); + writeFileSync(sessionFile, `${readFileSync(sessionFile, "utf8")}not-json\n`, "utf8"); + + const result = runCodexNpmPluginLiveFollowthroughAssertions(fixture); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("invalid OpenClaw legacy transcript event"); + }); + + it("rejects completion sent before the artifact write succeeds", () => { + const root = makeTempDir(tempDirs, "openclaw-codex-npm-followthrough-pending-write-"); + const fixture = createCodexNpmPluginLiveFollowthroughFixture({ + root, + workPlacement: "write-result-after-completion", + }); + + const result = runCodexNpmPluginLiveFollowthroughAssertions(fixture); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("expected all workspace work to settle before completion"); + }); + + it.each([ + ["explicit progress", [false, true]], + ["missing completion", [undefined, undefined]], + ] as const)("rejects %s Codex message final controls", (_label, messageFinals) => { + const root = makeTempDir(tempDirs, "openclaw-codex-npm-followthrough-final-controls-"); + const fixture = createCodexNpmPluginLiveFollowthroughFixture({ + root, + messageFinals: [...messageFinals], + }); + + const result = runCodexNpmPluginLiveFollowthroughAssertions(fixture); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("expected exact message final controls"); + }); + it("accepts the explicit frozen-target JSON session and sidecar binding contract", () => { const root = makeTempDir(tempDirs, "openclaw-codex-npm-live-legacy-"); const fixture = createLegacyCodexNpmPluginLiveFixture(root); diff --git a/test/scripts/docker-build-helper.test.ts b/test/scripts/docker-build-helper.test.ts index 4516bc6c3524..61c49c98a2ec 100644 --- a/test/scripts/docker-build-helper.test.ts +++ b/test/scripts/docker-build-helper.test.ts @@ -51,6 +51,8 @@ const OPENAI_CHAT_TOOLS_SCENARIO_PATH = "scripts/e2e/lib/openai-chat-tools/scena const CODEX_NPM_PLUGIN_LIVE_DOCKER_E2E_PATH = "scripts/e2e/codex-npm-plugin-live-docker.sh"; const CODEX_NPM_PLUGIN_LIVE_ASSERTIONS_PATH = "scripts/e2e/lib/codex-npm-plugin-live/assertions.mjs"; +const CODEX_NPM_PLUGIN_LIVE_FOLLOWTHROUGH_PATH = + "scripts/e2e/lib/codex-npm-plugin-live/followthrough-turn.mjs"; const LIVE_PLUGIN_TOOL_DOCKER_E2E_PATH = "scripts/e2e/live-plugin-tool-docker.sh"; const NPM_ONBOARD_CHANNEL_AGENT_DOCKER_E2E_PATH = "scripts/e2e/npm-onboard-channel-agent-docker.sh"; const SKILL_INSTALL_DOCKER_E2E_PATH = "scripts/e2e/skill-install-docker.sh"; @@ -2735,6 +2737,7 @@ grep -Fxq preserved "$TMPDIR/caller-fd" it("bounds Codex npm plugin live assertion file and transcript reads", () => { const assertions = readFileSync(CODEX_NPM_PLUGIN_LIVE_ASSERTIONS_PATH, "utf8"); + const followthrough = readFileSync(CODEX_NPM_PLUGIN_LIVE_FOLLOWTHROUGH_PATH, "utf8"); const runner = readFileSync(CODEX_NPM_PLUGIN_LIVE_DOCKER_E2E_PATH, "utf8"); expect(assertions).toContain("OPENCLAW_CODEX_NPM_PLUGIN_ASSERT_MAX_TEXT_FILE_BYTES"); @@ -2742,6 +2745,11 @@ grep -Fxq preserved "$TMPDIR/caller-fd" expect(assertions).toContain("OPENCLAW_CODEX_NPM_PLUGIN_ASSERT_MAX_TRANSCRIPT_FILES"); expect(assertions).toContain("OPENCLAW_CODEX_NPM_PLUGIN_ASSERT_MAX_TRANSCRIPT_WALK_ENTRIES"); expect(assertions).toContain("OPENCLAW_CODEX_NPM_PLUGIN_ASSERT_MAX_TRANSCRIPT_SCAN_BYTES"); + expect(assertions).toContain( + "COALESCE(SUM(length(CAST(event_json AS BLOB))), 0) AS transcript_bytes", + ); + expect(assertions).toContain('db.exec("BEGIN")'); + expect(assertions).toContain('db.exec("ROLLBACK")'); expect(assertions).toContain("const AGENT_TURN_TIMEOUT_SECONDS = readPositiveIntEnv("); expect(assertions).toContain('"OPENCLAW_CODEX_NPM_PLUGIN_AGENT_TIMEOUT_SECONDS"'); expect(assertions).toContain("requestTimeoutMs: AGENT_TURN_TIMEOUT_SECONDS * 1000"); @@ -2762,6 +2770,19 @@ grep -Fxq preserved "$TMPDIR/caller-fd" 'readTextFileTail(\n "/tmp/openclaw-codex-agent-after-uninstall.err",', ); expect(runner).toContain('assert-agent-error "$post_uninstall_status"'); + expect(runner).toContain("assert-followthrough"); + expect(runner).toContain("followthrough-turn.mjs"); + expect(followthrough).toContain('"dist",\n "plugin-sdk",\n "agent-runtime.js"'); + expect(followthrough).toContain("agentCommandFromIngress"); + expect(followthrough).toContain('sourceReplyDeliveryMode: "message_tool_only"'); + expect(followthrough).toContain('thinking: "medium"'); + expect(runner).toContain("without passing final"); + expect(runner).toContain("omitted, not false"); + expect(runner).not.toContain("final=false"); + expect(runner).toContain("final=true"); + expect(assertions).toContain('"assert-followthrough": assertFollowthrough'); + expect(assertions).toContain("expected exact progress and completion replies"); + expect(assertions).toContain("unexpected Codex follow-through artifact"); expect(runner).not.toContain("tail -n 120 /tmp/openclaw-codex-agent-after-uninstall.err"); expect(runner).not.toContain("cat /tmp/openclaw-codex-agent-after-uninstall.err"); const earlyAgentTimeoutEnvIndex = runner.indexOf( @@ -2799,6 +2820,59 @@ grep -Fxq preserved "$TMPDIR/caller-fd" expect(runner).not.toContain("--timeout 420"); }); + it("writes the packaged Codex follow-through result independently of stdout logs", () => { + const workDir = tempDirs.make("openclaw-codex-followthrough-"); + const packageRoot = join(workDir, "package"); + const runtimeDir = join(packageRoot, "dist", "plugin-sdk"); + const outputPath = join(workDir, "result.json"); + mkdirSync(runtimeDir, { recursive: true }); + writeFileSync(join(packageRoot, "package.json"), '{"type":"module"}\n'); + writeFileSync( + join(runtimeDir, "agent-runtime.js"), + [ + "export async function agentCommandFromIngress(opts, runtime) {", + ' runtime.log("unexpected runtime output");', + ' console.log("unexpected subsystem output");', + " return { captured: opts };", + "}", + ].join("\n"), + ); + + const result = spawnSync( + process.execPath, + [ + CODEX_NPM_PLUGIN_LIVE_FOLLOWTHROUGH_PATH, + packageRoot, + "followthrough-session", + "openai/gpt-5.4", + "90", + outputPath, + "follow through", + ], + { encoding: "utf8" }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).toBe(""); + expect(result.stdout).toContain("unexpected subsystem output"); + expect(JSON.parse(readFileSync(outputPath, "utf8"))).toEqual({ + captured: expect.objectContaining({ + sessionId: "followthrough-session", + model: "openai/gpt-5.4", + message: "follow through", + thinking: "medium", + timeout: "90", + json: true, + sourceReplyDeliveryMode: "message_tool_only", + senderIsOwner: true, + allowModelOverride: true, + cleanupBundleMcpOnRunEnd: true, + cleanupCliLiveSessionOnRunEnd: true, + oneShotCliRun: true, + }), + }); + }); + it.each([ [ "Codex npm plugin live",