diff --git a/extensions/codex/src/app-server/event-projector-reasoning.ts b/extensions/codex/src/app-server/event-projector-reasoning.ts index bfbf0677802b..2a7e30008fa0 100644 --- a/extensions/codex/src/app-server/event-projector-reasoning.ts +++ b/extensions/codex/src/app-server/event-projector-reasoning.ts @@ -103,8 +103,7 @@ export class CodexReasoningProjection { .filter((part): part is string => Boolean(part)) .join("\n"); if (planText) { - // Structured turn updates are the canonical latest plan. Retain the last - // non-empty update so the terminal transcript proves planning occurred. + // Structured turn updates are the canonical latest plan for terminal classification. this.turnPlanText = planText; } if (source === "codex-app-server" && plan) { diff --git a/extensions/codex/src/app-server/event-projector-result.ts b/extensions/codex/src/app-server/event-projector-result.ts index c38db652474e..cb4ec7f2ab70 100644 --- a/extensions/codex/src/app-server/event-projector-result.ts +++ b/extensions/codex/src/app-server/event-projector-result.ts @@ -154,7 +154,6 @@ export function buildCodexAttemptResult( turnId: input.turnId, upstreamUserText: input.upstreamUserText, reasoningText, - planText, asyncMessages, commentaryMessages, toolMessages: input.toolTranscriptProjection.transcriptMessages, diff --git a/extensions/codex/src/app-server/event-projector-snapshot.test.ts b/extensions/codex/src/app-server/event-projector-snapshot.test.ts index 098089268b07..8077b0748476 100644 --- a/extensions/codex/src/app-server/event-projector-snapshot.test.ts +++ b/extensions/codex/src/app-server/event-projector-snapshot.test.ts @@ -16,7 +16,6 @@ function buildSnapshot(trigger: EmbeddedRunAttemptParams["trigger"]): AgentMessa turnId: "turn-1", upstreamUserText: undefined, reasoningText: "checking memory", - planText: undefined, asyncMessages: [], commentaryMessages: [], toolMessages: [ diff --git a/extensions/codex/src/app-server/event-projector-snapshot.ts b/extensions/codex/src/app-server/event-projector-snapshot.ts index 1654015395b4..f8f39b5b8dc6 100644 --- a/extensions/codex/src/app-server/event-projector-snapshot.ts +++ b/extensions/codex/src/app-server/event-projector-snapshot.ts @@ -38,7 +38,6 @@ export function buildCodexMessagesSnapshot(params: { turnId: string; upstreamUserText: string | undefined; reasoningText: string | undefined; - planText: string | undefined; asyncMessages: ReadonlyArray<{ itemId: string; message: AssistantMessage }>; commentaryMessages: ReadonlyArray<{ itemId: string; message: AssistantMessage }>; toolMessages: readonly AgentMessage[]; @@ -55,14 +54,6 @@ export function buildCodexMessagesSnapshot(params: { ), ); } - if (params.planText) { - messages.push( - attachCodexMirrorIdentity( - params.createAssistantMirrorMessage("Codex plan", params.planText), - `${params.turnId}:plan`, - ), - ); - } const commentaryMessages = params.runParams.config?.ui?.prefs?.chatPersistCommentary === false ? [] @@ -120,7 +111,6 @@ export function buildCodexSteeringMessagesSnapshot(params: { turnId: params.turnId, upstreamUserText: params.upstreamUserText, reasoningText: undefined, - planText: undefined, asyncMessages, commentaryMessages, toolMessages: params.toolMessages, diff --git a/extensions/codex/src/app-server/event-projector.reasoning.test.ts b/extensions/codex/src/app-server/event-projector.reasoning.test.ts index 4f375f009e9a..a33babb91453 100644 --- a/extensions/codex/src/app-server/event-projector.reasoning.test.ts +++ b/extensions/codex/src/app-server/event-projector.reasoning.test.ts @@ -531,15 +531,9 @@ describe("CodexAppServerEventProjector reasoning and guardian projection", () => }, ); expect(result.toolMetas).toEqual([{ toolName: "sessions_send", isError: false }]); - expect(result.messagesSnapshot.map((message) => message.role)).toEqual([ - "user", - "assistant", - "assistant", - ]); + expect(result.messagesSnapshot.map((message) => message.role)).toEqual(["user", "assistant"]); expect(JSON.stringify(result.messagesSnapshot[1])).toContain("Codex reasoning"); - expect(JSON.stringify(result.messagesSnapshot[2])).toContain("Codex plan"); - expect(JSON.stringify(result.messagesSnapshot[2])).toContain("next"); - expect(JSON.stringify(result.messagesSnapshot[2])).toContain("[in_progress] patch"); + expect(JSON.stringify(result.messagesSnapshot)).not.toContain("Codex plan:"); expect(result.compactionCount).toBe(1); expect(requireRecord(result.itemLifecycle, "item lifecycle")).not.toHaveProperty( "compactionCount", diff --git a/extensions/codex/src/app-server/event-projector.verbose-hooks.test.ts b/extensions/codex/src/app-server/event-projector.verbose-hooks.test.ts index 763f8ba06baa..1c4e30dc9452 100644 --- a/extensions/codex/src/app-server/event-projector.verbose-hooks.test.ts +++ b/extensions/codex/src/app-server/event-projector.verbose-hooks.test.ts @@ -337,7 +337,7 @@ describe("CodexAppServerEventProjector verbose output and hook projection", () = { step: "step two", status: "pending" }, ]); expect(result.assistantTexts).toEqual(["final answer"]); - expect(JSON.stringify(result.messagesSnapshot)).toContain("Codex plan"); + expect(JSON.stringify(result.messagesSnapshot)).not.toContain("Codex plan:"); }); it("fires before_compaction and after_compaction hooks for codex compaction items", async () => { diff --git a/extensions/codex/src/app-server/outcome-fallback-runtime-contract.test.ts b/extensions/codex/src/app-server/outcome-fallback-runtime-contract.test.ts index 67ed2fb6f2cb..dd957bbfcf4f 100644 --- a/extensions/codex/src/app-server/outcome-fallback-runtime-contract.test.ts +++ b/extensions/codex/src/app-server/outcome-fallback-runtime-contract.test.ts @@ -233,41 +233,8 @@ describe("Outcome/fallback runtime contract - Codex app-server adapter", () => { expect(result.assistantTexts).toStrictEqual([]); expect(result.lastAssistant).toBeUndefined(); expect(readAttemptTerminal(result).promptError).toBeNull(); - expect(result.messagesSnapshot.map((message) => message.role)).toStrictEqual([ - "user", - "assistant", - ]); - const planMessage = result.messagesSnapshot[1]; - if (planMessage?.role !== "assistant") { - throw new Error("expected Codex plan mirror assistant message"); - } - expect(readMirrorIdentity(planMessage)).toBe(`${TURN_ID}:plan`); - expect(planMessage.content).toStrictEqual([ - { - type: "text", - text: `Codex plan:\n${OUTCOME_FALLBACK_RUNTIME_CONTRACT.planningOnlyText}`, - }, - ]); - expect(planMessage.api).toBe("openai-chatgpt-responses"); - expect(planMessage.provider).toBe("codex"); - expect(planMessage.model).toBe(OUTCOME_FALLBACK_RUNTIME_CONTRACT.primaryModel); - expect(planMessage.usage).toStrictEqual({ - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 0, - cost: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - total: 0, - }, - }); - expect(planMessage.stopReason).toBe("stop"); - expect(typeof planMessage.timestamp).toBe("number"); - expect(planMessage.timestamp).toBeGreaterThan(0); + expect(result.messagesSnapshot.map((message) => message.role)).toStrictEqual(["user"]); + expect(result.agentHarnessResultClassification).toBe("planning-only"); }); it("preserves tool side-effect telemetry so fallback can stay disabled", async () => { @@ -354,6 +321,25 @@ describe("Outcome/fallback runtime contract - Codex app-server adapter", () => { return projector.buildResult(buildToolTelemetry()); }, }, + { + name: "structured planning-only", + classification: "planning-only", + expectedCode: "planning_only_result", + build: async () => { + const projector = await createProjector(); + await projector.handleNotification( + forCurrentTurn("turn/plan/updated", { + plan: [{ step: OUTCOME_FALLBACK_RUNTIME_CONTRACT.planningOnlyText, status: "pending" }], + }), + ); + await projector.handleNotification( + forCurrentTurn("turn/completed", { + turn: { id: TURN_ID, status: "completed", items: [] }, + }), + ); + return projector.buildResult(buildToolTelemetry()); + }, + }, ] as const)( "keeps $name terminal turns fallback-ready with adapter-produced classification", async ({ build, classification, expectedCode }) => { diff --git a/extensions/codex/src/app-server/run-attempt.test.ts b/extensions/codex/src/app-server/run-attempt.test.ts index 9fc875ef5385..1af7d4078921 100644 --- a/extensions/codex/src/app-server/run-attempt.test.ts +++ b/extensions/codex/src/app-server/run-attempt.test.ts @@ -76,6 +76,7 @@ import { } from "./protocol.js"; import { itemNotification, rawItemCompleted, turnCompleted } from "./protocol.test-helpers.js"; import { resolveCodexDynamicToolDirectNames } from "./run-attempt-tools.js"; +import { readMirrorIdentity } from "./upstream-prompt-provenance.js"; import * as userInputBridge from "./user-input-bridge.js"; type CodexAppServerToolTelemetry = Parameters[0]; @@ -2613,7 +2614,19 @@ describe("runCodexAppServerAttempt", () => { ).toHaveLength(2); await harness.completeTurn({ threadId: "thread-1", turnId: "turn-1" }); - await run; + const result = await run; + expect(persistedProgressCardInputs[0]).toEqual({ markdown: "Plan restored", plan }); + expect(result.messagesSnapshot).toContainEqual( + expect.objectContaining({ + role: "toolResult", + toolName: "progress_card", + isError: false, + }), + ); + expect( + result.messagesSnapshot.some((message) => readMirrorIdentity(message) === "turn-1:plan"), + ).toBe(false); + expect(JSON.stringify(result.messagesSnapshot)).not.toContain("Codex plan:"); }); it("does not inject plan state after compaction when the turn has no plan", async () => { diff --git a/extensions/qa-lab/src/scenario-flow-runner.test.ts b/extensions/qa-lab/src/scenario-flow-runner.test.ts index b7ce5d1b641f..92a348f6140f 100644 --- a/extensions/qa-lab/src/scenario-flow-runner.test.ts +++ b/extensions/qa-lab/src/scenario-flow-runner.test.ts @@ -129,7 +129,11 @@ function readCurrentRunProviderPromptEvidenceFlow(trajectoryEvents: unknown[]): }; } -const planningEvidenceCoverageIds = new Set(["runtime.no-meta-leak", "workspace.planning"]); +const planningEvidenceCoverageIds = new Set([ + "agent-runtime.external-harness-selection-planning", + "openai.codex-harness-no-meta-leak", + "openai.codex-harness-planning", +]); type PlanningEvidenceScenario = QaSeedScenarioWithSource & { execution: Extract & { flow?: QaScenarioFlow }; @@ -200,14 +204,11 @@ function createPlanningEvidenceFixture( return { scenario, outboundText: expectedReply, - failureMessage: "missing marked Codex internal plan/reasoning mirror evidence", + failureMessage: "missing successful current-attempt progress_card update", currentSummary: { eventCursor: 9, - assistantMirrors: [ - { identity: "current-turn:plan", text: `Codex plan:\n${internalMarker}` }, - { identity: "current-turn:assistant", text: expectedReply }, - ], - successfulToolCallCounts: {}, + assistantMirrors: [{ identity: "current-turn:assistant", text: expectedReply }], + successfulToolCallCounts: { progress_card: 1 }, }, }; } @@ -216,14 +217,11 @@ function createPlanningEvidenceFixture( return { scenario, outboundText, - failureMessage: "missing Codex App Server plan signal", + failureMessage: "missing Codex harness progress_card signal", currentSummary: { eventCursor: 9, - assistantMirrors: [ - { identity: "current-turn:plan", text: "Codex plan:\n- build the game" }, - { identity: "current-turn:assistant", text: outboundText }, - ], - successfulToolCallCounts: {}, + assistantMirrors: [{ identity: "current-turn:assistant", text: outboundText }], + successfulToolCallCounts: { progress_card: 1 }, }, }; } @@ -250,15 +248,13 @@ function runPlanningEvidenceFixture( const summaries = [ { eventCursor: 7, - assistantMirrors: [ - { identity: "old-turn:plan", text: "Codex plan:\nQA_INTERNAL_PLAN_DO_NOT_SEND" }, - { identity: "old-turn:assistant", text: fixture.outboundText }, - ], + assistantMirrors: [{ identity: "old-turn:assistant", text: fixture.outboundText }], successfulToolCallCounts: { progress_card: 1 }, }, currentSummary, ]; let readIndex = 0; + const cardStep = fixture.scenario.execution.config?.internalMarker; const result = runLoadedScenarioFlow(fixture.scenario.id, { flow: readPlanningEvidenceFlow(fixture.scenario), state, @@ -273,6 +269,12 @@ function runPlanningEvidenceFixture( env: { providerMode: "live-frontier", primaryModel: "openai/gpt-5.6-luna", + gateway: { + call: async (method: string) => + method === "progressCard.get" + ? { card: { revision: 1, steps: [{ step: cardStep }] } } + : { messages: [{ role: "assistant", content: fixture.outboundText }] }, + }, }, readSessionTranscriptSummary: async (...args: unknown[]) => { readOptions.push(args[2]); diff --git a/extensions/qa-lab/src/suite-runtime-agent-session.test.ts b/extensions/qa-lab/src/suite-runtime-agent-session.test.ts index 92214fe2c8cb..7e27b485b976 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-session.test.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-session.test.ts @@ -479,8 +479,8 @@ describe("qa suite runtime agent session helpers", () => { sessionId: "session-mirrors", message: { role: "assistant", - content: "Codex plan:\n- inspect\n- build", - __openclaw: { mirrorIdentity: "turn-123:plan" }, + content: "Checking the workspace.", + __openclaw: { mirrorIdentity: "turn-123:commentary:message-1" }, }, }); @@ -494,8 +494,8 @@ describe("qa suite runtime agent session helpers", () => { ).resolves.toMatchObject({ assistantMirrors: [ { - identity: "turn-123:plan", - text: "Codex plan:\n- inspect\n- build", + identity: "turn-123:commentary:message-1", + text: "Checking the workspace.", }, ], }); @@ -512,8 +512,8 @@ describe("qa suite runtime agent session helpers", () => { message: { role: "assistant", content: [ - { type: "toolCall", id: "plan-ok", name: "update_plan", arguments: {} }, - { type: "toolCall", id: "plan-error", name: "update_plan", arguments: {} }, + { type: "toolCall", id: "plan-ok", name: "progress_card", arguments: {} }, + { type: "toolCall", id: "plan-error", name: "progress_card", arguments: {} }, { type: "toolCall", id: "write-mismatch", name: "write", arguments: {} }, ], }, @@ -522,15 +522,15 @@ describe("qa suite runtime agent session helpers", () => { { role: "toolResult", toolCallId: "plan-ok", - toolName: "update_plan", - content: [{ type: "text", text: "Plan updated" }], + toolName: "progress_card", + content: [{ type: "text", text: "Progress card updated" }], isError: false, timestamp: 100, }, { role: "toolResult", toolCallId: "plan-ok", - toolName: "update_plan", + toolName: "progress_card", content: [{ type: "text", text: "duplicate" }], isError: false, timestamp: 200, @@ -538,7 +538,7 @@ describe("qa suite runtime agent session helpers", () => { { role: "toolResult", toolCallId: "plan-error", - toolName: "update_plan", + toolName: "progress_card", content: [{ type: "text", text: "failed" }], isError: true, timestamp: 300, @@ -568,10 +568,10 @@ describe("qa suite runtime agent session helpers", () => { sessionKey, ), ).resolves.toMatchObject({ - assistantToolCallCounts: { update_plan: 2, write: 1 }, - completedToolCallCounts: { update_plan: 2 }, - successfulToolCallCounts: { update_plan: 1 }, - successfulToolCallEvents: [{ name: "update_plan", timestamp: 100, toolCallId: "plan-ok" }], + assistantToolCallCounts: { progress_card: 2, write: 1 }, + completedToolCallCounts: { progress_card: 2 }, + successfulToolCallCounts: { progress_card: 1 }, + successfulToolCallEvents: [{ name: "progress_card", timestamp: 100, toolCallId: "plan-ok" }], }); }); @@ -807,13 +807,13 @@ describe("qa suite runtime agent session helpers", () => { for (const message of [ { role: "assistant", - content: [{ type: "toolCall", id: "old-plan", name: "update_plan", arguments: {} }], + content: [{ type: "toolCall", id: "old-plan", name: "progress_card", arguments: {} }], }, { role: "toolResult", toolCallId: "old-plan", - toolName: "update_plan", - content: [{ type: "text", text: "Plan updated" }], + toolName: "progress_card", + content: [{ type: "text", text: "Progress card updated" }], isError: false, timestamp: 100, }, @@ -830,7 +830,7 @@ describe("qa suite runtime agent session helpers", () => { sessionKey, ); expect(checkpoint.successfulToolCallEvents).toEqual([ - { name: "update_plan", timestamp: 100, toolCallId: "old-plan" }, + { name: "progress_card", timestamp: 100, toolCallId: "old-plan" }, ]); await appendQaTranscriptMessage({ tempRoot, diff --git a/qa/scenarios/models/codex-harness-no-meta-leak.yaml b/qa/scenarios/models/codex-harness-no-meta-leak.yaml index e4624e088ae0..4a15c57fbefd 100644 --- a/qa/scenarios/models/codex-harness-no-meta-leak.yaml +++ b/qa/scenarios/models/codex-harness-no-meta-leak.yaml @@ -36,8 +36,8 @@ scenario: expectedReply: QA_LEAK_OK internalMarker: QA_INTERNAL_PLAN_DO_NOT_SEND prompt: |- - First call update_plan with one pending step whose text is exactly QA_INTERNAL_PLAN_DO_NOT_SEND. - Keep that plan and any reasoning or coordination text internal. + First call progress_card with one pending step whose text is exactly QA_INTERNAL_PLAN_DO_NOT_SEND. + Keep that plan and any reasoning or coordination text out of your visible reply. Reply with exactly one line: QA_LEAK_OK @@ -115,12 +115,24 @@ flow: - set: appServerTurn value: expr: "assistantMirror.identity.slice(0, -':assistant'.length)" + - assert: + expr: "(transcript.successfulToolCallCounts.progress_card ?? 0) > 0" + message: missing successful current-attempt progress_card update + - set: progressCard + value: + expr: "(await env.gateway.call('progressCard.get', { sessionKey: 'agent:qa:codex-meta-leak' })).card" + - assert: + expr: "progressCard?.steps?.some((step) => step.step === config.internalMarker)" + message: missing marked durable progress_card evidence - set: internalMirrors value: - expr: "(transcript.assistantMirrors ?? []).filter((mirror) => mirror.identity.startsWith(appServerTurn + ':') && (mirror.identity.endsWith(':reasoning') || mirror.identity.endsWith(':plan')))" + expr: "(transcript.assistantMirrors ?? []).filter((mirror) => mirror.identity === appServerTurn + ':reasoning')" + - set: visibleHistory + value: + expr: "(await env.gateway.call('chat.history', { sessionKey: 'agent:qa:codex-meta-leak', limit: 100 })).messages ?? []" - assert: - expr: "internalMirrors.some((mirror) => mirror.text.includes(config.internalMarker))" - message: missing marked Codex internal plan/reasoning mirror evidence + expr: "!visibleHistory.some((message) => message.role === 'assistant' && (typeof message.content === 'string' ? message.content : (message.content ?? []).filter((part) => part.type === 'text').map((part) => part.text).join('\\n')).startsWith('Codex plan:\\n'))" + message: Codex plan leaked into visible Gateway history - set: outboundLower value: expr: normalizeLowercaseStringOrEmpty(outbound.text) @@ -150,4 +162,4 @@ flow: expr: "!outboundLower.includes(normalizeLowercaseStringOrEmpty(forbidden))" message: expr: "`visible reply leaked internal meta text (${forbidden}): ${outbound.text}`" - detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${config.harnessRuntime} agentRun=${turn.started.runId} appServerTurn=${appServerTurn} internalMirrors=${internalMirrors.map((mirror) => mirror.identity).join(',')} planSignal=${internalMirrors.find((mirror) => mirror.identity.endsWith(':plan'))?.identity ?? 'n/a'} artifact=n/a visibleReply=${JSON.stringify(outbound.text)}`" + detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${config.harnessRuntime} agentRun=${turn.started.runId} appServerTurn=${appServerTurn} internalMirrors=${internalMirrors.map((mirror) => mirror.identity).join(',')} planSignal=progress_card:revision:${progressCard.revision} artifact=n/a visibleReply=${JSON.stringify(outbound.text)}`" diff --git a/qa/scenarios/workspace/medium-game-plan-codex-harness.yaml b/qa/scenarios/workspace/medium-game-plan-codex-harness.yaml index 60c642241753..0d9412cb4fe7 100644 --- a/qa/scenarios/workspace/medium-game-plan-codex-harness.yaml +++ b/qa/scenarios/workspace/medium-game-plan-codex-harness.yaml @@ -14,7 +14,7 @@ scenario: successCriteria: - A live-frontier run fails fast unless the selected primary model is openai/gpt-5.6-luna with the Codex harness forced. - The scenario forces the Codex embedded harness. - - The Codex App Server emits a turn-scoped plan signal before the artifact assertion passes. + - The Codex harness records a successful `progress_card` tool call before the artifact assertion passes. - The agent writes a self-contained HTML game with a canvas loop, controls, scoring, waves, pause, and restart. docsRefs: - docs/plugins/sdk-agent-harness.md @@ -37,7 +37,7 @@ scenario: gameTitle: Star Garden Defenders minBytes: 5000 buildPrompt: |- - Call update_plan with a short implementation plan before editing. + Call progress_card with a short implementation plan before editing. Then build a medium-complex, self-contained browser game at ./star-garden-defenders-codex.html. @@ -118,12 +118,9 @@ flow: - set: appServerTurn value: expr: "assistantMirror.identity.slice(0, -':assistant'.length)" - - set: planMirror - value: - expr: "transcript.assistantMirrors?.find((mirror) => mirror.identity === appServerTurn + ':plan' && mirror.text.startsWith('Codex plan:\\n'))" - assert: - expr: "planMirror?.identity" - message: missing Codex App Server plan signal + expr: "(transcript.successfulToolCallCounts.progress_card ?? 0) > 0" + message: missing Codex harness progress_card signal - set: artifactPath value: expr: "path.join(env.gateway.workspaceDir, config.artifactFile)" @@ -164,4 +161,4 @@ flow: expr: "outbound.text.includes(config.artifactFile)" message: expr: "`final reply did not mention ${config.artifactFile}: ${outbound.text}`" - detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${config.harnessRuntime} agentRun=${turn.started.runId} appServerTurn=${appServerTurn} planSignal=${planMirror.identity} artifact=${artifactPath} bytes=${artifact.length} visibleReply=${JSON.stringify(outbound.text)}`" + detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${config.harnessRuntime} agentRun=${turn.started.runId} appServerTurn=${appServerTurn} planSignal=progress_card:success:${transcript.successfulToolCallCounts.progress_card ?? 0} artifact=${artifactPath} bytes=${artifact.length} visibleReply=${JSON.stringify(outbound.text)}`"