fix(codex): keep progress cards out of chat history (#130022)

* fix(codex): keep progress cards out of chat history

* test(codex): simplify progress card regressions
This commit is contained in:
Peter Steinberger
2026-08-26 02:28:31 -07:00
committed by GitHub
parent 7da8c19fd0
commit ea49e77bbc
12 changed files with 99 additions and 108 deletions
@@ -103,8 +103,7 @@ export class CodexReasoningProjection {
.filter((part): part is string => Boolean(part))
.join("\n");
if (planText) {
// Structured turn updates are the canonical latest plan. Retain the last
// non-empty update so the terminal transcript proves planning occurred.
// Structured turn updates are the canonical latest plan for terminal classification.
this.turnPlanText = planText;
}
if (source === "codex-app-server" && plan) {
@@ -154,7 +154,6 @@ export function buildCodexAttemptResult(
turnId: input.turnId,
upstreamUserText: input.upstreamUserText,
reasoningText,
planText,
asyncMessages,
commentaryMessages,
toolMessages: input.toolTranscriptProjection.transcriptMessages,
@@ -16,7 +16,6 @@ function buildSnapshot(trigger: EmbeddedRunAttemptParams["trigger"]): AgentMessa
turnId: "turn-1",
upstreamUserText: undefined,
reasoningText: "checking memory",
planText: undefined,
asyncMessages: [],
commentaryMessages: [],
toolMessages: [
@@ -38,7 +38,6 @@ export function buildCodexMessagesSnapshot(params: {
turnId: string;
upstreamUserText: string | undefined;
reasoningText: string | undefined;
planText: string | undefined;
asyncMessages: ReadonlyArray<{ itemId: string; message: AssistantMessage }>;
commentaryMessages: ReadonlyArray<{ itemId: string; message: AssistantMessage }>;
toolMessages: readonly AgentMessage[];
@@ -55,14 +54,6 @@ export function buildCodexMessagesSnapshot(params: {
),
);
}
if (params.planText) {
messages.push(
attachCodexMirrorIdentity(
params.createAssistantMirrorMessage("Codex plan", params.planText),
`${params.turnId}:plan`,
),
);
}
const commentaryMessages =
params.runParams.config?.ui?.prefs?.chatPersistCommentary === false
? []
@@ -120,7 +111,6 @@ export function buildCodexSteeringMessagesSnapshot(params: {
turnId: params.turnId,
upstreamUserText: params.upstreamUserText,
reasoningText: undefined,
planText: undefined,
asyncMessages,
commentaryMessages,
toolMessages: params.toolMessages,
@@ -531,15 +531,9 @@ describe("CodexAppServerEventProjector reasoning and guardian projection", () =>
},
);
expect(result.toolMetas).toEqual([{ toolName: "sessions_send", isError: false }]);
expect(result.messagesSnapshot.map((message) => message.role)).toEqual([
"user",
"assistant",
"assistant",
]);
expect(result.messagesSnapshot.map((message) => message.role)).toEqual(["user", "assistant"]);
expect(JSON.stringify(result.messagesSnapshot[1])).toContain("Codex reasoning");
expect(JSON.stringify(result.messagesSnapshot[2])).toContain("Codex plan");
expect(JSON.stringify(result.messagesSnapshot[2])).toContain("next");
expect(JSON.stringify(result.messagesSnapshot[2])).toContain("[in_progress] patch");
expect(JSON.stringify(result.messagesSnapshot)).not.toContain("Codex plan:");
expect(result.compactionCount).toBe(1);
expect(requireRecord(result.itemLifecycle, "item lifecycle")).not.toHaveProperty(
"compactionCount",
@@ -337,7 +337,7 @@ describe("CodexAppServerEventProjector verbose output and hook projection", () =
{ step: "step two", status: "pending" },
]);
expect(result.assistantTexts).toEqual(["final answer"]);
expect(JSON.stringify(result.messagesSnapshot)).toContain("Codex plan");
expect(JSON.stringify(result.messagesSnapshot)).not.toContain("Codex plan:");
});
it("fires before_compaction and after_compaction hooks for codex compaction items", async () => {
@@ -233,41 +233,8 @@ describe("Outcome/fallback runtime contract - Codex app-server adapter", () => {
expect(result.assistantTexts).toStrictEqual([]);
expect(result.lastAssistant).toBeUndefined();
expect(readAttemptTerminal(result).promptError).toBeNull();
expect(result.messagesSnapshot.map((message) => message.role)).toStrictEqual([
"user",
"assistant",
]);
const planMessage = result.messagesSnapshot[1];
if (planMessage?.role !== "assistant") {
throw new Error("expected Codex plan mirror assistant message");
}
expect(readMirrorIdentity(planMessage)).toBe(`${TURN_ID}:plan`);
expect(planMessage.content).toStrictEqual([
{
type: "text",
text: `Codex plan:\n${OUTCOME_FALLBACK_RUNTIME_CONTRACT.planningOnlyText}`,
},
]);
expect(planMessage.api).toBe("openai-chatgpt-responses");
expect(planMessage.provider).toBe("codex");
expect(planMessage.model).toBe(OUTCOME_FALLBACK_RUNTIME_CONTRACT.primaryModel);
expect(planMessage.usage).toStrictEqual({
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
total: 0,
},
});
expect(planMessage.stopReason).toBe("stop");
expect(typeof planMessage.timestamp).toBe("number");
expect(planMessage.timestamp).toBeGreaterThan(0);
expect(result.messagesSnapshot.map((message) => message.role)).toStrictEqual(["user"]);
expect(result.agentHarnessResultClassification).toBe("planning-only");
});
it("preserves tool side-effect telemetry so fallback can stay disabled", async () => {
@@ -354,6 +321,25 @@ describe("Outcome/fallback runtime contract - Codex app-server adapter", () => {
return projector.buildResult(buildToolTelemetry());
},
},
{
name: "structured planning-only",
classification: "planning-only",
expectedCode: "planning_only_result",
build: async () => {
const projector = await createProjector();
await projector.handleNotification(
forCurrentTurn("turn/plan/updated", {
plan: [{ step: OUTCOME_FALLBACK_RUNTIME_CONTRACT.planningOnlyText, status: "pending" }],
}),
);
await projector.handleNotification(
forCurrentTurn("turn/completed", {
turn: { id: TURN_ID, status: "completed", items: [] },
}),
);
return projector.buildResult(buildToolTelemetry());
},
},
] as const)(
"keeps $name terminal turns fallback-ready with adapter-produced classification",
async ({ build, classification, expectedCode }) => {
@@ -76,6 +76,7 @@ import {
} from "./protocol.js";
import { itemNotification, rawItemCompleted, turnCompleted } from "./protocol.test-helpers.js";
import { resolveCodexDynamicToolDirectNames } from "./run-attempt-tools.js";
import { readMirrorIdentity } from "./upstream-prompt-provenance.js";
import * as userInputBridge from "./user-input-bridge.js";
type CodexAppServerToolTelemetry = Parameters<CodexAppServerEventProjector["buildResult"]>[0];
@@ -2613,7 +2614,19 @@ describe("runCodexAppServerAttempt", () => {
).toHaveLength(2);
await harness.completeTurn({ threadId: "thread-1", turnId: "turn-1" });
await run;
const result = await run;
expect(persistedProgressCardInputs[0]).toEqual({ markdown: "Plan restored", plan });
expect(result.messagesSnapshot).toContainEqual(
expect.objectContaining({
role: "toolResult",
toolName: "progress_card",
isError: false,
}),
);
expect(
result.messagesSnapshot.some((message) => readMirrorIdentity(message) === "turn-1:plan"),
).toBe(false);
expect(JSON.stringify(result.messagesSnapshot)).not.toContain("Codex plan:");
});
it("does not inject plan state after compaction when the turn has no plan", async () => {
@@ -129,7 +129,11 @@ function readCurrentRunProviderPromptEvidenceFlow(trajectoryEvents: unknown[]):
};
}
const planningEvidenceCoverageIds = new Set(["runtime.no-meta-leak", "workspace.planning"]);
const planningEvidenceCoverageIds = new Set([
"agent-runtime.external-harness-selection-planning",
"openai.codex-harness-no-meta-leak",
"openai.codex-harness-planning",
]);
type PlanningEvidenceScenario = QaSeedScenarioWithSource & {
execution: Extract<QaScenarioExecution, { kind: "flow" }> & { flow?: QaScenarioFlow };
@@ -200,14 +204,11 @@ function createPlanningEvidenceFixture(
return {
scenario,
outboundText: expectedReply,
failureMessage: "missing marked Codex internal plan/reasoning mirror evidence",
failureMessage: "missing successful current-attempt progress_card update",
currentSummary: {
eventCursor: 9,
assistantMirrors: [
{ identity: "current-turn:plan", text: `Codex plan:\n${internalMarker}` },
{ identity: "current-turn:assistant", text: expectedReply },
],
successfulToolCallCounts: {},
assistantMirrors: [{ identity: "current-turn:assistant", text: expectedReply }],
successfulToolCallCounts: { progress_card: 1 },
},
};
}
@@ -216,14 +217,11 @@ function createPlanningEvidenceFixture(
return {
scenario,
outboundText,
failureMessage: "missing Codex App Server plan signal",
failureMessage: "missing Codex harness progress_card signal",
currentSummary: {
eventCursor: 9,
assistantMirrors: [
{ identity: "current-turn:plan", text: "Codex plan:\n- build the game" },
{ identity: "current-turn:assistant", text: outboundText },
],
successfulToolCallCounts: {},
assistantMirrors: [{ identity: "current-turn:assistant", text: outboundText }],
successfulToolCallCounts: { progress_card: 1 },
},
};
}
@@ -250,15 +248,13 @@ function runPlanningEvidenceFixture(
const summaries = [
{
eventCursor: 7,
assistantMirrors: [
{ identity: "old-turn:plan", text: "Codex plan:\nQA_INTERNAL_PLAN_DO_NOT_SEND" },
{ identity: "old-turn:assistant", text: fixture.outboundText },
],
assistantMirrors: [{ identity: "old-turn:assistant", text: fixture.outboundText }],
successfulToolCallCounts: { progress_card: 1 },
},
currentSummary,
];
let readIndex = 0;
const cardStep = fixture.scenario.execution.config?.internalMarker;
const result = runLoadedScenarioFlow(fixture.scenario.id, {
flow: readPlanningEvidenceFlow(fixture.scenario),
state,
@@ -273,6 +269,12 @@ function runPlanningEvidenceFixture(
env: {
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.6-luna",
gateway: {
call: async (method: string) =>
method === "progressCard.get"
? { card: { revision: 1, steps: [{ step: cardStep }] } }
: { messages: [{ role: "assistant", content: fixture.outboundText }] },
},
},
readSessionTranscriptSummary: async (...args: unknown[]) => {
readOptions.push(args[2]);
@@ -479,8 +479,8 @@ describe("qa suite runtime agent session helpers", () => {
sessionId: "session-mirrors",
message: {
role: "assistant",
content: "Codex plan:\n- inspect\n- build",
__openclaw: { mirrorIdentity: "turn-123:plan" },
content: "Checking the workspace.",
__openclaw: { mirrorIdentity: "turn-123:commentary:message-1" },
},
});
@@ -494,8 +494,8 @@ describe("qa suite runtime agent session helpers", () => {
).resolves.toMatchObject({
assistantMirrors: [
{
identity: "turn-123:plan",
text: "Codex plan:\n- inspect\n- build",
identity: "turn-123:commentary:message-1",
text: "Checking the workspace.",
},
],
});
@@ -512,8 +512,8 @@ describe("qa suite runtime agent session helpers", () => {
message: {
role: "assistant",
content: [
{ type: "toolCall", id: "plan-ok", name: "update_plan", arguments: {} },
{ type: "toolCall", id: "plan-error", name: "update_plan", arguments: {} },
{ type: "toolCall", id: "plan-ok", name: "progress_card", arguments: {} },
{ type: "toolCall", id: "plan-error", name: "progress_card", arguments: {} },
{ type: "toolCall", id: "write-mismatch", name: "write", arguments: {} },
],
},
@@ -522,15 +522,15 @@ describe("qa suite runtime agent session helpers", () => {
{
role: "toolResult",
toolCallId: "plan-ok",
toolName: "update_plan",
content: [{ type: "text", text: "Plan updated" }],
toolName: "progress_card",
content: [{ type: "text", text: "Progress card updated" }],
isError: false,
timestamp: 100,
},
{
role: "toolResult",
toolCallId: "plan-ok",
toolName: "update_plan",
toolName: "progress_card",
content: [{ type: "text", text: "duplicate" }],
isError: false,
timestamp: 200,
@@ -538,7 +538,7 @@ describe("qa suite runtime agent session helpers", () => {
{
role: "toolResult",
toolCallId: "plan-error",
toolName: "update_plan",
toolName: "progress_card",
content: [{ type: "text", text: "failed" }],
isError: true,
timestamp: 300,
@@ -568,10 +568,10 @@ describe("qa suite runtime agent session helpers", () => {
sessionKey,
),
).resolves.toMatchObject({
assistantToolCallCounts: { update_plan: 2, write: 1 },
completedToolCallCounts: { update_plan: 2 },
successfulToolCallCounts: { update_plan: 1 },
successfulToolCallEvents: [{ name: "update_plan", timestamp: 100, toolCallId: "plan-ok" }],
assistantToolCallCounts: { progress_card: 2, write: 1 },
completedToolCallCounts: { progress_card: 2 },
successfulToolCallCounts: { progress_card: 1 },
successfulToolCallEvents: [{ name: "progress_card", timestamp: 100, toolCallId: "plan-ok" }],
});
});
@@ -807,13 +807,13 @@ describe("qa suite runtime agent session helpers", () => {
for (const message of [
{
role: "assistant",
content: [{ type: "toolCall", id: "old-plan", name: "update_plan", arguments: {} }],
content: [{ type: "toolCall", id: "old-plan", name: "progress_card", arguments: {} }],
},
{
role: "toolResult",
toolCallId: "old-plan",
toolName: "update_plan",
content: [{ type: "text", text: "Plan updated" }],
toolName: "progress_card",
content: [{ type: "text", text: "Progress card updated" }],
isError: false,
timestamp: 100,
},
@@ -830,7 +830,7 @@ describe("qa suite runtime agent session helpers", () => {
sessionKey,
);
expect(checkpoint.successfulToolCallEvents).toEqual([
{ name: "update_plan", timestamp: 100, toolCallId: "old-plan" },
{ name: "progress_card", timestamp: 100, toolCallId: "old-plan" },
]);
await appendQaTranscriptMessage({
tempRoot,
@@ -36,8 +36,8 @@ scenario:
expectedReply: QA_LEAK_OK
internalMarker: QA_INTERNAL_PLAN_DO_NOT_SEND
prompt: |-
First call update_plan with one pending step whose text is exactly QA_INTERNAL_PLAN_DO_NOT_SEND.
Keep that plan and any reasoning or coordination text internal.
First call progress_card with one pending step whose text is exactly QA_INTERNAL_PLAN_DO_NOT_SEND.
Keep that plan and any reasoning or coordination text out of your visible reply.
Reply with exactly one line:
QA_LEAK_OK
@@ -115,12 +115,24 @@ flow:
- set: appServerTurn
value:
expr: "assistantMirror.identity.slice(0, -':assistant'.length)"
- assert:
expr: "(transcript.successfulToolCallCounts.progress_card ?? 0) > 0"
message: missing successful current-attempt progress_card update
- set: progressCard
value:
expr: "(await env.gateway.call('progressCard.get', { sessionKey: 'agent:qa:codex-meta-leak' })).card"
- assert:
expr: "progressCard?.steps?.some((step) => step.step === config.internalMarker)"
message: missing marked durable progress_card evidence
- set: internalMirrors
value:
expr: "(transcript.assistantMirrors ?? []).filter((mirror) => mirror.identity.startsWith(appServerTurn + ':') && (mirror.identity.endsWith(':reasoning') || mirror.identity.endsWith(':plan')))"
expr: "(transcript.assistantMirrors ?? []).filter((mirror) => mirror.identity === appServerTurn + ':reasoning')"
- set: visibleHistory
value:
expr: "(await env.gateway.call('chat.history', { sessionKey: 'agent:qa:codex-meta-leak', limit: 100 })).messages ?? []"
- assert:
expr: "internalMirrors.some((mirror) => mirror.text.includes(config.internalMarker))"
message: missing marked Codex internal plan/reasoning mirror evidence
expr: "!visibleHistory.some((message) => message.role === 'assistant' && (typeof message.content === 'string' ? message.content : (message.content ?? []).filter((part) => part.type === 'text').map((part) => part.text).join('\\n')).startsWith('Codex plan:\\n'))"
message: Codex plan leaked into visible Gateway history
- set: outboundLower
value:
expr: normalizeLowercaseStringOrEmpty(outbound.text)
@@ -150,4 +162,4 @@ flow:
expr: "!outboundLower.includes(normalizeLowercaseStringOrEmpty(forbidden))"
message:
expr: "`visible reply leaked internal meta text (${forbidden}): ${outbound.text}`"
detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${config.harnessRuntime} agentRun=${turn.started.runId} appServerTurn=${appServerTurn} internalMirrors=${internalMirrors.map((mirror) => mirror.identity).join(',')} planSignal=${internalMirrors.find((mirror) => mirror.identity.endsWith(':plan'))?.identity ?? 'n/a'} artifact=n/a visibleReply=${JSON.stringify(outbound.text)}`"
detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${config.harnessRuntime} agentRun=${turn.started.runId} appServerTurn=${appServerTurn} internalMirrors=${internalMirrors.map((mirror) => mirror.identity).join(',')} planSignal=progress_card:revision:${progressCard.revision} artifact=n/a visibleReply=${JSON.stringify(outbound.text)}`"
@@ -14,7 +14,7 @@ scenario:
successCriteria:
- A live-frontier run fails fast unless the selected primary model is openai/gpt-5.6-luna with the Codex harness forced.
- The scenario forces the Codex embedded harness.
- The Codex App Server emits a turn-scoped plan signal before the artifact assertion passes.
- The Codex harness records a successful `progress_card` tool call before the artifact assertion passes.
- The agent writes a self-contained HTML game with a canvas loop, controls, scoring, waves, pause, and restart.
docsRefs:
- docs/plugins/sdk-agent-harness.md
@@ -37,7 +37,7 @@ scenario:
gameTitle: Star Garden Defenders
minBytes: 5000
buildPrompt: |-
Call update_plan with a short implementation plan before editing.
Call progress_card with a short implementation plan before editing.
Then build a medium-complex, self-contained browser game at ./star-garden-defenders-codex.html.
@@ -118,12 +118,9 @@ flow:
- set: appServerTurn
value:
expr: "assistantMirror.identity.slice(0, -':assistant'.length)"
- set: planMirror
value:
expr: "transcript.assistantMirrors?.find((mirror) => mirror.identity === appServerTurn + ':plan' && mirror.text.startsWith('Codex plan:\\n'))"
- assert:
expr: "planMirror?.identity"
message: missing Codex App Server plan signal
expr: "(transcript.successfulToolCallCounts.progress_card ?? 0) > 0"
message: missing Codex harness progress_card signal
- set: artifactPath
value:
expr: "path.join(env.gateway.workspaceDir, config.artifactFile)"
@@ -164,4 +161,4 @@ flow:
expr: "outbound.text.includes(config.artifactFile)"
message:
expr: "`final reply did not mention ${config.artifactFile}: ${outbound.text}`"
detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${config.harnessRuntime} agentRun=${turn.started.runId} appServerTurn=${appServerTurn} planSignal=${planMirror.identity} artifact=${artifactPath} bytes=${artifact.length} visibleReply=${JSON.stringify(outbound.text)}`"
detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${config.harnessRuntime} agentRun=${turn.started.runId} appServerTurn=${appServerTurn} planSignal=progress_card:success:${transcript.successfulToolCallCounts.progress_card ?? 0} artifact=${artifactPath} bytes=${artifact.length} visibleReply=${JSON.stringify(outbound.text)}`"