diff --git a/extensions/codex/src/app-server/event-projector-reasoning.ts b/extensions/codex/src/app-server/event-projector-reasoning.ts index b59e7f059831..f1d99dc73198 100644 --- a/extensions/codex/src/app-server/event-projector-reasoning.ts +++ b/extensions/codex/src/app-server/event-projector-reasoning.ts @@ -23,6 +23,7 @@ export class CodexReasoningProjection { private readonly reasoningTextByGroup = new Map(); private readonly reasoningItemOrder = new Map(); private readonly planTextByItem = new Map(); + private turnPlanText: string | undefined; private reasoningStarted = false; private reasoningEnded = false; @@ -75,6 +76,7 @@ export class CodexReasoningProjection { } handleTurnPlanUpdated(params: JsonObject): void { + const explanation = readNullableString(params, "explanation"); const plan = Array.isArray(params.plan) ? params.plan.flatMap((entry) => { if (!entry || typeof entry !== "object" || Array.isArray(entry)) { @@ -88,8 +90,19 @@ export class CodexReasoningProjection { return [{ step, status: normalizePlanStepStatus(readString(record, "status")) }]; }) : undefined; + const planText = [ + explanation, + ...(plan ?? []).map(({ step, status }) => `- [${status}] ${step}`), + ] + .filter((part): part is string => Boolean(part)) + .join("\n"); + if (planText) { + // Structured turn updates are the canonical latest plan. Retain the last + // non-empty update so the terminal transcript proves planning occurred. + this.turnPlanText = planText; + } this.emitPlanUpdate({ - explanation: readNullableString(params, "explanation"), + explanation, steps: plan, }); } @@ -119,7 +132,10 @@ export class CodexReasoningProjection { } planText(): string { - return [...this.planTextByItem.values()].filter((text) => text.trim().length > 0).join("\n\n"); + return ( + this.turnPlanText ?? + [...this.planTextByItem.values()].filter((text) => text.trim().length > 0).join("\n\n") + ); } private emitPlanUpdate(params: { explanation?: string | null; steps?: AgentPlanStep[] }): void { diff --git a/extensions/codex/src/app-server/event-projector.test.ts b/extensions/codex/src/app-server/event-projector.test.ts index bff4ba07ec22..3e8ae5f61661 100644 --- a/extensions/codex/src/app-server/event-projector.test.ts +++ b/extensions/codex/src/app-server/event-projector.test.ts @@ -2996,6 +2996,8 @@ describe("CodexAppServerEventProjector", () => { ]); expect(JSON.stringify(result.messagesSnapshot[1])).toContain("Codex reasoning"); expect(JSON.stringify(result.messagesSnapshot[2])).toContain("Codex plan"); + expect(JSON.stringify(result.messagesSnapshot[2])).toContain("next"); + expect(JSON.stringify(result.messagesSnapshot[2])).toContain("[in_progress] patch"); expect(requireRecord(result.itemLifecycle, "item lifecycle").compactionCount).toBe(1); expect(onContextCompacted).toHaveBeenCalledOnce(); }); diff --git a/extensions/qa-lab/src/scenario-catalog.test.ts b/extensions/qa-lab/src/scenario-catalog.test.ts index 0f798377e81c..6afc7224cf28 100644 --- a/extensions/qa-lab/src/scenario-catalog.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.test.ts @@ -140,14 +140,6 @@ describe("qa scenario catalog", () => { it("loads scenario-specific execution config from per-scenario YAML", () => { const discovery = readQaScenarioById("source-docs-discovery-report"); const discoveryConfig = readQaScenarioExecutionConfig("source-docs-discovery-report"); - const codexLeak = readQaScenarioById("codex-harness-no-meta-leak"); - const codexLeakConfig = readQaScenarioExecutionConfig("codex-harness-no-meta-leak") as - | { - harnessRuntime?: string; - expectedReply?: string; - forbiddenReplySubstrings?: string[]; - } - | undefined; const fallbackConfig = readQaScenarioExecutionConfig("memory-failure-fallback"); const bundledSkill = readQaScenarioById("bundled-plugin-skill-runtime"); const bundledSkillConfig = readQaScenarioExecutionConfig("bundled-plugin-skill-runtime") as @@ -161,12 +153,6 @@ describe("qa scenario catalog", () => { expect((discoveryConfig?.requiredFiles as string[] | undefined)?.[0]).toBe( "repo/qa/scenarios/index.yaml", ); - expect(codexLeak.title).toBe("Codex harness no meta leak"); - expect(codexLeakConfig?.harnessRuntime).toBe("codex"); - expect(JSON.stringify(codexLeak.execution.flow)).toContain("agentRuntime"); - expect(JSON.stringify(codexLeak.execution.flow)).not.toContain("embeddedHarness"); - expect(codexLeakConfig?.expectedReply).toBe("QA_LEAK_OK"); - expect(codexLeakConfig?.forbiddenReplySubstrings).toContain("checking thread context"); expect(fallbackConfig?.gracefulFallbackAny as string[] | undefined).toContain( "will not reveal", ); diff --git a/extensions/qa-lab/src/scenario-flow-runner.test.ts b/extensions/qa-lab/src/scenario-flow-runner.test.ts index 3723179cd7a4..3822bc1107fd 100644 --- a/extensions/qa-lab/src/scenario-flow-runner.test.ts +++ b/extensions/qa-lab/src/scenario-flow-runner.test.ts @@ -1,7 +1,13 @@ // Qa Lab tests cover scenario flow runner plugin behavior. import { describe, expect, it } from "vitest"; import { createQaBusState } from "./bus-state.js"; -import { readQaScenarioById, type QaScenarioFlow } from "./scenario-catalog.js"; +import { + readQaScenarioById, + readQaScenarioPack, + type QaScenarioExecution, + type QaScenarioFlow, + type QaSeedScenarioWithSource, +} from "./scenario-catalog.js"; import { runScenarioFlow } from "./scenario-flow-runner.js"; type QaFlowStep = { @@ -207,7 +213,205 @@ async function runWebchatTranscriptWait( }); } +const planningEvidenceCoverageIds = new Set(["runtime.no-meta-leak", "workspace.planning"]); + +type PlanningEvidenceScenario = QaSeedScenarioWithSource & { + execution: Extract & { flow?: QaScenarioFlow }; +}; + +function isPlanningEvidenceScenario( + scenario: QaSeedScenarioWithSource, +): scenario is PlanningEvidenceScenario { + return ( + scenario.execution.kind === "flow" && + [...(scenario.coverage?.primary ?? []), ...(scenario.coverage?.secondary ?? [])].some( + (coverageId) => planningEvidenceCoverageIds.has(coverageId), + ) + ); +} + +type PlanningEvidenceFixture = { + currentSummary: Record; + failureMessage: string; + outboundText: string; + scenario: PlanningEvidenceScenario; +}; + +function readPlanningEvidenceFlow(scenario: PlanningEvidenceScenario): QaScenarioFlow { + const step = scenario.execution.flow?.steps.find((candidate) => + candidate.actions.some( + (action) => + typeof action === "object" && + action !== null && + "call" in action && + action.call === "runAgentPrompt", + ), + ); + if (!step) { + throw new Error(`planning scenario has no agent turn: ${scenario.id}`); + } + const artifactIndex = step.actions.findIndex( + (action) => + typeof action === "object" && + action !== null && + "set" in action && + action.set === "artifactPath", + ); + const evidenceActions = artifactIndex >= 0 ? step.actions.slice(0, artifactIndex) : step.actions; + return { + steps: [ + { + name: "proves current-attempt planning evidence", + actions: [ + { set: "selected", value: { provider: "openai", model: "gpt-5.6-luna" } }, + ...evidenceActions, + ], + }, + ], + }; +} + +function createPlanningEvidenceFixture( + scenario: PlanningEvidenceScenario, +): PlanningEvidenceFixture { + const config = scenario.execution.config ?? {}; + const artifactFile = typeof config.artifactFile === "string" ? config.artifactFile : undefined; + const expectedReply = typeof config.expectedReply === "string" ? config.expectedReply : undefined; + const internalMarker = + typeof config.internalMarker === "string" ? config.internalMarker : undefined; + + if (scenario.execution.runtime === "codex" && expectedReply && internalMarker) { + return { + scenario, + outboundText: expectedReply, + failureMessage: "missing marked Codex internal plan/reasoning mirror evidence", + currentSummary: { + eventCursor: 9, + assistantMirrors: [ + { identity: "current-turn:plan", text: `Codex plan:\n${internalMarker}` }, + { identity: "current-turn:assistant", text: expectedReply }, + ], + successfulToolCallCounts: {}, + }, + }; + } + if (scenario.execution.runtime === "codex" && artifactFile) { + const outboundText = `Built ${artifactFile}`; + return { + scenario, + outboundText, + failureMessage: "missing Codex App Server plan signal", + currentSummary: { + eventCursor: 9, + assistantMirrors: [ + { identity: "current-turn:plan", text: "Codex plan:\n- build the game" }, + { identity: "current-turn:assistant", text: outboundText }, + ], + successfulToolCallCounts: {}, + }, + }; + } + if (scenario.execution.runtime === "openclaw" && artifactFile) { + return { + scenario, + outboundText: `Built ${artifactFile}`, + failureMessage: "missing OpenClaw update_plan signal", + currentSummary: { + eventCursor: 9, + successfulToolCallCounts: { update_plan: 1 }, + }, + }; + } + throw new Error(`unsupported planning evidence metadata: ${scenario.id}`); +} + +function runPlanningEvidenceFixture( + fixture: PlanningEvidenceFixture, + currentSummary = fixture.currentSummary, +) { + const state = createQaBusState(); + const readOptions: unknown[] = []; + const summaries = [ + { + eventCursor: 7, + assistantMirrors: [ + { identity: "old-turn:plan", text: "Codex plan:\nQA_INTERNAL_PLAN_DO_NOT_SEND" }, + { identity: "old-turn:assistant", text: fixture.outboundText }, + ], + successfulToolCallCounts: { update_plan: 1 }, + }, + currentSummary, + ]; + let readIndex = 0; + const result = runLoadedScenarioFlow(fixture.scenario.id, { + flow: readPlanningEvidenceFlow(fixture.scenario), + state, + onWaitForOutboundMessage: ({ state: currentState }) => { + currentState.addOutboundMessage({ + accountId: "qa-channel", + to: "dm:qa-operator", + text: fixture.outboundText, + }); + }, + api: { + env: { + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.6-luna", + }, + readSessionTranscriptSummary: async (...args: unknown[]) => { + readOptions.push(args[2]); + const summary = summaries[readIndex]; + readIndex += 1; + if (!summary) { + throw new Error("unexpected transcript summary read"); + } + return summary; + }, + resolveQaLiveTurnTimeoutMs: (_env: unknown, timeoutMs: number) => timeoutMs, + normalizeLowercaseStringOrEmpty: (value: unknown) => + typeof value === "string" ? value.trim().toLowerCase() : "", + runAgentPrompt: async () => ({ started: { runId: "current-run" }, waited: { status: "ok" } }), + }, + }); + return { readOptions, result }; +} + +const planningEvidenceFixtures = readQaScenarioPack() + .scenarios.filter(isPlanningEvidenceScenario) + .map(createPlanningEvidenceFixture); + describe("scenario-flow-runner", () => { + it.each(planningEvidenceFixtures)( + "accepts current-attempt planning evidence for $scenario.id", + async (fixture) => { + const { readOptions, result } = runPlanningEvidenceFixture(fixture); + + await expect(result).resolves.toMatchObject({ status: "pass" }); + expect(readOptions).toEqual([{ allowEmpty: true }, { afterEventCursor: 7 }]); + }, + ); + + it.each(planningEvidenceFixtures)( + "rejects stale prior-attempt planning evidence for $scenario.id", + async (fixture) => { + const currentSummary = { + eventCursor: 8, + ...(fixture.scenario.execution.runtime === "codex" + ? { + assistantMirrors: [ + { identity: "current-turn:assistant", text: fixture.outboundText }, + ], + } + : {}), + successfulToolCallCounts: {}, + }; + const { readOptions, result } = runPlanningEvidenceFixture(fixture, currentSummary); + + await expect(result).rejects.toThrow(fixture.failureMessage); + expect(readOptions).toEqual([{ allowEmpty: true }, { afterEventCursor: 7 }]); + }, + ); + it("runs the canonical reaction lifecycle with target-bound actions", async () => { const state = createQaBusState(); const actionTargets: unknown[] = []; diff --git a/extensions/qa-lab/src/scenario-lane.test.ts b/extensions/qa-lab/src/scenario-lane.test.ts index e740ce0dc227..c8f6d74cd02f 100644 --- a/extensions/qa-lab/src/scenario-lane.test.ts +++ b/extensions/qa-lab/src/scenario-lane.test.ts @@ -1,5 +1,6 @@ // Qa Lab tests cover canonical scenario lane matching behavior. import { describe, expect, it } from "vitest"; +import { readQaScenarioPack } from "./scenario-catalog.js"; import { describeQaProviderLaneMismatches, scenarioMatchesQaProviderLane, @@ -7,6 +8,30 @@ import { import { makeQaSuiteTestScenario } from "./suite-test-helpers.js"; describe("QA scenario lane matching", () => { + const planningCoverageIds = new Set(["runtime.no-meta-leak", "workspace.planning"]); + const planningScenarios = readQaScenarioPack().scenarios.filter((scenario) => + [...(scenario.coverage?.primary ?? []), ...(scenario.coverage?.secondary ?? [])].some( + (coverageId) => planningCoverageIds.has(coverageId), + ), + ); + + it.each(planningScenarios)("selects $id for the GPT-5.6 Luna live lane", (scenario) => { + expect( + scenarioMatchesQaProviderLane({ + scenario, + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.6-luna", + }), + ).toBe(true); + expect( + scenarioMatchesQaProviderLane({ + scenario, + providerMode: "mock-openai", + primaryModel: "openai/gpt-5.6-luna", + }), + ).toBe(false); + }); + it("reports every declared mismatch in one decision", () => { const scenario = makeQaSuiteTestScenario("strict-live-lane", { channel: "matrix", diff --git a/extensions/qa-lab/src/suite-runtime-agent-session.test.ts b/extensions/qa-lab/src/suite-runtime-agent-session.test.ts index 187c015218ca..8ce70d0c565c 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-session.test.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-session.test.ts @@ -351,6 +351,8 @@ describe("qa suite runtime agent session helpers", () => { ), ).resolves.toEqual({ assistantToolCallCounts: { message: 1 }, + eventCursor: 2, + successfulToolCallCounts: {}, finalText: "", hasDirectReplySelfMessage: false, lastAssistantContentTypes: ["tool_use"], @@ -375,6 +377,8 @@ describe("qa suite runtime agent session helpers", () => { ), ).resolves.toEqual({ assistantToolCallCounts: { message: 1 }, + eventCursor: 3, + successfulToolCallCounts: {}, finalText: "Sent.", hasDirectReplySelfMessage: true, lastMessageRole: "assistant", @@ -427,6 +431,8 @@ describe("qa suite runtime agent session helpers", () => { ), ).resolves.toEqual({ assistantToolCallCounts: { message: 1 }, + eventCursor: 4, + successfulToolCallCounts: {}, finalText: "Sent.", hasDirectReplySelfMessage: true, lastAssistantErrorMessage: "Request was aborted", @@ -435,6 +441,174 @@ describe("qa suite runtime agent session helpers", () => { }); }); + it("reports provider-owned assistant mirror identities", async () => { + const tempRoot = await makeTempDir("qa-session-transcript-mirrors-"); + const sessionKey = "agent:qa:provider-mirrors"; + await seedQaSession({ tempRoot, sessionKey, sessionId: "session-mirrors" }); + await appendQaTranscriptMessage({ + tempRoot, + sessionKey, + sessionId: "session-mirrors", + message: { + role: "assistant", + content: "Codex plan:\n- inspect\n- build", + __openclaw: { mirrorIdentity: "turn-123:plan" }, + }, + }); + + await expect( + readSessionTranscriptSummary( + { + gateway: { tempRoot }, + } as never, + sessionKey, + ), + ).resolves.toMatchObject({ + assistantMirrors: [ + { + identity: "turn-123:plan", + text: "Codex plan:\n- inspect\n- build", + }, + ], + }); + }); + + it("counts only correlated non-error tool results as successful", async () => { + const tempRoot = await makeTempDir("qa-session-transcript-tool-results-"); + const sessionKey = "agent:qa:tool-results"; + await seedQaSession({ tempRoot, sessionKey, sessionId: "session-tool-results" }); + await appendQaTranscriptMessage({ + tempRoot, + sessionKey, + sessionId: "session-tool-results", + message: { + role: "assistant", + content: [ + { type: "toolCall", id: "plan-ok", name: "update_plan", arguments: {} }, + { type: "toolCall", id: "plan-error", name: "update_plan", arguments: {} }, + { type: "toolCall", id: "write-mismatch", name: "write", arguments: {} }, + ], + }, + }); + for (const message of [ + { + role: "toolResult", + toolCallId: "plan-ok", + toolName: "update_plan", + content: [{ type: "text", text: "Plan updated" }], + isError: false, + }, + { + role: "toolResult", + toolCallId: "plan-ok", + toolName: "update_plan", + content: [{ type: "text", text: "duplicate" }], + isError: false, + }, + { + role: "toolResult", + toolCallId: "plan-error", + toolName: "update_plan", + content: [{ type: "text", text: "failed" }], + isError: true, + }, + { + role: "toolResult", + toolCallId: "write-mismatch", + toolName: "exec", + content: [{ type: "text", text: "wrong tool" }], + isError: false, + }, + ]) { + await appendQaTranscriptMessage({ + tempRoot, + sessionKey, + sessionId: "session-tool-results", + message, + }); + } + + await expect( + readSessionTranscriptSummary( + { + gateway: { tempRoot }, + } as never, + sessionKey, + ), + ).resolves.toMatchObject({ + assistantToolCallCounts: { update_plan: 2, write: 1 }, + successfulToolCallCounts: { update_plan: 1 }, + }); + }); + + it("scopes transcript evidence after an event cursor", async () => { + const tempRoot = await makeTempDir("qa-session-transcript-cursor-"); + const sessionKey = "agent:qa:cursor"; + const sessionId = "session-cursor"; + await seedQaSession({ tempRoot, sessionKey, sessionId }); + for (const message of [ + { + role: "assistant", + content: [{ type: "toolCall", id: "old-plan", name: "update_plan", arguments: {} }], + }, + { + role: "toolResult", + toolCallId: "old-plan", + toolName: "update_plan", + content: [{ type: "text", text: "Plan updated" }], + isError: false, + }, + { + role: "assistant", + content: "same visible reply", + __openclaw: { mirrorIdentity: "old-turn:assistant" }, + }, + ]) { + await appendQaTranscriptMessage({ tempRoot, sessionKey, sessionId, message }); + } + const checkpoint = await readSessionTranscriptSummary( + { gateway: { tempRoot } } as never, + sessionKey, + ); + await appendQaTranscriptMessage({ + tempRoot, + sessionKey, + sessionId, + message: { + role: "assistant", + content: "same visible reply", + __openclaw: { mirrorIdentity: "current-turn:assistant" }, + }, + }); + + await expect( + readSessionTranscriptSummary({ gateway: { tempRoot } } as never, sessionKey, { + afterEventCursor: checkpoint.eventCursor, + }), + ).resolves.toMatchObject({ + assistantMirrors: [{ identity: "current-turn:assistant", text: "same visible reply" }], + assistantToolCallCounts: {}, + eventCursor: 5, + successfulToolCallCounts: {}, + }); + }); + + it("returns an empty checkpoint before the session exists", async () => { + const tempRoot = await makeTempDir("qa-session-transcript-checkpoint-"); + + await expect( + readSessionTranscriptSummary({ gateway: { tempRoot } } as never, "agent:qa:not-created-yet", { + allowEmpty: true, + }), + ).resolves.toEqual({ + assistantToolCallCounts: {}, + eventCursor: 0, + successfulToolCallCounts: {}, + finalText: "", + hasDirectReplySelfMessage: false, + }); + }); + it("fails closed when a requested QA session transcript is empty", async () => { const tempRoot = await makeTempDir("qa-session-transcript-empty-"); await seedQaSession({ diff --git a/extensions/qa-lab/src/suite-runtime-agent-session.ts b/extensions/qa-lab/src/suite-runtime-agent-session.ts index 6545dbfaec74..9a01b9c92030 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-session.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-session.ts @@ -46,7 +46,10 @@ const SESSION_STORE_LOCK_RETRY_DELAYS_MS = [1_000, 3_000, 5_000] as const; const SESSION_STORE_FTS_SETTLE_RETRY_DELAYS_MS = [100, 250, 500, 1_000, 2_000] as const; type QaSessionTranscriptSummary = { + assistantMirrors?: Array<{ identity: string; text: string }>; assistantToolCallCounts: Record; + eventCursor: number; + successfulToolCallCounts: Record; finalText: string; hasDirectReplySelfMessage: boolean; lastAssistantContentTypes?: string[]; @@ -56,6 +59,11 @@ type QaSessionTranscriptSummary = { lastMessageRole?: string; }; +type QaSessionTranscriptSummaryOptions = { + afterEventCursor?: number; + allowEmpty?: boolean; +}; + function isSessionStoreLockTimeout(error: unknown) { const text = formatErrorMessage(error); return ( @@ -81,7 +89,10 @@ function readSessionTranscriptEventMessage(event: unknown) { return isRecord(event) && isRecord(event.message) ? event.message : undefined; } -function readAssistantToolNames(message: Record): string[] { +function readAssistantToolCalls(message: Record): Array<{ + id?: string; + name: string; +}> { if (!Array.isArray(message.content)) { return []; } @@ -94,16 +105,21 @@ function readAssistantToolNames(message: Record): string[] { return []; } const name = readNonEmptyString(block.name); - return name ? [name] : []; + return name ? [{ id: readNonEmptyString(block.id), name }] : []; }); } function summarizeSessionTranscriptEvents( events: unknown[], sessionKey: string, + eventCursor = events.length, ): QaSessionTranscriptSummary { const scanner = createDirectReplyTranscriptSentinelScanner(); + const assistantMirrors: Array<{ identity: string; text: string }> = []; const assistantToolCallCounts: Record = {}; + const successfulToolCallCounts: Record = {}; + const assistantToolNamesByCallId = new Map(); + const successfulToolCallIds = new Set(); let finalText = ""; let lastAssistantContentTypes: string[] = []; let lastAssistantErrorMessage: string | undefined; @@ -117,6 +133,21 @@ function summarizeSessionTranscriptEvents( continue; } lastMessageRole = readNonEmptyString(message.role); + if (message.role === "toolResult") { + const toolCallId = readNonEmptyString(message.toolCallId); + const toolName = readNonEmptyString(message.toolName); + if ( + toolCallId && + toolName && + message.isError === false && + assistantToolNamesByCallId.get(toolCallId) === toolName && + !successfulToolCallIds.has(toolCallId) + ) { + successfulToolCallIds.add(toolCallId); + successfulToolCallCounts[toolName] = (successfulToolCallCounts[toolName] ?? 0) + 1; + } + continue; + } if (message.role !== "assistant") { continue; } @@ -124,6 +155,11 @@ function summarizeSessionTranscriptEvents( if (text) { finalText = text; } + const openClawMeta = isRecord(message["__openclaw"]) ? message["__openclaw"] : undefined; + const mirrorIdentity = readNonEmptyString(openClawMeta?.mirrorIdentity); + if (mirrorIdentity && text) { + assistantMirrors.push({ identity: mirrorIdentity, text }); + } lastAssistantContentTypes = Array.isArray(message.content) ? message.content.flatMap((block) => { const type = isRecord(block) ? readNonEmptyString(block.type) : undefined; @@ -132,9 +168,13 @@ function summarizeSessionTranscriptEvents( : []; lastAssistantErrorMessage = readNonEmptyString(message.errorMessage); lastAssistantStopReason = readNonEmptyString(message.stopReason); - lastAssistantToolNames = readAssistantToolNames(message); - for (const toolName of lastAssistantToolNames) { - assistantToolCallCounts[toolName] = (assistantToolCallCounts[toolName] ?? 0) + 1; + const assistantToolCalls = readAssistantToolCalls(message); + lastAssistantToolNames = assistantToolCalls.map((toolCall) => toolCall.name); + for (const toolCall of assistantToolCalls) { + assistantToolCallCounts[toolCall.name] = (assistantToolCallCounts[toolCall.name] ?? 0) + 1; + if (toolCall.id) { + assistantToolNamesByCallId.set(toolCall.id, toolCall.name); + } } scanner.recordMessage(message); } @@ -144,7 +184,10 @@ function summarizeSessionTranscriptEvents( } return { + ...(assistantMirrors.length > 0 ? { assistantMirrors } : {}), assistantToolCallCounts, + eventCursor, + successfulToolCallCounts, finalText, hasDirectReplySelfMessage: scanner.findings().length > 0, ...(lastAssistantContentTypes.length > 0 ? { lastAssistantContentTypes } : {}), @@ -155,6 +198,16 @@ function summarizeSessionTranscriptEvents( }; } +function emptySessionTranscriptSummary(eventCursor: number): QaSessionTranscriptSummary { + return { + assistantToolCallCounts: {}, + eventCursor, + successfulToolCallCounts: {}, + finalText: "", + hasDirectReplySelfMessage: false, + }; +} + async function callGatewayWithSessionStoreLockRetry( env: QaGatewayCallEnv, method: string, @@ -329,6 +382,7 @@ async function readRawQaSessionStore( async function readSessionTranscriptSummary( env: Pick, sessionKey: string, + options: QaSessionTranscriptSummaryOptions = {}, ): Promise { const normalizedSessionKey = sessionKey.trim(); if (!normalizedSessionKey) { @@ -338,17 +392,32 @@ async function readSessionTranscriptSummary( const entry = store[normalizedSessionKey]; const sessionId = readNonEmptyString(entry?.sessionId); if (!sessionId) { + if (options.allowEmpty === true) { + return emptySessionTranscriptSummary(0); + } throw new Error(`session transcript entry not found for ${normalizedSessionKey}`); } - return summarizeSessionTranscriptEvents( - loadTranscriptEventsSync({ - agentId: "qa", - env: qaSessionRuntimeEnv(env.gateway.tempRoot), - sessionId, - sessionKey: normalizedSessionKey, - }), - normalizedSessionKey, - ); + const events = loadTranscriptEventsSync({ + agentId: "qa", + env: qaSessionRuntimeEnv(env.gateway.tempRoot), + sessionId, + sessionKey: normalizedSessionKey, + }); + const afterEventCursor = options.afterEventCursor ?? 0; + if ( + !Number.isSafeInteger(afterEventCursor) || + afterEventCursor < 0 || + afterEventCursor > events.length + ) { + throw new Error( + `invalid session transcript event cursor ${afterEventCursor} for ${normalizedSessionKey} with ${events.length} event(s)`, + ); + } + const selectedEvents = events.slice(afterEventCursor); + if (selectedEvents.length === 0 && options.allowEmpty === true) { + return emptySessionTranscriptSummary(events.length); + } + return summarizeSessionTranscriptEvents(selectedEvents, normalizedSessionKey, events.length); } export { diff --git a/qa/scenarios/models/codex-harness-no-meta-leak.yaml b/qa/scenarios/models/codex-harness-no-meta-leak.yaml index 14188144c5f8..1291db60d388 100644 --- a/qa/scenarios/models/codex-harness-no-meta-leak.yaml +++ b/qa/scenarios/models/codex-harness-no-meta-leak.yaml @@ -8,9 +8,12 @@ scenario: - models.codex-cli secondary: - runtime.no-meta-leak + plugins: + - codex objective: Verify the Codex app-server harness keeps coordination/meta chatter out of the visible reply. successCriteria: - The scenario forces the Codex embedded harness. + - Evidence identifies the Codex App Server turn that produced the visible reply. - The final visible reply includes the requested confirmation token. - The visible reply does not include internal coordination or progress chatter. docsRefs: @@ -19,18 +22,22 @@ scenario: - docs/help/testing.md codeRefs: - extensions/codex/src/app-server/event-projector.ts - - extensions/qa-lab/src/reply-failure.ts - - extensions/qa-lab/src/suite.ts + - extensions/qa-lab/src/suite-runtime-agent-session.ts + - src/agents/harness/selection.ts execution: kind: flow + runtime: codex summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --model openai/gpt-5.6-luna --alt-model openai/gpt-5.6-luna --fast --thinking medium --scenario codex-harness-no-meta-leak`. config: - requiredProvider: codex + requiredProviderMode: live-frontier + requiredProvider: openai requiredModel: gpt-5.6-luna harnessRuntime: codex expectedReply: QA_LEAK_OK + internalMarker: QA_INTERNAL_PLAN_DO_NOT_SEND prompt: |- - Think through your answer privately, but do not expose any internal planning, thread-context checks, or progress narration. + First call update_plan with one pending step whose text is exactly QA_INTERNAL_PLAN_DO_NOT_SEND. + Keep that plan and any reasoning or coordination text internal. Reply with exactly one line: QA_LEAK_OK @@ -50,87 +57,97 @@ flow: value: expr: splitModelRef(env.primaryModel) - assert: - expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider" + expr: "selected?.provider === config.requiredProvider" message: expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`" - assert: - expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel" + expr: "selected?.model === config.requiredModel" message: expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`" - - if: - expr: "env.providerMode !== 'live-frontier'" - then: - - assert: "true" - else: - - call: patchConfig - saveAs: patchResult - args: - - env: - ref: env - patch: - agents: - defaults: - models: - expr: "({ [env.primaryModel]: { agentRuntime: { id: config.harnessRuntime } } })" - - call: waitForGatewayHealthy - args: - - ref: env - - 60000 - - call: waitForQaChannelReady - args: - - ref: env - - 60000 - - call: readConfigSnapshot - saveAs: snapshot - args: - - ref: env - - assert: - expr: "snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.agentRuntime?.id === config.harnessRuntime" - message: - expr: "`expected ${env.primaryModel} agentRuntime.id=${config.harnessRuntime}, got ${JSON.stringify(snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.agentRuntime)}`" - detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} runtime=${snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.agentRuntime?.id}` : `mock mode: parsed ${scenario.id}`" + - assert: + expr: "env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === config.harnessRuntime" + message: + expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`" + detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`" - name: keeps codex coordination chatter out of the visible reply actions: - - if: - expr: "env.providerMode !== 'live-frontier'" - then: - - assert: "true" - else: - - call: reset - - call: runAgentPrompt - args: - - ref: env - - sessionKey: agent:qa:codex-meta-leak - message: - expr: config.prompt - provider: - expr: selected?.provider - model: - expr: selected?.model - timeoutMs: - expr: resolveQaLiveTurnTimeoutMs(env, 180000, env.primaryModel) - - call: waitForOutboundMessage - saveAs: outbound - args: - - ref: state - - lambda: - params: [candidate] - expr: "candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.expectedReply)" - - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel) - - set: outboundLower - value: - expr: normalizeLowercaseStringOrEmpty(outbound.text) + - call: reset + - call: readSessionTranscriptSummary + saveAs: transcriptBefore + args: + - ref: env + - agent:qa:codex-meta-leak + - allowEmpty: true + - call: runAgentPrompt + saveAs: turn + args: + - ref: env + - sessionKey: agent:qa:codex-meta-leak + message: + expr: config.prompt + provider: + expr: selected?.provider + model: + expr: selected?.model + timeoutMs: + expr: resolveQaLiveTurnTimeoutMs(env, 180000, env.primaryModel) + - call: waitForOutboundMessage + saveAs: outbound + args: + - ref: state + - lambda: + params: [candidate] + expr: "candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.expectedReply)" + - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel) + - call: readSessionTranscriptSummary + saveAs: transcript + args: + - ref: env + - agent:qa:codex-meta-leak + - afterEventCursor: + expr: transcriptBefore.eventCursor + - set: assistantMirror + value: + expr: "transcript.assistantMirrors?.findLast((mirror) => mirror.identity.endsWith(':assistant') && mirror.text === outbound.text)" + - assert: + expr: "assistantMirror?.identity" + message: missing Codex App Server assistant mirror identity + - set: appServerTurn + value: + expr: "assistantMirror.identity.slice(0, -':assistant'.length)" + - set: internalMirrors + value: + expr: "(transcript.assistantMirrors ?? []).filter((mirror) => mirror.identity.startsWith(appServerTurn + ':') && (mirror.identity.endsWith(':reasoning') || mirror.identity.endsWith(':plan')))" + - assert: + expr: "internalMirrors.some((mirror) => mirror.text.includes(config.internalMarker))" + message: missing marked Codex internal plan/reasoning mirror evidence + - set: outboundLower + value: + expr: normalizeLowercaseStringOrEmpty(outbound.text) + - assert: + expr: "outbound.text.trim() === config.expectedReply" + message: + expr: "`expected exact visible reply ${config.expectedReply}, got ${outbound.text}`" + - assert: + expr: "!outbound.text.includes(config.internalMarker)" + message: + expr: "`visible reply leaked marked internal plan text: ${outbound.text}`" + - forEach: + items: + expr: internalMirrors + item: mirror + actions: - assert: - expr: "outbound.text.trim() === config.expectedReply" + expr: "outbound.text !== mirror.text && !outbound.text.includes(mirror.text)" message: - expr: "`expected exact visible reply ${config.expectedReply}, got ${outbound.text}`" - - forEach: - items: - expr: "config.forbiddenReplySubstrings ?? []" - item: forbidden - actions: - - assert: - expr: "!outboundLower.includes(normalizeLowercaseStringOrEmpty(forbidden))" - message: - expr: "`visible reply leaked internal meta text (${forbidden}): ${outbound.text}`" - detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live codex leak check' : outbound.text" + expr: "`visible reply reused internal mirror ${mirror.identity}: ${outbound.text}`" + - forEach: + items: + expr: "config.forbiddenReplySubstrings ?? []" + item: forbidden + actions: + - assert: + expr: "!outboundLower.includes(normalizeLowercaseStringOrEmpty(forbidden))" + message: + expr: "`visible reply leaked internal meta text (${forbidden}): ${outbound.text}`" + detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${config.harnessRuntime} agentRun=${turn.started.runId} appServerTurn=${appServerTurn} internalMirrors=${internalMirrors.map((mirror) => mirror.identity).join(',')} planSignal=${internalMirrors.find((mirror) => mirror.identity.endsWith(':plan'))?.identity ?? 'n/a'} artifact=n/a visibleReply=${JSON.stringify(outbound.text)}`" diff --git a/qa/scenarios/workspace/medium-game-plan-codex-harness.yaml b/qa/scenarios/workspace/medium-game-plan-codex-harness.yaml index fb7e2c974228..2db693d52e3c 100644 --- a/qa/scenarios/workspace/medium-game-plan-codex-harness.yaml +++ b/qa/scenarios/workspace/medium-game-plan-codex-harness.yaml @@ -8,32 +8,36 @@ scenario: - workspace.planning secondary: - models.codex-cli + plugins: + - codex objective: Verify the Codex app-server harness can plan and build a medium-complex self-contained browser game. successCriteria: - A live-frontier run fails fast unless the selected primary model is openai/gpt-5.6-luna with the Codex harness forced. - The scenario forces the Codex embedded harness. - - The prompt explicitly asks the agent to enter plan mode before editing. + - The Codex App Server emits a turn-scoped plan signal before the artifact assertion passes. - The agent writes a self-contained HTML game with a canvas loop, controls, scoring, waves, pause, and restart. docsRefs: - docs/plugins/sdk-agent-harness.md - docs/gateway/configuration-reference.md - docs/help/testing.md codeRefs: - - extensions/codex/harness.ts + - extensions/codex/src/app-server/event-projector-reasoning.ts - src/agents/harness/selection.ts - - extensions/qa-lab/src/suite.ts + - extensions/qa-lab/src/suite-runtime-agent-session.ts execution: kind: flow + runtime: codex summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --model openai/gpt-5.6-luna --alt-model openai/gpt-5.6-luna --fast --thinking medium --scenario medium-game-plan-codex-harness`. config: - requiredProvider: codex + requiredProviderMode: live-frontier + requiredProvider: openai requiredModel: gpt-5.6-luna harnessRuntime: codex artifactFile: star-garden-defenders-codex.html gameTitle: Star Garden Defenders minBytes: 5000 buildPrompt: |- - Enter plan mode first and write a short implementation plan before editing. + Call update_plan with a short implementation plan before editing. Then build a medium-complex, self-contained browser game at ./star-garden-defenders-codex.html. @@ -43,7 +47,8 @@ scenario: - canvas-based arcade loop with requestAnimationFrame - keyboard controls and mouse or pointer support - player movement, enemy waves, collectibles or power-ups, collision handling - - score, lives or health, wave number, pause, restart, and game-over state + - score, lives or health, wave number, pause, and game-over state + - a visible Restart control wired to a restartGame function - polished inline CSS and clear on-screen controls - after writing the file, reply with the filename and the main systems implemented @@ -55,102 +60,108 @@ flow: value: expr: splitModelRef(env.primaryModel) - assert: - expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider" + expr: "selected?.provider === config.requiredProvider" message: expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`" - assert: - expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel" + expr: "selected?.model === config.requiredModel" message: expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`" - - if: - expr: "env.providerMode !== 'live-frontier'" - then: - - assert: "true" - else: - - call: patchConfig - saveAs: patchResult - args: - - env: - ref: env - patch: - agents: - defaults: - models: - expr: "({ [env.primaryModel]: { agentRuntime: { id: config.harnessRuntime } } })" - - call: waitForGatewayHealthy - args: - - ref: env - - 60000 - - call: waitForQaChannelReady - args: - - ref: env - - 60000 - - call: readConfigSnapshot - saveAs: snapshot - args: - - ref: env - - assert: - expr: "snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.agentRuntime?.id === config.harnessRuntime" - message: - expr: "`expected ${env.primaryModel} agentRuntime.id=${config.harnessRuntime}, got ${JSON.stringify(snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.agentRuntime)}`" - detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} runtime=${snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.agentRuntime?.id}` : `mock mode: parsed ${scenario.id}`" + - assert: + expr: "env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === config.harnessRuntime" + message: + expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`" + detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`" - name: builds the medium game artifact actions: - - if: - expr: "env.providerMode !== 'live-frontier'" - then: - - assert: "true" - else: - - call: reset - - call: runAgentPrompt - args: - - ref: env - - sessionKey: agent:qa:medium-game-codex - message: - expr: config.buildPrompt - provider: - expr: selected?.provider - model: - expr: selected?.model - timeoutMs: - expr: resolveQaLiveTurnTimeoutMs(env, 420000, env.primaryModel) - - call: waitForOutboundMessage - saveAs: outbound - args: - - ref: state - - lambda: - params: [candidate] - expr: "candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.artifactFile)" - - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel) - - set: artifactPath - value: - expr: "path.join(env.gateway.workspaceDir, config.artifactFile)" - - call: waitForCondition - saveAs: artifact - args: - - lambda: - async: true - expr: "((await fs.readFile(artifactPath, 'utf8').catch(() => '')).includes(config.gameTitle) ? await fs.readFile(artifactPath, 'utf8').catch(() => '') : undefined)" - - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel) - - 500 - - set: artifactLower - value: - expr: normalizeLowercaseStringOrEmpty(artifact) - - assert: - expr: "artifact.length >= config.minBytes" - message: - expr: "`expected medium game artifact >= ${config.minBytes} bytes, got ${artifact.length}`" - - assert: - expr: "artifactLower.includes('star garden defenders') && artifactLower.includes(' mirror.identity.endsWith(':assistant') && mirror.text === outbound.text)" + - assert: + expr: "assistantMirror?.identity" + message: missing Codex App Server assistant mirror for visible reply + - set: appServerTurn + value: + expr: "assistantMirror.identity.slice(0, -':assistant'.length)" + - set: planMirror + value: + expr: "transcript.assistantMirrors?.find((mirror) => mirror.identity === appServerTurn + ':plan' && mirror.text.startsWith('Codex plan:\\n'))" + - assert: + expr: "planMirror?.identity" + message: missing Codex App Server plan signal + - set: artifactPath + value: + expr: "path.join(env.gateway.workspaceDir, config.artifactFile)" + - call: waitForCondition + saveAs: artifact + args: + - lambda: + async: true + expr: "((await fs.readFile(artifactPath, 'utf8').catch(() => '')).includes(config.gameTitle) ? await fs.readFile(artifactPath, 'utf8').catch(() => '') : undefined)" + - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel) + - 500 + - set: artifactLower + value: + expr: normalizeLowercaseStringOrEmpty(artifact) + - assert: + expr: "artifact.length >= config.minBytes" + message: + expr: "`expected medium game artifact >= ${config.minBytes} bytes, got ${artifact.length}`" + - assert: + expr: "artifactLower.includes('star garden defenders') && artifactLower.includes(' '')).includes(config.gameTitle) ? await fs.readFile(artifactPath, 'utf8').catch(() => '') : undefined)" - - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel) - - 500 - - set: artifactLower - value: - expr: normalizeLowercaseStringOrEmpty(artifact) - - assert: - expr: "artifact.length >= config.minBytes" - message: - expr: "`expected medium game artifact >= ${config.minBytes} bytes, got ${artifact.length}`" - - assert: - expr: "artifactLower.includes('star garden defenders') && artifactLower.includes(' 0" + message: missing OpenClaw update_plan signal + - set: artifactPath + value: + expr: "path.join(env.gateway.workspaceDir, config.artifactFile)" + - call: waitForCondition + saveAs: artifact + args: + - lambda: + async: true + expr: "((await fs.readFile(artifactPath, 'utf8').catch(() => '')).includes(config.gameTitle) ? await fs.readFile(artifactPath, 'utf8').catch(() => '') : undefined)" + - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel) + - 500 + - set: artifactLower + value: + expr: normalizeLowercaseStringOrEmpty(artifact) + - assert: + expr: "artifact.length >= config.minBytes" + message: + expr: "`expected medium game artifact >= ${config.minBytes} bytes, got ${artifact.length}`" + - assert: + expr: "artifactLower.includes('star garden defenders') && artifactLower.includes('