fix: make live harness planning QA fail closed (#109282)

* test(qa): prove live harness planning signals * test(qa): satisfy scenario static checks * test(qa): scope planning evidence per attempt * test(qa): derive planning scenarios from coverage
2026-08-12 21:53:00 -06:00 · 2026-07-17 16:45:57 -07:00
parent 82fa956d5a
commit 14caaf3d4c
10 changed files with 805 additions and 304 deletions
@@ -23,6 +23,7 @@ export class CodexReasoningProjection {
  private readonly reasoningTextByGroup = new Map<string, ReasoningTextGroup>();
  private readonly reasoningItemOrder = new Map<string, number>();
  private readonly planTextByItem = new Map<string, string>();
+  private turnPlanText: string | undefined;
  private reasoningStarted = false;
  private reasoningEnded = false;

@@ -75,6 +76,7 @@ export class CodexReasoningProjection {
  }

  handleTurnPlanUpdated(params: JsonObject): void {
+    const explanation = readNullableString(params, "explanation");
    const plan = Array.isArray(params.plan)
      ? params.plan.flatMap((entry) => {
          if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
@@ -88,8 +90,19 @@ export class CodexReasoningProjection {
          return [{ step, status: normalizePlanStepStatus(readString(record, "status")) }];
        })
      : undefined;
+    const planText = [
+      explanation,
+      ...(plan ?? []).map(({ step, status }) => `- [${status}] ${step}`),
+    ]
+      .filter((part): part is string => Boolean(part))
+      .join("\n");
+    if (planText) {
+      // Structured turn updates are the canonical latest plan. Retain the last
+      // non-empty update so the terminal transcript proves planning occurred.
+      this.turnPlanText = planText;
+    }
    this.emitPlanUpdate({
-      explanation: readNullableString(params, "explanation"),
+      explanation,
      steps: plan,
    });
  }
@@ -119,7 +132,10 @@ export class CodexReasoningProjection {
  }

  planText(): string {
-    return [...this.planTextByItem.values()].filter((text) => text.trim().length > 0).join("\n\n");
+    return (
+      this.turnPlanText ??
+      [...this.planTextByItem.values()].filter((text) => text.trim().length > 0).join("\n\n")
+    );
  }

  private emitPlanUpdate(params: { explanation?: string | null; steps?: AgentPlanStep[] }): void {
@@ -2996,6 +2996,8 @@ describe("CodexAppServerEventProjector", () => {
    ]);
    expect(JSON.stringify(result.messagesSnapshot[1])).toContain("Codex reasoning");
    expect(JSON.stringify(result.messagesSnapshot[2])).toContain("Codex plan");
+    expect(JSON.stringify(result.messagesSnapshot[2])).toContain("next");
+    expect(JSON.stringify(result.messagesSnapshot[2])).toContain("[in_progress] patch");
    expect(requireRecord(result.itemLifecycle, "item lifecycle").compactionCount).toBe(1);
    expect(onContextCompacted).toHaveBeenCalledOnce();
  });
@@ -140,14 +140,6 @@ describe("qa scenario catalog", () => {
  it("loads scenario-specific execution config from per-scenario YAML", () => {
    const discovery = readQaScenarioById("source-docs-discovery-report");
    const discoveryConfig = readQaScenarioExecutionConfig("source-docs-discovery-report");
-    const codexLeak = readQaScenarioById("codex-harness-no-meta-leak");
-    const codexLeakConfig = readQaScenarioExecutionConfig("codex-harness-no-meta-leak") as
-      | {
-          harnessRuntime?: string;
-          expectedReply?: string;
-          forbiddenReplySubstrings?: string[];
-        }
-      | undefined;
    const fallbackConfig = readQaScenarioExecutionConfig("memory-failure-fallback");
    const bundledSkill = readQaScenarioById("bundled-plugin-skill-runtime");
    const bundledSkillConfig = readQaScenarioExecutionConfig("bundled-plugin-skill-runtime") as
@@ -161,12 +153,6 @@ describe("qa scenario catalog", () => {
    expect((discoveryConfig?.requiredFiles as string[] | undefined)?.[0]).toBe(
      "repo/qa/scenarios/index.yaml",
    );
-    expect(codexLeak.title).toBe("Codex harness no meta leak");
-    expect(codexLeakConfig?.harnessRuntime).toBe("codex");
-    expect(JSON.stringify(codexLeak.execution.flow)).toContain("agentRuntime");
-    expect(JSON.stringify(codexLeak.execution.flow)).not.toContain("embeddedHarness");
-    expect(codexLeakConfig?.expectedReply).toBe("QA_LEAK_OK");
-    expect(codexLeakConfig?.forbiddenReplySubstrings).toContain("checking thread context");
    expect(fallbackConfig?.gracefulFallbackAny as string[] | undefined).toContain(
      "will not reveal",
    );
@@ -1,7 +1,13 @@
 // Qa Lab tests cover scenario flow runner plugin behavior.
 import { describe, expect, it } from "vitest";
 import { createQaBusState } from "./bus-state.js";
-import { readQaScenarioById, type QaScenarioFlow } from "./scenario-catalog.js";
+import {
+  readQaScenarioById,
+  readQaScenarioPack,
+  type QaScenarioExecution,
+  type QaScenarioFlow,
+  type QaSeedScenarioWithSource,
+} from "./scenario-catalog.js";
 import { runScenarioFlow } from "./scenario-flow-runner.js";

 type QaFlowStep = {
@@ -207,7 +213,205 @@ async function runWebchatTranscriptWait(
  });
 }

+const planningEvidenceCoverageIds = new Set(["runtime.no-meta-leak", "workspace.planning"]);
+
+type PlanningEvidenceScenario = QaSeedScenarioWithSource & {
+  execution: Extract<QaScenarioExecution, { kind: "flow" }> & { flow?: QaScenarioFlow };
+};
+
+function isPlanningEvidenceScenario(
+  scenario: QaSeedScenarioWithSource,
+): scenario is PlanningEvidenceScenario {
+  return (
+    scenario.execution.kind === "flow" &&
+    [...(scenario.coverage?.primary ?? []), ...(scenario.coverage?.secondary ?? [])].some(
+      (coverageId) => planningEvidenceCoverageIds.has(coverageId),
+    )
+  );
+}
+
+type PlanningEvidenceFixture = {
+  currentSummary: Record<string, unknown>;
+  failureMessage: string;
+  outboundText: string;
+  scenario: PlanningEvidenceScenario;
+};
+
+function readPlanningEvidenceFlow(scenario: PlanningEvidenceScenario): QaScenarioFlow {
+  const step = scenario.execution.flow?.steps.find((candidate) =>
+    candidate.actions.some(
+      (action) =>
+        typeof action === "object" &&
+        action !== null &&
+        "call" in action &&
+        action.call === "runAgentPrompt",
+    ),
+  );
+  if (!step) {
+    throw new Error(`planning scenario has no agent turn: ${scenario.id}`);
+  }
+  const artifactIndex = step.actions.findIndex(
+    (action) =>
+      typeof action === "object" &&
+      action !== null &&
+      "set" in action &&
+      action.set === "artifactPath",
+  );
+  const evidenceActions = artifactIndex >= 0 ? step.actions.slice(0, artifactIndex) : step.actions;
+  return {
+    steps: [
+      {
+        name: "proves current-attempt planning evidence",
+        actions: [
+          { set: "selected", value: { provider: "openai", model: "gpt-5.6-luna" } },
+          ...evidenceActions,
+        ],
+      },
+    ],
+  };
+}
+
+function createPlanningEvidenceFixture(
+  scenario: PlanningEvidenceScenario,
+): PlanningEvidenceFixture {
+  const config = scenario.execution.config ?? {};
+  const artifactFile = typeof config.artifactFile === "string" ? config.artifactFile : undefined;
+  const expectedReply = typeof config.expectedReply === "string" ? config.expectedReply : undefined;
+  const internalMarker =
+    typeof config.internalMarker === "string" ? config.internalMarker : undefined;
+
+  if (scenario.execution.runtime === "codex" && expectedReply && internalMarker) {
+    return {
+      scenario,
+      outboundText: expectedReply,
+      failureMessage: "missing marked Codex internal plan/reasoning mirror evidence",
+      currentSummary: {
+        eventCursor: 9,
+        assistantMirrors: [
+          { identity: "current-turn:plan", text: `Codex plan:\n${internalMarker}` },
+          { identity: "current-turn:assistant", text: expectedReply },
+        ],
+        successfulToolCallCounts: {},
+      },
+    };
+  }
+  if (scenario.execution.runtime === "codex" && artifactFile) {
+    const outboundText = `Built ${artifactFile}`;
+    return {
+      scenario,
+      outboundText,
+      failureMessage: "missing Codex App Server plan signal",
+      currentSummary: {
+        eventCursor: 9,
+        assistantMirrors: [
+          { identity: "current-turn:plan", text: "Codex plan:\n- build the game" },
+          { identity: "current-turn:assistant", text: outboundText },
+        ],
+        successfulToolCallCounts: {},
+      },
+    };
+  }
+  if (scenario.execution.runtime === "openclaw" && artifactFile) {
+    return {
+      scenario,
+      outboundText: `Built ${artifactFile}`,
+      failureMessage: "missing OpenClaw update_plan signal",
+      currentSummary: {
+        eventCursor: 9,
+        successfulToolCallCounts: { update_plan: 1 },
+      },
+    };
+  }
+  throw new Error(`unsupported planning evidence metadata: ${scenario.id}`);
+}
+
+function runPlanningEvidenceFixture(
+  fixture: PlanningEvidenceFixture,
+  currentSummary = fixture.currentSummary,
+) {
+  const state = createQaBusState();
+  const readOptions: unknown[] = [];
+  const summaries = [
+    {
+      eventCursor: 7,
+      assistantMirrors: [
+        { identity: "old-turn:plan", text: "Codex plan:\nQA_INTERNAL_PLAN_DO_NOT_SEND" },
+        { identity: "old-turn:assistant", text: fixture.outboundText },
+      ],
+      successfulToolCallCounts: { update_plan: 1 },
+    },
+    currentSummary,
+  ];
+  let readIndex = 0;
+  const result = runLoadedScenarioFlow(fixture.scenario.id, {
+    flow: readPlanningEvidenceFlow(fixture.scenario),
+    state,
+    onWaitForOutboundMessage: ({ state: currentState }) => {
+      currentState.addOutboundMessage({
+        accountId: "qa-channel",
+        to: "dm:qa-operator",
+        text: fixture.outboundText,
+      });
+    },
+    api: {
+      env: {
+        providerMode: "live-frontier",
+        primaryModel: "openai/gpt-5.6-luna",
+      },
+      readSessionTranscriptSummary: async (...args: unknown[]) => {
+        readOptions.push(args[2]);
+        const summary = summaries[readIndex];
+        readIndex += 1;
+        if (!summary) {
+          throw new Error("unexpected transcript summary read");
+        }
+        return summary;
+      },
+      resolveQaLiveTurnTimeoutMs: (_env: unknown, timeoutMs: number) => timeoutMs,
+      normalizeLowercaseStringOrEmpty: (value: unknown) =>
+        typeof value === "string" ? value.trim().toLowerCase() : "",
+      runAgentPrompt: async () => ({ started: { runId: "current-run" }, waited: { status: "ok" } }),
+    },
+  });
+  return { readOptions, result };
+}
+
+const planningEvidenceFixtures = readQaScenarioPack()
+  .scenarios.filter(isPlanningEvidenceScenario)
+  .map(createPlanningEvidenceFixture);
+
 describe("scenario-flow-runner", () => {
+  it.each(planningEvidenceFixtures)(
+    "accepts current-attempt planning evidence for $scenario.id",
+    async (fixture) => {
+      const { readOptions, result } = runPlanningEvidenceFixture(fixture);
+
+      await expect(result).resolves.toMatchObject({ status: "pass" });
+      expect(readOptions).toEqual([{ allowEmpty: true }, { afterEventCursor: 7 }]);
+    },
+  );
+
+  it.each(planningEvidenceFixtures)(
+    "rejects stale prior-attempt planning evidence for $scenario.id",
+    async (fixture) => {
+      const currentSummary = {
+        eventCursor: 8,
+        ...(fixture.scenario.execution.runtime === "codex"
+          ? {
+              assistantMirrors: [
+                { identity: "current-turn:assistant", text: fixture.outboundText },
+              ],
+            }
+          : {}),
+        successfulToolCallCounts: {},
+      };
+      const { readOptions, result } = runPlanningEvidenceFixture(fixture, currentSummary);
+
+      await expect(result).rejects.toThrow(fixture.failureMessage);
+      expect(readOptions).toEqual([{ allowEmpty: true }, { afterEventCursor: 7 }]);
+    },
+  );
+
  it("runs the canonical reaction lifecycle with target-bound actions", async () => {
    const state = createQaBusState();
    const actionTargets: unknown[] = [];
@@ -1,5 +1,6 @@
 // Qa Lab tests cover canonical scenario lane matching behavior.
 import { describe, expect, it } from "vitest";
+import { readQaScenarioPack } from "./scenario-catalog.js";
 import {
  describeQaProviderLaneMismatches,
  scenarioMatchesQaProviderLane,
@@ -7,6 +8,30 @@ import {
 import { makeQaSuiteTestScenario } from "./suite-test-helpers.js";

 describe("QA scenario lane matching", () => {
+  const planningCoverageIds = new Set(["runtime.no-meta-leak", "workspace.planning"]);
+  const planningScenarios = readQaScenarioPack().scenarios.filter((scenario) =>
+    [...(scenario.coverage?.primary ?? []), ...(scenario.coverage?.secondary ?? [])].some(
+      (coverageId) => planningCoverageIds.has(coverageId),
+    ),
+  );
+
+  it.each(planningScenarios)("selects $id for the GPT-5.6 Luna live lane", (scenario) => {
+    expect(
+      scenarioMatchesQaProviderLane({
+        scenario,
+        providerMode: "live-frontier",
+        primaryModel: "openai/gpt-5.6-luna",
+      }),
+    ).toBe(true);
+    expect(
+      scenarioMatchesQaProviderLane({
+        scenario,
+        providerMode: "mock-openai",
+        primaryModel: "openai/gpt-5.6-luna",
+      }),
+    ).toBe(false);
+  });
+
  it("reports every declared mismatch in one decision", () => {
    const scenario = makeQaSuiteTestScenario("strict-live-lane", {
      channel: "matrix",
@@ -351,6 +351,8 @@ describe("qa suite runtime agent session helpers", () => {
      ),
    ).resolves.toEqual({
      assistantToolCallCounts: { message: 1 },
+      eventCursor: 2,
+      successfulToolCallCounts: {},
      finalText: "",
      hasDirectReplySelfMessage: false,
      lastAssistantContentTypes: ["tool_use"],
@@ -375,6 +377,8 @@ describe("qa suite runtime agent session helpers", () => {
      ),
    ).resolves.toEqual({
      assistantToolCallCounts: { message: 1 },
+      eventCursor: 3,
+      successfulToolCallCounts: {},
      finalText: "Sent.",
      hasDirectReplySelfMessage: true,
      lastMessageRole: "assistant",
@@ -427,6 +431,8 @@ describe("qa suite runtime agent session helpers", () => {
      ),
    ).resolves.toEqual({
      assistantToolCallCounts: { message: 1 },
+      eventCursor: 4,
+      successfulToolCallCounts: {},
      finalText: "Sent.",
      hasDirectReplySelfMessage: true,
      lastAssistantErrorMessage: "Request was aborted",
@@ -435,6 +441,174 @@ describe("qa suite runtime agent session helpers", () => {
    });
  });

+  it("reports provider-owned assistant mirror identities", async () => {
+    const tempRoot = await makeTempDir("qa-session-transcript-mirrors-");
+    const sessionKey = "agent:qa:provider-mirrors";
+    await seedQaSession({ tempRoot, sessionKey, sessionId: "session-mirrors" });
+    await appendQaTranscriptMessage({
+      tempRoot,
+      sessionKey,
+      sessionId: "session-mirrors",
+      message: {
+        role: "assistant",
+        content: "Codex plan:\n- inspect\n- build",
+        __openclaw: { mirrorIdentity: "turn-123:plan" },
+      },
+    });
+
+    await expect(
+      readSessionTranscriptSummary(
+        {
+          gateway: { tempRoot },
+        } as never,
+        sessionKey,
+      ),
+    ).resolves.toMatchObject({
+      assistantMirrors: [
+        {
+          identity: "turn-123:plan",
+          text: "Codex plan:\n- inspect\n- build",
+        },
+      ],
+    });
+  });
+
+  it("counts only correlated non-error tool results as successful", async () => {
+    const tempRoot = await makeTempDir("qa-session-transcript-tool-results-");
+    const sessionKey = "agent:qa:tool-results";
+    await seedQaSession({ tempRoot, sessionKey, sessionId: "session-tool-results" });
+    await appendQaTranscriptMessage({
+      tempRoot,
+      sessionKey,
+      sessionId: "session-tool-results",
+      message: {
+        role: "assistant",
+        content: [
+          { type: "toolCall", id: "plan-ok", name: "update_plan", arguments: {} },
+          { type: "toolCall", id: "plan-error", name: "update_plan", arguments: {} },
+          { type: "toolCall", id: "write-mismatch", name: "write", arguments: {} },
+        ],
+      },
+    });
+    for (const message of [
+      {
+        role: "toolResult",
+        toolCallId: "plan-ok",
+        toolName: "update_plan",
+        content: [{ type: "text", text: "Plan updated" }],
+        isError: false,
+      },
+      {
+        role: "toolResult",
+        toolCallId: "plan-ok",
+        toolName: "update_plan",
+        content: [{ type: "text", text: "duplicate" }],
+        isError: false,
+      },
+      {
+        role: "toolResult",
+        toolCallId: "plan-error",
+        toolName: "update_plan",
+        content: [{ type: "text", text: "failed" }],
+        isError: true,
+      },
+      {
+        role: "toolResult",
+        toolCallId: "write-mismatch",
+        toolName: "exec",
+        content: [{ type: "text", text: "wrong tool" }],
+        isError: false,
+      },
+    ]) {
+      await appendQaTranscriptMessage({
+        tempRoot,
+        sessionKey,
+        sessionId: "session-tool-results",
+        message,
+      });
+    }
+
+    await expect(
+      readSessionTranscriptSummary(
+        {
+          gateway: { tempRoot },
+        } as never,
+        sessionKey,
+      ),
+    ).resolves.toMatchObject({
+      assistantToolCallCounts: { update_plan: 2, write: 1 },
+      successfulToolCallCounts: { update_plan: 1 },
+    });
+  });
+
+  it("scopes transcript evidence after an event cursor", async () => {
+    const tempRoot = await makeTempDir("qa-session-transcript-cursor-");
+    const sessionKey = "agent:qa:cursor";
+    const sessionId = "session-cursor";
+    await seedQaSession({ tempRoot, sessionKey, sessionId });
+    for (const message of [
+      {
+        role: "assistant",
+        content: [{ type: "toolCall", id: "old-plan", name: "update_plan", arguments: {} }],
+      },
+      {
+        role: "toolResult",
+        toolCallId: "old-plan",
+        toolName: "update_plan",
+        content: [{ type: "text", text: "Plan updated" }],
+        isError: false,
+      },
+      {
+        role: "assistant",
+        content: "same visible reply",
+        __openclaw: { mirrorIdentity: "old-turn:assistant" },
+      },
+    ]) {
+      await appendQaTranscriptMessage({ tempRoot, sessionKey, sessionId, message });
+    }
+    const checkpoint = await readSessionTranscriptSummary(
+      { gateway: { tempRoot } } as never,
+      sessionKey,
+    );
+    await appendQaTranscriptMessage({
+      tempRoot,
+      sessionKey,
+      sessionId,
+      message: {
+        role: "assistant",
+        content: "same visible reply",
+        __openclaw: { mirrorIdentity: "current-turn:assistant" },
+      },
+    });
+
+    await expect(
+      readSessionTranscriptSummary({ gateway: { tempRoot } } as never, sessionKey, {
+        afterEventCursor: checkpoint.eventCursor,
+      }),
+    ).resolves.toMatchObject({
+      assistantMirrors: [{ identity: "current-turn:assistant", text: "same visible reply" }],
+      assistantToolCallCounts: {},
+      eventCursor: 5,
+      successfulToolCallCounts: {},
+    });
+  });
+
+  it("returns an empty checkpoint before the session exists", async () => {
+    const tempRoot = await makeTempDir("qa-session-transcript-checkpoint-");
+
+    await expect(
+      readSessionTranscriptSummary({ gateway: { tempRoot } } as never, "agent:qa:not-created-yet", {
+        allowEmpty: true,
+      }),
+    ).resolves.toEqual({
+      assistantToolCallCounts: {},
+      eventCursor: 0,
+      successfulToolCallCounts: {},
+      finalText: "",
+      hasDirectReplySelfMessage: false,
+    });
+  });
+
  it("fails closed when a requested QA session transcript is empty", async () => {
    const tempRoot = await makeTempDir("qa-session-transcript-empty-");
    await seedQaSession({
@@ -46,7 +46,10 @@ const SESSION_STORE_LOCK_RETRY_DELAYS_MS = [1_000, 3_000, 5_000] as const;
 const SESSION_STORE_FTS_SETTLE_RETRY_DELAYS_MS = [100, 250, 500, 1_000, 2_000] as const;

 type QaSessionTranscriptSummary = {
+  assistantMirrors?: Array<{ identity: string; text: string }>;
  assistantToolCallCounts: Record<string, number>;
+  eventCursor: number;
+  successfulToolCallCounts: Record<string, number>;
  finalText: string;
  hasDirectReplySelfMessage: boolean;
  lastAssistantContentTypes?: string[];
@@ -56,6 +59,11 @@ type QaSessionTranscriptSummary = {
  lastMessageRole?: string;
 };

+type QaSessionTranscriptSummaryOptions = {
+  afterEventCursor?: number;
+  allowEmpty?: boolean;
+};
+
 function isSessionStoreLockTimeout(error: unknown) {
  const text = formatErrorMessage(error);
  return (
@@ -81,7 +89,10 @@ function readSessionTranscriptEventMessage(event: unknown) {
  return isRecord(event) && isRecord(event.message) ? event.message : undefined;
 }

-function readAssistantToolNames(message: Record<string, unknown>): string[] {
+function readAssistantToolCalls(message: Record<string, unknown>): Array<{
+  id?: string;
+  name: string;
+}> {
  if (!Array.isArray(message.content)) {
    return [];
  }
@@ -94,16 +105,21 @@ function readAssistantToolNames(message: Record<string, unknown>): string[] {
      return [];
    }
    const name = readNonEmptyString(block.name);
-    return name ? [name] : [];
+    return name ? [{ id: readNonEmptyString(block.id), name }] : [];
  });
 }

 function summarizeSessionTranscriptEvents(
  events: unknown[],
  sessionKey: string,
+  eventCursor = events.length,
 ): QaSessionTranscriptSummary {
  const scanner = createDirectReplyTranscriptSentinelScanner();
+  const assistantMirrors: Array<{ identity: string; text: string }> = [];
  const assistantToolCallCounts: Record<string, number> = {};
+  const successfulToolCallCounts: Record<string, number> = {};
+  const assistantToolNamesByCallId = new Map<string, string>();
+  const successfulToolCallIds = new Set<string>();
  let finalText = "";
  let lastAssistantContentTypes: string[] = [];
  let lastAssistantErrorMessage: string | undefined;
@@ -117,6 +133,21 @@ function summarizeSessionTranscriptEvents(
      continue;
    }
    lastMessageRole = readNonEmptyString(message.role);
+    if (message.role === "toolResult") {
+      const toolCallId = readNonEmptyString(message.toolCallId);
+      const toolName = readNonEmptyString(message.toolName);
+      if (
+        toolCallId &&
+        toolName &&
+        message.isError === false &&
+        assistantToolNamesByCallId.get(toolCallId) === toolName &&
+        !successfulToolCallIds.has(toolCallId)
+      ) {
+        successfulToolCallIds.add(toolCallId);
+        successfulToolCallCounts[toolName] = (successfulToolCallCounts[toolName] ?? 0) + 1;
+      }
+      continue;
+    }
    if (message.role !== "assistant") {
      continue;
    }
@@ -124,6 +155,11 @@ function summarizeSessionTranscriptEvents(
    if (text) {
      finalText = text;
    }
+    const openClawMeta = isRecord(message["__openclaw"]) ? message["__openclaw"] : undefined;
+    const mirrorIdentity = readNonEmptyString(openClawMeta?.mirrorIdentity);
+    if (mirrorIdentity && text) {
+      assistantMirrors.push({ identity: mirrorIdentity, text });
+    }
    lastAssistantContentTypes = Array.isArray(message.content)
      ? message.content.flatMap((block) => {
          const type = isRecord(block) ? readNonEmptyString(block.type) : undefined;
@@ -132,9 +168,13 @@ function summarizeSessionTranscriptEvents(
      : [];
    lastAssistantErrorMessage = readNonEmptyString(message.errorMessage);
    lastAssistantStopReason = readNonEmptyString(message.stopReason);
-    lastAssistantToolNames = readAssistantToolNames(message);
-    for (const toolName of lastAssistantToolNames) {
-      assistantToolCallCounts[toolName] = (assistantToolCallCounts[toolName] ?? 0) + 1;
+    const assistantToolCalls = readAssistantToolCalls(message);
+    lastAssistantToolNames = assistantToolCalls.map((toolCall) => toolCall.name);
+    for (const toolCall of assistantToolCalls) {
+      assistantToolCallCounts[toolCall.name] = (assistantToolCallCounts[toolCall.name] ?? 0) + 1;
+      if (toolCall.id) {
+        assistantToolNamesByCallId.set(toolCall.id, toolCall.name);
+      }
    }
    scanner.recordMessage(message);
  }
@@ -144,7 +184,10 @@ function summarizeSessionTranscriptEvents(
  }

  return {
+    ...(assistantMirrors.length > 0 ? { assistantMirrors } : {}),
    assistantToolCallCounts,
+    eventCursor,
+    successfulToolCallCounts,
    finalText,
    hasDirectReplySelfMessage: scanner.findings().length > 0,
    ...(lastAssistantContentTypes.length > 0 ? { lastAssistantContentTypes } : {}),
@@ -155,6 +198,16 @@ function summarizeSessionTranscriptEvents(
  };
 }

+function emptySessionTranscriptSummary(eventCursor: number): QaSessionTranscriptSummary {
+  return {
+    assistantToolCallCounts: {},
+    eventCursor,
+    successfulToolCallCounts: {},
+    finalText: "",
+    hasDirectReplySelfMessage: false,
+  };
+}
+
 async function callGatewayWithSessionStoreLockRetry<T>(
  env: QaGatewayCallEnv,
  method: string,
@@ -329,6 +382,7 @@ async function readRawQaSessionStore(
 async function readSessionTranscriptSummary(
  env: Pick<QaSuiteRuntimeEnv, "gateway">,
  sessionKey: string,
+  options: QaSessionTranscriptSummaryOptions = {},
 ): Promise<QaSessionTranscriptSummary> {
  const normalizedSessionKey = sessionKey.trim();
  if (!normalizedSessionKey) {
@@ -338,17 +392,32 @@ async function readSessionTranscriptSummary(
  const entry = store[normalizedSessionKey];
  const sessionId = readNonEmptyString(entry?.sessionId);
  if (!sessionId) {
+    if (options.allowEmpty === true) {
+      return emptySessionTranscriptSummary(0);
+    }
    throw new Error(`session transcript entry not found for ${normalizedSessionKey}`);
  }
-  return summarizeSessionTranscriptEvents(
-    loadTranscriptEventsSync({
-      agentId: "qa",
-      env: qaSessionRuntimeEnv(env.gateway.tempRoot),
-      sessionId,
-      sessionKey: normalizedSessionKey,
-    }),
-    normalizedSessionKey,
-  );
+  const events = loadTranscriptEventsSync({
+    agentId: "qa",
+    env: qaSessionRuntimeEnv(env.gateway.tempRoot),
+    sessionId,
+    sessionKey: normalizedSessionKey,
+  });
+  const afterEventCursor = options.afterEventCursor ?? 0;
+  if (
+    !Number.isSafeInteger(afterEventCursor) ||
+    afterEventCursor < 0 ||
+    afterEventCursor > events.length
+  ) {
+    throw new Error(
+      `invalid session transcript event cursor ${afterEventCursor} for ${normalizedSessionKey} with ${events.length} event(s)`,
+    );
+  }
+  const selectedEvents = events.slice(afterEventCursor);
+  if (selectedEvents.length === 0 && options.allowEmpty === true) {
+    return emptySessionTranscriptSummary(events.length);
+  }
+  return summarizeSessionTranscriptEvents(selectedEvents, normalizedSessionKey, events.length);
 }

 export {
@@ -8,9 +8,12 @@ scenario:
      - models.codex-cli
    secondary:
      - runtime.no-meta-leak
+  plugins:
+    - codex
  objective: Verify the Codex app-server harness keeps coordination/meta chatter out of the visible reply.
  successCriteria:
    - The scenario forces the Codex embedded harness.
+    - Evidence identifies the Codex App Server turn that produced the visible reply.
    - The final visible reply includes the requested confirmation token.
    - The visible reply does not include internal coordination or progress chatter.
  docsRefs:
@@ -19,18 +22,22 @@ scenario:
    - docs/help/testing.md
  codeRefs:
    - extensions/codex/src/app-server/event-projector.ts
-    - extensions/qa-lab/src/reply-failure.ts
-    - extensions/qa-lab/src/suite.ts
+    - extensions/qa-lab/src/suite-runtime-agent-session.ts
+    - src/agents/harness/selection.ts
  execution:
    kind: flow
+    runtime: codex
    summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --model openai/gpt-5.6-luna --alt-model openai/gpt-5.6-luna --fast --thinking medium --scenario codex-harness-no-meta-leak`.
    config:
-      requiredProvider: codex
+      requiredProviderMode: live-frontier
+      requiredProvider: openai
      requiredModel: gpt-5.6-luna
      harnessRuntime: codex
      expectedReply: QA_LEAK_OK
+      internalMarker: QA_INTERNAL_PLAN_DO_NOT_SEND
      prompt: |-
-        Think through your answer privately, but do not expose any internal planning, thread-context checks, or progress narration.
+        First call update_plan with one pending step whose text is exactly QA_INTERNAL_PLAN_DO_NOT_SEND.
+        Keep that plan and any reasoning or coordination text internal.

        Reply with exactly one line:
        QA_LEAK_OK
@@ -50,87 +57,97 @@ flow:
          value:
            expr: splitModelRef(env.primaryModel)
        - assert:
-            expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider"
+            expr: "selected?.provider === config.requiredProvider"
            message:
              expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`"
        - assert:
-            expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel"
+            expr: "selected?.model === config.requiredModel"
            message:
              expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`"
-        - if:
-            expr: "env.providerMode !== 'live-frontier'"
-            then:
-              - assert: "true"
-            else:
-              - call: patchConfig
-                saveAs: patchResult
-                args:
-                  - env:
-                      ref: env
-                    patch:
-                      agents:
-                        defaults:
-                          models:
-                            expr: "({ [env.primaryModel]: { agentRuntime: { id: config.harnessRuntime } } })"
-              - call: waitForGatewayHealthy
-                args:
-                  - ref: env
-                  - 60000
-              - call: waitForQaChannelReady
-                args:
-                  - ref: env
-                  - 60000
-              - call: readConfigSnapshot
-                saveAs: snapshot
-                args:
-                  - ref: env
-              - assert:
-                  expr: "snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.agentRuntime?.id === config.harnessRuntime"
-                  message:
-                    expr: "`expected ${env.primaryModel} agentRuntime.id=${config.harnessRuntime}, got ${JSON.stringify(snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.agentRuntime)}`"
-      detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} runtime=${snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.agentRuntime?.id}` : `mock mode: parsed ${scenario.id}`"
+        - assert:
+            expr: "env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === config.harnessRuntime"
+            message:
+              expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`"
+      detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`"
    - name: keeps codex coordination chatter out of the visible reply
      actions:
-        - if:
-            expr: "env.providerMode !== 'live-frontier'"
-            then:
-              - assert: "true"
-            else:
-              - call: reset
-              - call: runAgentPrompt
-                args:
-                  - ref: env
-                  - sessionKey: agent:qa:codex-meta-leak
-                    message:
-                      expr: config.prompt
-                    provider:
-                      expr: selected?.provider
-                    model:
-                      expr: selected?.model
-                    timeoutMs:
-                      expr: resolveQaLiveTurnTimeoutMs(env, 180000, env.primaryModel)
-              - call: waitForOutboundMessage
-                saveAs: outbound
-                args:
-                  - ref: state
-                  - lambda:
-                      params: [candidate]
-                      expr: "candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.expectedReply)"
-                  - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
-              - set: outboundLower
-                value:
-                  expr: normalizeLowercaseStringOrEmpty(outbound.text)
+        - call: reset
+        - call: readSessionTranscriptSummary
+          saveAs: transcriptBefore
+          args:
+            - ref: env
+            - agent:qa:codex-meta-leak
+            - allowEmpty: true
+        - call: runAgentPrompt
+          saveAs: turn
+          args:
+            - ref: env
+            - sessionKey: agent:qa:codex-meta-leak
+              message:
+                expr: config.prompt
+              provider:
+                expr: selected?.provider
+              model:
+                expr: selected?.model
+              timeoutMs:
+                expr: resolveQaLiveTurnTimeoutMs(env, 180000, env.primaryModel)
+        - call: waitForOutboundMessage
+          saveAs: outbound
+          args:
+            - ref: state
+            - lambda:
+                params: [candidate]
+                expr: "candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.expectedReply)"
+            - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
+        - call: readSessionTranscriptSummary
+          saveAs: transcript
+          args:
+            - ref: env
+            - agent:qa:codex-meta-leak
+            - afterEventCursor:
+                expr: transcriptBefore.eventCursor
+        - set: assistantMirror
+          value:
+            expr: "transcript.assistantMirrors?.findLast((mirror) => mirror.identity.endsWith(':assistant') && mirror.text === outbound.text)"
+        - assert:
+            expr: "assistantMirror?.identity"
+            message: missing Codex App Server assistant mirror identity
+        - set: appServerTurn
+          value:
+            expr: "assistantMirror.identity.slice(0, -':assistant'.length)"
+        - set: internalMirrors
+          value:
+            expr: "(transcript.assistantMirrors ?? []).filter((mirror) => mirror.identity.startsWith(appServerTurn + ':') && (mirror.identity.endsWith(':reasoning') || mirror.identity.endsWith(':plan')))"
+        - assert:
+            expr: "internalMirrors.some((mirror) => mirror.text.includes(config.internalMarker))"
+            message: missing marked Codex internal plan/reasoning mirror evidence
+        - set: outboundLower
+          value:
+            expr: normalizeLowercaseStringOrEmpty(outbound.text)
+        - assert:
+            expr: "outbound.text.trim() === config.expectedReply"
+            message:
+              expr: "`expected exact visible reply ${config.expectedReply}, got ${outbound.text}`"
+        - assert:
+            expr: "!outbound.text.includes(config.internalMarker)"
+            message:
+              expr: "`visible reply leaked marked internal plan text: ${outbound.text}`"
+        - forEach:
+            items:
+              expr: internalMirrors
+            item: mirror
+            actions:
              - assert:
-                  expr: "outbound.text.trim() === config.expectedReply"
+                  expr: "outbound.text !== mirror.text && !outbound.text.includes(mirror.text)"
                  message:
-                    expr: "`expected exact visible reply ${config.expectedReply}, got ${outbound.text}`"
-              - forEach:
-                  items:
-                    expr: "config.forbiddenReplySubstrings ?? []"
-                  item: forbidden
-                  actions:
-                    - assert:
-                        expr: "!outboundLower.includes(normalizeLowercaseStringOrEmpty(forbidden))"
-                        message:
-                          expr: "`visible reply leaked internal meta text (${forbidden}): ${outbound.text}`"
-      detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live codex leak check' : outbound.text"
+                    expr: "`visible reply reused internal mirror ${mirror.identity}: ${outbound.text}`"
+        - forEach:
+            items:
+              expr: "config.forbiddenReplySubstrings ?? []"
+            item: forbidden
+            actions:
+              - assert:
+                  expr: "!outboundLower.includes(normalizeLowercaseStringOrEmpty(forbidden))"
+                  message:
+                    expr: "`visible reply leaked internal meta text (${forbidden}): ${outbound.text}`"
+      detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${config.harnessRuntime} agentRun=${turn.started.runId} appServerTurn=${appServerTurn} internalMirrors=${internalMirrors.map((mirror) => mirror.identity).join(',')} planSignal=${internalMirrors.find((mirror) => mirror.identity.endsWith(':plan'))?.identity ?? 'n/a'} artifact=n/a visibleReply=${JSON.stringify(outbound.text)}`"
@@ -8,32 +8,36 @@ scenario:
      - workspace.planning
    secondary:
      - models.codex-cli
+  plugins:
+    - codex
  objective: Verify the Codex app-server harness can plan and build a medium-complex self-contained browser game.
  successCriteria:
    - A live-frontier run fails fast unless the selected primary model is openai/gpt-5.6-luna with the Codex harness forced.
    - The scenario forces the Codex embedded harness.
-    - The prompt explicitly asks the agent to enter plan mode before editing.
+    - The Codex App Server emits a turn-scoped plan signal before the artifact assertion passes.
    - The agent writes a self-contained HTML game with a canvas loop, controls, scoring, waves, pause, and restart.
  docsRefs:
    - docs/plugins/sdk-agent-harness.md
    - docs/gateway/configuration-reference.md
    - docs/help/testing.md
  codeRefs:
-    - extensions/codex/harness.ts
+    - extensions/codex/src/app-server/event-projector-reasoning.ts
    - src/agents/harness/selection.ts
-    - extensions/qa-lab/src/suite.ts
+    - extensions/qa-lab/src/suite-runtime-agent-session.ts
  execution:
    kind: flow
+    runtime: codex
    summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --model openai/gpt-5.6-luna --alt-model openai/gpt-5.6-luna --fast --thinking medium --scenario medium-game-plan-codex-harness`.
    config:
-      requiredProvider: codex
+      requiredProviderMode: live-frontier
+      requiredProvider: openai
      requiredModel: gpt-5.6-luna
      harnessRuntime: codex
      artifactFile: star-garden-defenders-codex.html
      gameTitle: Star Garden Defenders
      minBytes: 5000
      buildPrompt: |-
-        Enter plan mode first and write a short implementation plan before editing.
+        Call update_plan with a short implementation plan before editing.

        Then build a medium-complex, self-contained browser game at ./star-garden-defenders-codex.html.

@@ -43,7 +47,8 @@ scenario:
        - canvas-based arcade loop with requestAnimationFrame
        - keyboard controls and mouse or pointer support
        - player movement, enemy waves, collectibles or power-ups, collision handling
-        - score, lives or health, wave number, pause, restart, and game-over state
+        - score, lives or health, wave number, pause, and game-over state
+        - a visible Restart control wired to a restartGame function
        - polished inline CSS and clear on-screen controls
        - after writing the file, reply with the filename and the main systems implemented

@@ -55,102 +60,108 @@ flow:
          value:
            expr: splitModelRef(env.primaryModel)
        - assert:
-            expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider"
+            expr: "selected?.provider === config.requiredProvider"
            message:
              expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`"
        - assert:
-            expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel"
+            expr: "selected?.model === config.requiredModel"
            message:
              expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`"
-        - if:
-            expr: "env.providerMode !== 'live-frontier'"
-            then:
-              - assert: "true"
-            else:
-              - call: patchConfig
-                saveAs: patchResult
-                args:
-                  - env:
-                      ref: env
-                    patch:
-                      agents:
-                        defaults:
-                          models:
-                            expr: "({ [env.primaryModel]: { agentRuntime: { id: config.harnessRuntime } } })"
-              - call: waitForGatewayHealthy
-                args:
-                  - ref: env
-                  - 60000
-              - call: waitForQaChannelReady
-                args:
-                  - ref: env
-                  - 60000
-              - call: readConfigSnapshot
-                saveAs: snapshot
-                args:
-                  - ref: env
-              - assert:
-                  expr: "snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.agentRuntime?.id === config.harnessRuntime"
-                  message:
-                    expr: "`expected ${env.primaryModel} agentRuntime.id=${config.harnessRuntime}, got ${JSON.stringify(snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.agentRuntime)}`"
-      detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} runtime=${snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.agentRuntime?.id}` : `mock mode: parsed ${scenario.id}`"
+        - assert:
+            expr: "env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === config.harnessRuntime"
+            message:
+              expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`"
+      detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`"
    - name: builds the medium game artifact
      actions:
-        - if:
-            expr: "env.providerMode !== 'live-frontier'"
-            then:
-              - assert: "true"
-            else:
-              - call: reset
-              - call: runAgentPrompt
-                args:
-                  - ref: env
-                  - sessionKey: agent:qa:medium-game-codex
-                    message:
-                      expr: config.buildPrompt
-                    provider:
-                      expr: selected?.provider
-                    model:
-                      expr: selected?.model
-                    timeoutMs:
-                      expr: resolveQaLiveTurnTimeoutMs(env, 420000, env.primaryModel)
-              - call: waitForOutboundMessage
-                saveAs: outbound
-                args:
-                  - ref: state
-                  - lambda:
-                      params: [candidate]
-                      expr: "candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.artifactFile)"
-                  - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
-              - set: artifactPath
-                value:
-                  expr: "path.join(env.gateway.workspaceDir, config.artifactFile)"
-              - call: waitForCondition
-                saveAs: artifact
-                args:
-                  - lambda:
-                      async: true
-                      expr: "((await fs.readFile(artifactPath, 'utf8').catch(() => '')).includes(config.gameTitle) ? await fs.readFile(artifactPath, 'utf8').catch(() => '') : undefined)"
-                  - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
-                  - 500
-              - set: artifactLower
-                value:
-                  expr: normalizeLowercaseStringOrEmpty(artifact)
-              - assert:
-                  expr: "artifact.length >= config.minBytes"
-                  message:
-                    expr: "`expected medium game artifact >= ${config.minBytes} bytes, got ${artifact.length}`"
-              - assert:
-                  expr: "artifactLower.includes('star garden defenders') && artifactLower.includes('<canvas') && artifactLower.includes('requestanimationframe')"
-                  message: missing title, canvas, or animation loop
-              - assert:
-                  expr: "artifactLower.includes('keydown') || artifactLower.includes('keyup')"
-                  message: missing keyboard controls
-              - assert:
-                  expr: "artifactLower.includes('score') && artifactLower.includes('wave') && artifactLower.includes('pause') && artifactLower.includes('restart')"
-                  message: missing score, wave, pause, or restart systems
-              - assert:
-                  expr: "outbound.text.includes(config.artifactFile)"
-                  message:
-                    expr: "`final reply did not mention ${config.artifactFile}: ${outbound.text}`"
-      detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live medium-game build' : `${config.artifactFile} bytes=${artifact.length}`"
+        - call: reset
+        - call: readSessionTranscriptSummary
+          saveAs: transcriptBefore
+          args:
+            - ref: env
+            - agent:qa:medium-game-codex
+            - allowEmpty: true
+        - call: runAgentPrompt
+          saveAs: turn
+          args:
+            - ref: env
+            - sessionKey: agent:qa:medium-game-codex
+              message:
+                expr: config.buildPrompt
+              provider:
+                expr: selected?.provider
+              model:
+                expr: selected?.model
+              timeoutMs:
+                expr: resolveQaLiveTurnTimeoutMs(env, 420000, env.primaryModel)
+        - call: waitForOutboundMessage
+          saveAs: outbound
+          args:
+            - ref: state
+            - lambda:
+                params: [candidate]
+                expr: "candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.artifactFile)"
+            - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
+        - call: readSessionTranscriptSummary
+          saveAs: transcript
+          args:
+            - ref: env
+            - agent:qa:medium-game-codex
+            - afterEventCursor:
+                expr: transcriptBefore.eventCursor
+        - set: assistantMirror
+          value:
+            expr: "transcript.assistantMirrors?.findLast((mirror) => mirror.identity.endsWith(':assistant') && mirror.text === outbound.text)"
+        - assert:
+            expr: "assistantMirror?.identity"
+            message: missing Codex App Server assistant mirror for visible reply
+        - set: appServerTurn
+          value:
+            expr: "assistantMirror.identity.slice(0, -':assistant'.length)"
+        - set: planMirror
+          value:
+            expr: "transcript.assistantMirrors?.find((mirror) => mirror.identity === appServerTurn + ':plan' && mirror.text.startsWith('Codex plan:\\n'))"
+        - assert:
+            expr: "planMirror?.identity"
+            message: missing Codex App Server plan signal
+        - set: artifactPath
+          value:
+            expr: "path.join(env.gateway.workspaceDir, config.artifactFile)"
+        - call: waitForCondition
+          saveAs: artifact
+          args:
+            - lambda:
+                async: true
+                expr: "((await fs.readFile(artifactPath, 'utf8').catch(() => '')).includes(config.gameTitle) ? await fs.readFile(artifactPath, 'utf8').catch(() => '') : undefined)"
+            - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
+            - 500
+        - set: artifactLower
+          value:
+            expr: normalizeLowercaseStringOrEmpty(artifact)
+        - assert:
+            expr: "artifact.length >= config.minBytes"
+            message:
+              expr: "`expected medium game artifact >= ${config.minBytes} bytes, got ${artifact.length}`"
+        - assert:
+            expr: "artifactLower.includes('star garden defenders') && artifactLower.includes('<canvas') && artifactLower.includes('requestanimationframe')"
+            message: missing title, canvas, or animation loop
+        - assert:
+            expr: "artifactLower.includes('keydown') || artifactLower.includes('keyup')"
+            message: missing keyboard controls
+        - assert:
+            expr: "artifactLower.includes('score')"
+            message: missing score system
+        - assert:
+            expr: "artifactLower.includes('wave')"
+            message: missing wave system
+        - assert:
+            expr: "artifactLower.includes('pause')"
+            message: missing pause system
+        - assert:
+            expr: "artifactLower.includes('restart') || artifactLower.includes('resetgame')"
+            message: missing restart system
+        - assert:
+            expr: "outbound.text.includes(config.artifactFile)"
+            message:
+              expr: "`final reply did not mention ${config.artifactFile}: ${outbound.text}`"
+      detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${config.harnessRuntime} agentRun=${turn.started.runId} appServerTurn=${appServerTurn} planSignal=${planMirror.identity} artifact=${artifactPath} bytes=${artifact.length} visibleReply=${JSON.stringify(outbound.text)}`"
@@ -12,7 +12,7 @@ scenario:
  successCriteria:
    - A live-frontier run fails fast unless the selected primary model is openai/gpt-5.6-luna.
    - The scenario forces the embedded OpenClaw harness before the build turn.
-    - The prompt explicitly asks the agent to enter plan mode before editing.
+    - The OpenClaw harness records a successful `update_plan` tool call before the artifact assertion passes.
    - The agent writes a self-contained HTML game with a canvas loop, controls, scoring, waves, pause, and restart.
  docsRefs:
    - docs/plugins/sdk-agent-harness.md
@@ -21,11 +21,13 @@ scenario:
  codeRefs:
    - src/agents/harness/selection.ts
    - src/agents/harness/builtin-openclaw.ts
-    - extensions/qa-lab/src/suite.ts
+    - extensions/qa-lab/src/suite-runtime-agent-session.ts
  execution:
    kind: flow
+    runtime: openclaw
    summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --model openai/gpt-5.6-luna --alt-model openai/gpt-5.6-luna --fast --thinking medium --scenario medium-game-plan-openclaw-harness`.
    config:
+      requiredProviderMode: live-frontier
      requiredProvider: openai
      requiredModel: gpt-5.6-luna
      harnessRuntime: openclaw
@@ -33,7 +35,7 @@ scenario:
      gameTitle: Star Garden Defenders
      minBytes: 5000
      buildPrompt: |-
-        Enter plan mode first and write a short implementation plan before editing.
+        Call update_plan with a short implementation plan before editing.

        Then build a medium-complex, self-contained browser game at ./star-garden-defenders-openclaw.html.

@@ -43,7 +45,8 @@ scenario:
        - canvas-based arcade loop with requestAnimationFrame
        - keyboard controls and mouse or pointer support
        - player movement, enemy waves, collectibles or power-ups, collision handling
-        - score, lives or health, wave number, pause, restart, and game-over state
+        - score, lives or health, wave number, pause, and game-over state
+        - a visible Restart control wired to a restartGame function
        - polished inline CSS and clear on-screen controls
        - after writing the file, reply with the filename and the main systems implemented

@@ -55,102 +58,96 @@ flow:
          value:
            expr: splitModelRef(env.primaryModel)
        - assert:
-            expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider"
+            expr: "selected?.provider === config.requiredProvider"
            message:
              expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`"
        - assert:
-            expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel"
+            expr: "selected?.model === config.requiredModel"
            message:
              expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`"
-        - if:
-            expr: "env.providerMode !== 'live-frontier'"
-            then:
-              - assert: "true"
-            else:
-              - call: patchConfig
-                saveAs: patchResult
-                args:
-                  - env:
-                      ref: env
-                    patch:
-                      agents:
-                        defaults:
-                          models:
-                            expr: "({ [env.primaryModel]: { agentRuntime: { id: config.harnessRuntime } } })"
-              - call: waitForGatewayHealthy
-                args:
-                  - ref: env
-                  - 60000
-              - call: waitForQaChannelReady
-                args:
-                  - ref: env
-                  - 60000
-              - call: readConfigSnapshot
-                saveAs: snapshot
-                args:
-                  - ref: env
-              - assert:
-                  expr: "snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.agentRuntime?.id === config.harnessRuntime"
-                  message:
-                    expr: "`expected ${env.primaryModel} agentRuntime.id=${config.harnessRuntime}, got ${JSON.stringify(snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.agentRuntime)}`"
-      detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} runtime=${snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.agentRuntime?.id}` : `mock mode: parsed ${scenario.id}`"
+        - assert:
+            expr: "env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === config.harnessRuntime"
+            message:
+              expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`"
+      detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME}`"
    - name: builds the medium game artifact
      actions:
-        - if:
-            expr: "env.providerMode !== 'live-frontier'"
-            then:
-              - assert: "true"
-            else:
-              - call: reset
-              - call: runAgentPrompt
-                args:
-                  - ref: env
-                  - sessionKey: agent:qa:medium-game-openclaw
-                    message:
-                      expr: config.buildPrompt
-                    provider:
-                      expr: selected?.provider
-                    model:
-                      expr: selected?.model
-                    timeoutMs:
-                      expr: resolveQaLiveTurnTimeoutMs(env, 420000, env.primaryModel)
-              - call: waitForOutboundMessage
-                saveAs: outbound
-                args:
-                  - ref: state
-                  - lambda:
-                      params: [candidate]
-                      expr: "candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.artifactFile)"
-                  - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
-              - set: artifactPath
-                value:
-                  expr: "path.join(env.gateway.workspaceDir, config.artifactFile)"
-              - call: waitForCondition
-                saveAs: artifact
-                args:
-                  - lambda:
-                      async: true
-                      expr: "((await fs.readFile(artifactPath, 'utf8').catch(() => '')).includes(config.gameTitle) ? await fs.readFile(artifactPath, 'utf8').catch(() => '') : undefined)"
-                  - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
-                  - 500
-              - set: artifactLower
-                value:
-                  expr: normalizeLowercaseStringOrEmpty(artifact)
-              - assert:
-                  expr: "artifact.length >= config.minBytes"
-                  message:
-                    expr: "`expected medium game artifact >= ${config.minBytes} bytes, got ${artifact.length}`"
-              - assert:
-                  expr: "artifactLower.includes('star garden defenders') && artifactLower.includes('<canvas') && artifactLower.includes('requestanimationframe')"
-                  message: missing title, canvas, or animation loop
-              - assert:
-                  expr: "artifactLower.includes('keydown') || artifactLower.includes('keyup')"
-                  message: missing keyboard controls
-              - assert:
-                  expr: "artifactLower.includes('score') && artifactLower.includes('wave') && artifactLower.includes('pause') && artifactLower.includes('restart')"
-                  message: missing score, wave, pause, or restart systems
-              - assert:
-                  expr: "outbound.text.includes(config.artifactFile)"
-                  message:
-                    expr: "`final reply did not mention ${config.artifactFile}: ${outbound.text}`"
-      detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live medium-game build' : `${config.artifactFile} bytes=${artifact.length}`"
+        - call: reset
+        - call: readSessionTranscriptSummary
+          saveAs: transcriptBefore
+          args:
+            - ref: env
+            - agent:qa:medium-game-openclaw
+            - allowEmpty: true
+        - call: runAgentPrompt
+          saveAs: turn
+          args:
+            - ref: env
+            - sessionKey: agent:qa:medium-game-openclaw
+              message:
+                expr: config.buildPrompt
+              provider:
+                expr: selected?.provider
+              model:
+                expr: selected?.model
+              timeoutMs:
+                expr: resolveQaLiveTurnTimeoutMs(env, 420000, env.primaryModel)
+        - call: waitForOutboundMessage
+          saveAs: outbound
+          args:
+            - ref: state
+            - lambda:
+                params: [candidate]
+                expr: "candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.artifactFile)"
+            - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
+        - call: readSessionTranscriptSummary
+          saveAs: transcript
+          args:
+            - ref: env
+            - agent:qa:medium-game-openclaw
+            - afterEventCursor:
+                expr: transcriptBefore.eventCursor
+        - assert:
+            expr: "(transcript.successfulToolCallCounts.update_plan ?? 0) > 0"
+            message: missing OpenClaw update_plan signal
+        - set: artifactPath
+          value:
+            expr: "path.join(env.gateway.workspaceDir, config.artifactFile)"
+        - call: waitForCondition
+          saveAs: artifact
+          args:
+            - lambda:
+                async: true
+                expr: "((await fs.readFile(artifactPath, 'utf8').catch(() => '')).includes(config.gameTitle) ? await fs.readFile(artifactPath, 'utf8').catch(() => '') : undefined)"
+            - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
+            - 500
+        - set: artifactLower
+          value:
+            expr: normalizeLowercaseStringOrEmpty(artifact)
+        - assert:
+            expr: "artifact.length >= config.minBytes"
+            message:
+              expr: "`expected medium game artifact >= ${config.minBytes} bytes, got ${artifact.length}`"
+        - assert:
+            expr: "artifactLower.includes('star garden defenders') && artifactLower.includes('<canvas') && artifactLower.includes('requestanimationframe')"
+            message: missing title, canvas, or animation loop
+        - assert:
+            expr: "artifactLower.includes('keydown') || artifactLower.includes('keyup')"
+            message: missing keyboard controls
+        - assert:
+            expr: "artifactLower.includes('score')"
+            message: missing score system
+        - assert:
+            expr: "artifactLower.includes('wave')"
+            message: missing wave system
+        - assert:
+            expr: "artifactLower.includes('pause')"
+            message: missing pause system
+        - assert:
+            expr: "artifactLower.includes('restart') || artifactLower.includes('resetgame')"
+            message: missing restart system
+        - assert:
+            expr: "outbound.text.includes(config.artifactFile)"
+            message:
+              expr: "`final reply did not mention ${config.artifactFile}: ${outbound.text}`"
+      detailsExpr: "`provider=${selected?.provider} model=${selected?.model} runtime=${config.harnessRuntime} agentRun=${turn.started.runId} appServerTurn=n/a planSignal=update_plan:success:${transcript.successfulToolCallCounts.update_plan ?? 0} artifact=${artifactPath} bytes=${artifact.length} visibleReply=${JSON.stringify(outbound.text)}`"