diff --git a/qa/scenarios/ui/tui-streaming-tool-cards-pty.yaml b/qa/scenarios/ui/tui-streaming-tool-cards-pty.yaml new file mode 100644 index 000000000000..cf9084ea365c --- /dev/null +++ b/qa/scenarios/ui/tui-streaming-tool-cards-pty.yaml @@ -0,0 +1,33 @@ +title: TUI streaming rendering and tool card PTY contracts +scenario: + id: tui-streaming-tool-cards-pty + surface: tui + category: tui.rendering-and-output-safety + coverage: + primary: + - tui.streaming-message-rendering + - tui.tool-cards + risk: medium + objective: Prove streaming progression and tool card lifecycle rendering through synchronized frames in the real runTui PTY loop. + successCriteria: + - A release-gated stream authenticates its exact 64-token prefix before continuing. + - The latest final frame contains all 128 tokens exactly once and in order. + - A release-gated verbose-full tool card authenticates running partial output before completion. + - The latest tool frame contains the final result and no running or stale partial marker. + codeRefs: + - src/tui/tui-pty-rendering-test-support.ts + - src/tui/tui-pty-harness-fixture-test-support.ts + - src/tui/tui-pty-harness.e2e.test.ts + execution: + kind: script + path: test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts + timeoutMs: 180000 + args: [--artifact-base, "${outputDir}", --scenario-id, "${scenarioId}"] + config: + tuiPtyCases: + - coverageId: tui.streaming-message-rendering + testFile: src/tui/tui-pty-harness.e2e.test.ts + testNamePattern: ^TUI PTY harness authenticates a streamed prefix before the complete ordered final frame$ + - coverageId: tui.tool-cards + testFile: src/tui/tui-pty-harness.e2e.test.ts + testNamePattern: ^TUI PTY harness authenticates running partial and completed tool cards in real terminal frames$ diff --git a/src/tui/tui-pty-harness-fixture-test-support.ts b/src/tui/tui-pty-harness-fixture-test-support.ts index 704bd250542f..33d7ab717bd2 100644 --- a/src/tui/tui-pty-harness-fixture-test-support.ts +++ b/src/tui/tui-pty-harness-fixture-test-support.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; import { TUI_PTY_ASSISTANT_FIXTURE_SCRIPT } from "./tui-pty-assistant-fixture-test-support.js"; import { TUI_PTY_GAP_HISTORY_FIXTURE_SCRIPT } from "./tui-pty-gap-fixture-test-support.js"; +import { TUI_PTY_RENDERING_FIXTURE_SCRIPT } from "./tui-pty-rendering-test-support.js"; import { TUI_PTY_RESET_FIXTURE } from "./tui-pty-reset-fixture-test-support.js"; import { TUI_PTY_SESSION_SUBSCRIPTION_FIXTURE_SCRIPT } from "./tui-pty-subscription-fixture-test-support.js"; @@ -25,7 +26,8 @@ export async function writeTuiPtyFixtureScript(dir: string) { await writeFile( scriptPath, ` - import { appendFileSync, existsSync } from "node:fs"; + import { appendFileSync, existsSync, watch } from "node:fs"; + import { dirname } from "node:path"; import { buildEmbeddedRunPayloads } from ${JSON.stringify(payloadsModuleUrl)}; import { getReplyPayloadMetadata } from ${JSON.stringify(replyPayloadModuleUrl)}; import { normalizeReplyPayloadsForDelivery } from ${JSON.stringify(outboundPayloadsModuleUrl)}; @@ -117,6 +119,7 @@ export async function writeTuiPtyFixtureScript(dir: string) { ${TUI_PTY_GAP_HISTORY_FIXTURE_SCRIPT} ${TUI_PTY_ASSISTANT_FIXTURE_SCRIPT} + ${TUI_PTY_RENDERING_FIXTURE_SCRIPT} class FixtureBackend implements TuiBackend { connection = { url: "pty-fixture://local" }; @@ -158,31 +161,7 @@ export async function writeTuiPtyFixtureScript(dir: string) { cause: new Error(escape + "[31mAuthorization: Bearer sk-abcdefghijklmnopqrstuv" + escape + "[0m"), }); } - if (opts.message === "tool chronology proof") { - setTimeout(() => { - const emitAssistant = (state, text) => { - const message = { - role: "assistant", - content: [{ type: "text", text }], - timestamp: Date.now(), - }; - this.onEvent?.({ event: "chat", payload: { runId, sessionKey: opts.sessionKey, state, message } }); - }; - emitAssistant("delta", "PTY_BEFORE_TOOL"); - const data = { - phase: "start", - toolCallId: "pty-chronology-tool", - name: process.env.OPENCLAW_TUI_PTY_TOOL_NAME ?? "read_file", - args: { path: "chronology-proof.txt" }, - }; - this.onEvent?.({ event: "agent", payload: { runId, sessionKey: opts.sessionKey, stream: "tool", data } }); - const completeText = "PTY_BEFORE_TOOL\\n\\nPTY_AFTER_TOOL"; - emitAssistant("delta", completeText); - emitAssistant("final", completeText); - record("toolChronologyComplete", { runId }); - }, 0); - return { runId }; - } + if (startRenderingFixture(this, opts.message, runId, opts.sessionKey)) return { runId }; if (opts.message === "/btw picker focus proof") { queueMicrotask(() => { record("pickerSideResult", { runId, sessionKey: opts.sessionKey }); @@ -227,50 +206,6 @@ export async function writeTuiPtyFixtureScript(dir: string) { } return { runId }; } - if (opts.message === "burst streaming proof") { - const tokens = Array.from({ length: 128 }, (_, index) => - "T" + String(index).padStart(3, "0"), - ); - setTimeout(() => { - for (let index = 0; index < tokens.length; index += 1) { - this.onEvent?.({ - event: "chat", - payload: { - runId, - sessionKey: opts.sessionKey, - state: "delta", - message: { - role: "assistant", - content: [ - { - type: "text", - text: "PTY_STREAM_BURST: " + tokens.slice(0, index + 1).join(" "), - }, - ], - timestamp: Date.now(), - }, - }, - }); - } - this.onEvent?.({ - event: "chat", - payload: { - runId, - sessionKey: opts.sessionKey, - state: "final", - message: { - role: "assistant", - content: [ - { type: "text", text: "PTY_STREAM_BURST: " + tokens.join(" ") }, - ], - timestamp: Date.now(), - }, - }, - }); - record("streamBurstComplete", { count: tokens.length }); - }, 0); - return { runId }; - } if (opts.message === "history gap proof") { return beginGapHistoryRecovery(this, runId, opts.sessionKey); } if (opts.message === "skill approval proof" || opts.message === "skill approval gap proof") { pendingPluginApproval = { diff --git a/src/tui/tui-pty-harness.e2e.test.ts b/src/tui/tui-pty-harness.e2e.test.ts index 34c6ea3f8b50..9542c8ca8a01 100644 --- a/src/tui/tui-pty-harness.e2e.test.ts +++ b/src/tui/tui-pty-harness.e2e.test.ts @@ -16,6 +16,12 @@ import { writeTuiPtyFixtureScript, type FixtureLogEntry, } from "./tui-pty-harness-fixture-test-support.js"; +import { + exerciseStreamingRendering, + exerciseToolCardRendering, + streamingPrefixFrame, + toolFrame, +} from "./tui-pty-rendering-test-support.js"; import { sleep, startPty, type PtyRun } from "./tui-pty-test-support.js"; const activeRuns: PtyRun[] = []; @@ -54,6 +60,14 @@ async function startTuiFixture(opts: { env?: NodeJS.ProcessEnv } = {}) { }; } +it("rejects rendering oracle false positives", () => { + const tokens = Array.from({ length: 64 }, (_, i) => `T${String(i).padStart(3, "0")}`); + const promptFrame = [`burst streaming proof ${tokens.join(" ")}`, "local ready | idle"]; + const reversedTool = ["PTY_BEFORE_TOOL PTY_TOOL_PARTIAL Read File (running)"]; + expect(streamingPrefixFrame(promptFrame)).toBe(false); + expect(toolFrame(reversedTool, false)).toBe(false); +}); + describe.sequential("TUI PTY harness", () => { let fixture: Awaited>; let compactFooterFixture: Awaited>; @@ -728,34 +742,8 @@ describe.sequential("TUI PTY harness", () => { ); it( - "renders cumulative streamed text below the intervening tool in a real terminal", - async () => { - const chronologyFixture = await startTuiFixture({ - env: { - OPENCLAW_TUI_PTY_MODEL: "fixture-provider/fixture-model", - OPENCLAW_TUI_PTY_VERBOSE_LEVEL: "on", - }, - }); - - try { - await chronologyFixture.run.waitForOutput("local ready", STARTUP_TIMEOUT_MS); - await chronologyFixture.run.write("tool chronology proof\r"); - await chronologyFixture.waitForLogEntry( - (entry) => entry.method === "toolChronologyComplete", - ); - await chronologyFixture.run.waitForOutput("PTY_AFTER_TOOL"); - - const rendered = chronologyFixture.run.visibleOutput(); - expect(rendered.lastIndexOf("PTY_BEFORE_TOOL")).toBeLessThan( - rendered.lastIndexOf("Read File"), - ); - expect(rendered.lastIndexOf("Read File")).toBeLessThan( - rendered.lastIndexOf("PTY_AFTER_TOOL"), - ); - } finally { - await chronologyFixture.cleanup(); - } - }, + "authenticates running partial and completed tool cards in real terminal frames", + async () => await exerciseToolCardRendering(startTuiFixture, STARTUP_TIMEOUT_MS), STARTUP_TEST_TIMEOUT_MS, ); @@ -822,18 +810,9 @@ describe.sequential("TUI PTY harness", () => { ); it( - "renders all 128 ordered chat deltas without losing the final streamed token", - async () => { - await fixture.run.write("burst streaming proof\r", { delay: false }); - const burst = await fixture.waitForLogEntry( - (entry) => entry.method === "streamBurstComplete" && objectFieldEquals(entry, "count", 128), - ); - - expect(burst.payload).toMatchObject({ count: 128 }); - await fixture.run.waitForOutput("PTY_STREAM_BURST:"); - await fixture.run.waitForOutput("T127"); - }, - TEST_TIMEOUT_MS, + "authenticates a streamed prefix before the complete ordered final frame", + async () => await exerciseStreamingRendering(startTuiFixture, STARTUP_TIMEOUT_MS), + STARTUP_TEST_TIMEOUT_MS, ); it( diff --git a/src/tui/tui-pty-rendering-test-support.ts b/src/tui/tui-pty-rendering-test-support.ts new file mode 100644 index 000000000000..8e1a0b2cdc63 --- /dev/null +++ b/src/tui/tui-pty-rendering-test-support.ts @@ -0,0 +1,126 @@ +import { writeFile } from "node:fs/promises"; +import { + type StartTuiPtyFixture, + waitForSynchronizedFrameRows, +} from "./tui-pty-harness-assertion-test-support.js"; +const STREAM_PROMPT = "burst streaming proof"; +const TOOL_PROMPT = "tool chronology proof"; +const TOKENS = Array.from({ length: 128 }, (_, index) => `T${String(index).padStart(3, "0")}`); +const dimensions = { OPENCLAW_TUI_PTY_COLS: "120", OPENCLAW_TUI_PTY_ROWS: "32" }; +type Fixture = Awaited>; +const text = (rows: string[]) => rows.join("\n"); +const tokens = (rows: string[]) => text(rows).match(/\bT\d{3}\b/gu) ?? []; +const occurrences = (value: string, marker: string) => value.split(marker).length - 1; +export function streamingPrefixFrame(rows: string[]) { + return ( + tokens(rows).join(",") === TOKENS.slice(0, 64).join(",") && + rows.some((row) => row.includes("streaming") && row.endsWith("| local ready")) + ); +} +export function toolFrame(rows: string[], complete: boolean) { + const frame = text(rows); + if (!complete) { + const before = frame.indexOf("PTY_BEFORE_TOOL"); + const running = frame.indexOf("Read File (running)"); + const partial = frame.indexOf("PTY_TOOL_PARTIAL"); + return before >= 0 && running >= 0 && partial >= 0 && before < running && running < partial; + } + const markers = ["PTY_BEFORE_TOOL", "Read File", "PTY_TOOL_RESULT", "PTY_AFTER_TOOL"]; + return ( + markers.every( + (marker, index) => + occurrences(frame, marker) === 1 && + (index === 0 || frame.indexOf(markers[index - 1]!) < frame.indexOf(marker)), + ) && + !frame.includes("(running)") && + !frame.includes("PTY_TOOL_PARTIAL") && + frame.includes("idle") + ); +} +const release = async (fixture: Fixture, gate: string) => + await writeFile(`${fixture.logPath}.${gate}.release`, "release\n", "utf8"); +async function withFixture( + start: StartTuiPtyFixture, + env: NodeJS.ProcessEnv, + timeoutMs: number, + exercise: (fixture: Fixture) => Promise, +) { + const fixture = await start({ env: { ...dimensions, ...env } }); + try { + await fixture.run.waitForOutput("local ready", timeoutMs); + await exercise(fixture); + } finally { + await fixture.cleanup(); + } +} +export async function exerciseStreamingRendering(start: StartTuiPtyFixture, timeoutMs: number) { + await withFixture(start, {}, timeoutMs, async (fixture) => { + await fixture.run.write(`${STREAM_PROMPT}\r`, { delay: false }); + await waitForSynchronizedFrameRows( + fixture.run, + (rows) => streamingPrefixFrame(rows), + timeoutMs, + ); + await release(fixture, "streaming"); + await waitForSynchronizedFrameRows( + fixture.run, + (rows) => tokens(rows).join(",") === TOKENS.join(",") && text(rows).includes("idle"), + timeoutMs, + ); + }); +} +export async function exerciseToolCardRendering(start: StartTuiPtyFixture, timeoutMs: number) { + await withFixture( + start, + { + OPENCLAW_TUI_PTY_MODEL: "fixture-provider/fixture-model", + OPENCLAW_TUI_PTY_VERBOSE_LEVEL: "full", + }, + timeoutMs, + async (fixture) => { + await fixture.run.write(`${TOOL_PROMPT}\r`, { delay: false }); + await waitForSynchronizedFrameRows(fixture.run, (rows) => toolFrame(rows, false), timeoutMs); + await release(fixture, "tool"); + await waitForSynchronizedFrameRows(fixture.run, (rows) => toolFrame(rows, true), timeoutMs); + }, + ); +} +export const TUI_PTY_RENDERING_FIXTURE_SCRIPT = ` + const renderingTokens = Array.from({ length: 128 }, (_, i) => "T" + String(i).padStart(3, "0")); + async function waitForRenderingRelease(gate: string) { + const target = actionLogPath + "." + gate + ".release"; + if (existsSync(target)) return; + await new Promise((resolve, reject) => { + const watcher = watch(dirname(target), () => { + if (existsSync(target)) { watcher.close(); resolve(); } + }); + watcher.on("error", (error) => { watcher.close(); reject(error); }); + if (existsSync(target)) { watcher.close(); resolve(); } + }); + } + function emitAssistant(backend, runId, sessionKey, state, value) { + backend.onEvent?.({ event: "chat", payload: { runId, sessionKey, state, + message: { role: "assistant", content: [{ type: "text", text: value }], timestamp: Date.now() } } }); + } + async function runStreamingRendering(backend, runId, sessionKey) { + for (let i = 0; i < 64; i += 1) emitAssistant(backend, runId, sessionKey, "delta", "PTY_STREAM_BURST: " + renderingTokens.slice(0, i + 1).join(" ")); + record("streamingPrefixReady", { runId, count: 64 }); await waitForRenderingRelease("streaming"); + for (let i = 64; i < 128; i += 1) emitAssistant(backend, runId, sessionKey, "delta", "PTY_STREAM_BURST: " + renderingTokens.slice(0, i + 1).join(" ")); + emitAssistant(backend, runId, sessionKey, "final", "PTY_STREAM_BURST: " + renderingTokens.join(" ")); record("streamingComplete", { runId, count: 128 }); + } + async function runToolCardRendering(backend, runId, sessionKey) { + emitAssistant(backend, runId, sessionKey, "delta", "PTY_BEFORE_TOOL"); + const base = { toolCallId: "pty-rendering-tool", name: process.env.OPENCLAW_TUI_PTY_TOOL_NAME ?? "read_file" }; + backend.onEvent?.({ event: "agent", payload: { runId, sessionKey, stream: "tool", data: { ...base, phase: "start", args: { path: "chronology-proof.txt" } } } }); + if (process.env.OPENCLAW_TUI_PTY_VERBOSE_LEVEL === "full") { + backend.onEvent?.({ event: "agent", payload: { runId, sessionKey, stream: "tool", data: { ...base, phase: "update", partialResult: { content: [{ type: "text", text: "PTY_TOOL_PARTIAL" }] } } } }); + record("toolPartialReady", { runId }); await waitForRenderingRelease("tool"); + backend.onEvent?.({ event: "agent", payload: { runId, sessionKey, stream: "tool", data: { ...base, phase: "result", result: { content: [{ type: "text", text: "PTY_TOOL_RESULT" }] } } } }); + } + const finalText = "PTY_BEFORE_TOOL\\n\\nPTY_AFTER_TOOL"; emitAssistant(backend, runId, sessionKey, "delta", finalText); emitAssistant(backend, runId, sessionKey, "final", finalText); record("toolComplete", { runId }); record("toolChronologyComplete", { runId }); + } + function startRenderingFixture(backend, message, runId, sessionKey) { + const task = message === ${JSON.stringify(STREAM_PROMPT)} ? runStreamingRendering : message === ${JSON.stringify(TOOL_PROMPT)} ? runToolCardRendering : undefined; + if (!task) return false; void task(backend, runId, sessionKey); return true; + } +`;