ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

test(tui): prove streaming and tool card rendering (#119251)

Browse Source
This commit is contained in:
Vincent Koc
2026-08-04 23:32:59 +08:00
committed by GitHub
parent cb2965ddde
commit cdd0856f4d
4 changed files with 183 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
qa/scenarios/ui/tui-streaming-tool-cards-pty.yaml
+33
View File
@@ -0,0 +1,33 @@
title: TUI streaming rendering and tool card PTY contracts
scenario:
id: tui-streaming-tool-cards-pty
surface: tui
category: tui.rendering-and-output-safety
coverage:
primary:
- tui.streaming-message-rendering
- tui.tool-cards
risk: medium
objective: Prove streaming progression and tool card lifecycle rendering through synchronized frames in the real runTui PTY loop.
successCriteria:
- A release-gated stream authenticates its exact 64-token prefix before continuing.
- The latest final frame contains all 128 tokens exactly once and in order.
- A release-gated verbose-full tool card authenticates running partial output before completion.
- The latest tool frame contains the final result and no running or stale partial marker.
codeRefs:
- src/tui/tui-pty-rendering-test-support.ts
- src/tui/tui-pty-harness-fixture-test-support.ts
- src/tui/tui-pty-harness.e2e.test.ts
execution:
kind: script
path: test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts
timeoutMs: 180000
args: [--artifact-base, "${outputDir}", --scenario-id, "${scenarioId}"]
config:
tuiPtyCases:
- coverageId: tui.streaming-message-rendering
testFile: src/tui/tui-pty-harness.e2e.test.ts
testNamePattern: ^TUI PTY harness authenticates a streamed prefix before the complete ordered final frame$
- coverageId: tui.tool-cards
testFile: src/tui/tui-pty-harness.e2e.test.ts
testNamePattern: ^TUI PTY harness authenticates running partial and completed tool cards in real terminal frames$
src/tui/tui-pty-harness-fixture-test-support.ts
+5 -70
View File
@@ -4,6 +4,7 @@ import path from "node:path";
import { pathToFileURL } from "node:url";
import { TUI_PTY_ASSISTANT_FIXTURE_SCRIPT } from "./tui-pty-assistant-fixture-test-support.js";
import { TUI_PTY_GAP_HISTORY_FIXTURE_SCRIPT } from "./tui-pty-gap-fixture-test-support.js";
import { TUI_PTY_RENDERING_FIXTURE_SCRIPT } from "./tui-pty-rendering-test-support.js";
import { TUI_PTY_RESET_FIXTURE } from "./tui-pty-reset-fixture-test-support.js";
import { TUI_PTY_SESSION_SUBSCRIPTION_FIXTURE_SCRIPT } from "./tui-pty-subscription-fixture-test-support.js";
@@ -25,7 +26,8 @@ export async function writeTuiPtyFixtureScript(dir: string) {
await writeFile(
scriptPath,
`
import { appendFileSync, existsSync } from "node:fs";
import { appendFileSync, existsSync, watch } from "node:fs";
import { dirname } from "node:path";
import { buildEmbeddedRunPayloads } from ${JSON.stringify(payloadsModuleUrl)};
import { getReplyPayloadMetadata } from ${JSON.stringify(replyPayloadModuleUrl)};
import { normalizeReplyPayloadsForDelivery } from ${JSON.stringify(outboundPayloadsModuleUrl)};
@@ -117,6 +119,7 @@ export async function writeTuiPtyFixtureScript(dir: string) {
${TUI_PTY_GAP_HISTORY_FIXTURE_SCRIPT}
${TUI_PTY_ASSISTANT_FIXTURE_SCRIPT}
${TUI_PTY_RENDERING_FIXTURE_SCRIPT}
class FixtureBackend implements TuiBackend {
connection = { url: "pty-fixture://local" };
@@ -158,31 +161,7 @@ export async function writeTuiPtyFixtureScript(dir: string) {
cause: new Error(escape + "[31mAuthorization: Bearer sk-abcdefghijklmnopqrstuv" + escape + "[0m"),
});
}
if (opts.message === "tool chronology proof") {
setTimeout(() => {
const emitAssistant = (state, text) => {
const message = {
role: "assistant",
content: [{ type: "text", text }],
timestamp: Date.now(),
};
this.onEvent?.({ event: "chat", payload: { runId, sessionKey: opts.sessionKey, state, message } });
};
emitAssistant("delta", "PTY_BEFORE_TOOL");
const data = {
phase: "start",
toolCallId: "pty-chronology-tool",
name: process.env.OPENCLAW_TUI_PTY_TOOL_NAME ?? "read_file",
args: { path: "chronology-proof.txt" },
};
this.onEvent?.({ event: "agent", payload: { runId, sessionKey: opts.sessionKey, stream: "tool", data } });
const completeText = "PTY_BEFORE_TOOL\\n\\nPTY_AFTER_TOOL";
emitAssistant("delta", completeText);
emitAssistant("final", completeText);
record("toolChronologyComplete", { runId });
}, 0);
return { runId };
}
if (startRenderingFixture(this, opts.message, runId, opts.sessionKey)) return { runId };
if (opts.message === "/btw picker focus proof") {
queueMicrotask(() => {
record("pickerSideResult", { runId, sessionKey: opts.sessionKey });
@@ -227,50 +206,6 @@ export async function writeTuiPtyFixtureScript(dir: string) {
}
return { runId };
}
if (opts.message === "burst streaming proof") {
const tokens = Array.from({ length: 128 }, (_, index) =>
"T" + String(index).padStart(3, "0"),
);
setTimeout(() => {
for (let index = 0; index < tokens.length; index += 1) {
this.onEvent?.({
event: "chat",
payload: {
runId,
sessionKey: opts.sessionKey,
state: "delta",
message: {
role: "assistant",
content: [
{
type: "text",
text: "PTY_STREAM_BURST: " + tokens.slice(0, index + 1).join(" "),
},
],
timestamp: Date.now(),
},
},
});
}
this.onEvent?.({
event: "chat",
payload: {
runId,
sessionKey: opts.sessionKey,
state: "final",
message: {
role: "assistant",
content: [
{ type: "text", text: "PTY_STREAM_BURST: " + tokens.join(" ") },
],
timestamp: Date.now(),
},
},
});
record("streamBurstComplete", { count: tokens.length });
}, 0);
return { runId };
}
if (opts.message === "history gap proof") { return beginGapHistoryRecovery(this, runId, opts.sessionKey); }
if (opts.message === "skill approval proof" || opts.message === "skill approval gap proof") {
pendingPluginApproval = {
src/tui/tui-pty-harness.e2e.test.ts
+19 -40
View File
@@ -16,6 +16,12 @@ import {
writeTuiPtyFixtureScript,
type FixtureLogEntry,
} from "./tui-pty-harness-fixture-test-support.js";
import {
exerciseStreamingRendering,
exerciseToolCardRendering,
streamingPrefixFrame,
toolFrame,
} from "./tui-pty-rendering-test-support.js";
import { sleep, startPty, type PtyRun } from "./tui-pty-test-support.js";
const activeRuns: PtyRun[] = [];
@@ -54,6 +60,14 @@ async function startTuiFixture(opts: { env?: NodeJS.ProcessEnv } = {}) {
};
}
it("rejects rendering oracle false positives", () => {
const tokens = Array.from({ length: 64 }, (_, i) => `T${String(i).padStart(3, "0")}`);
const promptFrame = [`burst streaming proof ${tokens.join(" ")}`, "local ready | idle"];
const reversedTool = ["PTY_BEFORE_TOOL PTY_TOOL_PARTIAL Read File (running)"];
expect(streamingPrefixFrame(promptFrame)).toBe(false);
expect(toolFrame(reversedTool, false)).toBe(false);
});
describe.sequential("TUI PTY harness", () => {
let fixture: Awaited<ReturnType<typeof startTuiFixture>>;
let compactFooterFixture: Awaited<ReturnType<typeof startTuiFixture>>;
@@ -728,34 +742,8 @@ describe.sequential("TUI PTY harness", () => {
);
it(
"renders cumulative streamed text below the intervening tool in a real terminal",
async () => {
const chronologyFixture = await startTuiFixture({
env: {
OPENCLAW_TUI_PTY_MODEL: "fixture-provider/fixture-model",
OPENCLAW_TUI_PTY_VERBOSE_LEVEL: "on",
},
});
try {
await chronologyFixture.run.waitForOutput("local ready", STARTUP_TIMEOUT_MS);
await chronologyFixture.run.write("tool chronology proof\r");
await chronologyFixture.waitForLogEntry(
(entry) => entry.method === "toolChronologyComplete",
);
await chronologyFixture.run.waitForOutput("PTY_AFTER_TOOL");
const rendered = chronologyFixture.run.visibleOutput();
expect(rendered.lastIndexOf("PTY_BEFORE_TOOL")).toBeLessThan(
rendered.lastIndexOf("Read File"),
);
expect(rendered.lastIndexOf("Read File")).toBeLessThan(
rendered.lastIndexOf("PTY_AFTER_TOOL"),
);
} finally {
await chronologyFixture.cleanup();
}
},
"authenticates running partial and completed tool cards in real terminal frames",
async () => await exerciseToolCardRendering(startTuiFixture, STARTUP_TIMEOUT_MS),
STARTUP_TEST_TIMEOUT_MS,
);
@@ -822,18 +810,9 @@ describe.sequential("TUI PTY harness", () => {
);
it(
"renders all 128 ordered chat deltas without losing the final streamed token",
async () => {
await fixture.run.write("burst streaming proof\r", { delay: false });
const burst = await fixture.waitForLogEntry(
(entry) => entry.method === "streamBurstComplete" && objectFieldEquals(entry, "count", 128),
);
expect(burst.payload).toMatchObject({ count: 128 });
await fixture.run.waitForOutput("PTY_STREAM_BURST:");
await fixture.run.waitForOutput("T127");
},
TEST_TIMEOUT_MS,
"authenticates a streamed prefix before the complete ordered final frame",
async () => await exerciseStreamingRendering(startTuiFixture, STARTUP_TIMEOUT_MS),
STARTUP_TEST_TIMEOUT_MS,
);
it(
src/tui/tui-pty-rendering-test-support.ts
+126
View File
@@ -0,0 +1,126 @@
import { writeFile } from "node:fs/promises";
import {
type StartTuiPtyFixture,
waitForSynchronizedFrameRows,
} from "./tui-pty-harness-assertion-test-support.js";
const STREAM_PROMPT = "burst streaming proof";
const TOOL_PROMPT = "tool chronology proof";
const TOKENS = Array.from({ length: 128 }, (_, index) => `T${String(index).padStart(3, "0")}`);
const dimensions = { OPENCLAW_TUI_PTY_COLS: "120", OPENCLAW_TUI_PTY_ROWS: "32" };
type Fixture = Awaited<ReturnType<StartTuiPtyFixture>>;
const text = (rows: string[]) => rows.join("\n");
const tokens = (rows: string[]) => text(rows).match(/\bT\d{3}\b/gu) ?? [];
const occurrences = (value: string, marker: string) => value.split(marker).length - 1;
export function streamingPrefixFrame(rows: string[]) {
return (
tokens(rows).join(",") === TOKENS.slice(0, 64).join(",") &&
rows.some((row) => row.includes("streaming") && row.endsWith("| local ready"))
);
}
export function toolFrame(rows: string[], complete: boolean) {
const frame = text(rows);
if (!complete) {
const before = frame.indexOf("PTY_BEFORE_TOOL");
const running = frame.indexOf("Read File (running)");
const partial = frame.indexOf("PTY_TOOL_PARTIAL");
return before >= 0 && running >= 0 && partial >= 0 && before < running && running < partial;
}
const markers = ["PTY_BEFORE_TOOL", "Read File", "PTY_TOOL_RESULT", "PTY_AFTER_TOOL"];
return (
markers.every(
(marker, index) =>
occurrences(frame, marker) === 1 &&
(index === 0 || frame.indexOf(markers[index - 1]!) < frame.indexOf(marker)),
) &&
!frame.includes("(running)") &&
!frame.includes("PTY_TOOL_PARTIAL") &&
frame.includes("idle")
);
}
const release = async (fixture: Fixture, gate: string) =>
await writeFile(`${fixture.logPath}.${gate}.release`, "release\n", "utf8");
async function withFixture(
start: StartTuiPtyFixture,
env: NodeJS.ProcessEnv,
timeoutMs: number,
exercise: (fixture: Fixture) => Promise<void>,
) {
const fixture = await start({ env: { ...dimensions, ...env } });
try {
await fixture.run.waitForOutput("local ready", timeoutMs);
await exercise(fixture);
} finally {
await fixture.cleanup();
}
}
export async function exerciseStreamingRendering(start: StartTuiPtyFixture, timeoutMs: number) {
await withFixture(start, {}, timeoutMs, async (fixture) => {
await fixture.run.write(`${STREAM_PROMPT}\r`, { delay: false });
await waitForSynchronizedFrameRows(
fixture.run,
(rows) => streamingPrefixFrame(rows),
timeoutMs,
);
await release(fixture, "streaming");
await waitForSynchronizedFrameRows(
fixture.run,
(rows) => tokens(rows).join(",") === TOKENS.join(",") && text(rows).includes("idle"),
timeoutMs,
);
});
}
export async function exerciseToolCardRendering(start: StartTuiPtyFixture, timeoutMs: number) {
await withFixture(
start,
{
OPENCLAW_TUI_PTY_MODEL: "fixture-provider/fixture-model",
OPENCLAW_TUI_PTY_VERBOSE_LEVEL: "full",
},
timeoutMs,
async (fixture) => {
await fixture.run.write(`${TOOL_PROMPT}\r`, { delay: false });
await waitForSynchronizedFrameRows(fixture.run, (rows) => toolFrame(rows, false), timeoutMs);
await release(fixture, "tool");
await waitForSynchronizedFrameRows(fixture.run, (rows) => toolFrame(rows, true), timeoutMs);
},
);
}
export const TUI_PTY_RENDERING_FIXTURE_SCRIPT = `
const renderingTokens = Array.from({ length: 128 }, (_, i) => "T" + String(i).padStart(3, "0"));
async function waitForRenderingRelease(gate: string) {
const target = actionLogPath + "." + gate + ".release";
if (existsSync(target)) return;
await new Promise<void>((resolve, reject) => {
const watcher = watch(dirname(target), () => {
if (existsSync(target)) { watcher.close(); resolve(); }
});
watcher.on("error", (error) => { watcher.close(); reject(error); });
if (existsSync(target)) { watcher.close(); resolve(); }
});
}
function emitAssistant(backend, runId, sessionKey, state, value) {
backend.onEvent?.({ event: "chat", payload: { runId, sessionKey, state,
message: { role: "assistant", content: [{ type: "text", text: value }], timestamp: Date.now() } } });
}
async function runStreamingRendering(backend, runId, sessionKey) {
for (let i = 0; i < 64; i += 1) emitAssistant(backend, runId, sessionKey, "delta", "PTY_STREAM_BURST: " + renderingTokens.slice(0, i + 1).join(" "));
record("streamingPrefixReady", { runId, count: 64 }); await waitForRenderingRelease("streaming");
for (let i = 64; i < 128; i += 1) emitAssistant(backend, runId, sessionKey, "delta", "PTY_STREAM_BURST: " + renderingTokens.slice(0, i + 1).join(" "));
emitAssistant(backend, runId, sessionKey, "final", "PTY_STREAM_BURST: " + renderingTokens.join(" ")); record("streamingComplete", { runId, count: 128 });
}
async function runToolCardRendering(backend, runId, sessionKey) {
emitAssistant(backend, runId, sessionKey, "delta", "PTY_BEFORE_TOOL");
const base = { toolCallId: "pty-rendering-tool", name: process.env.OPENCLAW_TUI_PTY_TOOL_NAME ?? "read_file" };
backend.onEvent?.({ event: "agent", payload: { runId, sessionKey, stream: "tool", data: { ...base, phase: "start", args: { path: "chronology-proof.txt" } } } });
if (process.env.OPENCLAW_TUI_PTY_VERBOSE_LEVEL === "full") {
backend.onEvent?.({ event: "agent", payload: { runId, sessionKey, stream: "tool", data: { ...base, phase: "update", partialResult: { content: [{ type: "text", text: "PTY_TOOL_PARTIAL" }] } } } });
record("toolPartialReady", { runId }); await waitForRenderingRelease("tool");
backend.onEvent?.({ event: "agent", payload: { runId, sessionKey, stream: "tool", data: { ...base, phase: "result", result: { content: [{ type: "text", text: "PTY_TOOL_RESULT" }] } } } });
}
const finalText = "PTY_BEFORE_TOOL\\n\\nPTY_AFTER_TOOL"; emitAssistant(backend, runId, sessionKey, "delta", finalText); emitAssistant(backend, runId, sessionKey, "final", finalText); record("toolComplete", { runId }); record("toolChronologyComplete", { runId });
}
function startRenderingFixture(backend, message, runId, sessionKey) {
const task = message === ${JSON.stringify(STREAM_PROMPT)} ? runStreamingRendering : message === ${JSON.stringify(TOOL_PROMPT)} ? runToolCardRendering : undefined;
if (!task) return false; void task(backend, runId, sessionKey); return true;
}
`;