diff --git a/config/knip.config.ts b/config/knip.config.ts index 31b66f0372cc..ee3e9ee02830 100644 --- a/config/knip.config.ts +++ b/config/knip.config.ts @@ -116,8 +116,6 @@ const rootEntries = [ "scripts/print-cli-backend-live-metadata.ts!", // Workflow/package-script entrypoints are not imported from production modules. "scripts/openclaw-cross-os-release-checks.ts!", - "scripts/bench-transcript-cursors.ts!", - "scripts/bench-codex-transcript-mirror.ts!", "scripts/bench-sqlite-reliability.ts!", // Docker/manual E2E executables and their nested assertion/probe entrypoints. "scripts/e2e/*.{js,mjs,ts}!", diff --git a/scripts/bench-codex-transcript-mirror.ts b/scripts/bench-codex-transcript-mirror.ts deleted file mode 100644 index d0de49bc492a..000000000000 --- a/scripts/bench-codex-transcript-mirror.ts +++ /dev/null @@ -1,349 +0,0 @@ -// Benchmarks the real Codex transcript mirror against a large indexed SQLite transcript. -import { execFileSync } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import type { SQLInputValue } from "node:sqlite"; -import { codexTranscriptMirrorRuntime } from "../extensions/codex/src/app-server/transcript-mirror.js"; -import { attachCodexMirrorIdentity } from "../extensions/codex/src/app-server/upstream-prompt-provenance.js"; -import { upsertSessionEntry } from "../src/config/sessions/session-accessor.js"; -import type { AgentMessage } from "../src/plugin-sdk/agent-core.js"; -import { - closeOpenClawAgentDatabasesForTest, - openOpenClawAgentDatabase, -} from "../src/state/openclaw-agent-db.js"; -import { closeOpenClawStateDatabaseForTest } from "../src/state/openclaw-state-db.js"; - -const DEFAULT_EVENT_COUNT = 100_000; -const DEFAULT_PAYLOAD_BYTES = 64; -const DEFAULT_RUNS = 8; -const DEFAULT_WARMUPS = 2; -const NEW_MESSAGES_PER_OPERATION = 2; - -type MirrorTarget = { - agentId: string; - sessionId: string; - sessionKey: string; - storePath: string; -}; - -type WorkCounters = { - fullTranscriptQueries: number; - seededEventJsonParses: number; - selectQueries: number; -}; - -function readIntegerArg(name: string, fallback: number): number { - const raw = process.argv.find((arg) => arg.startsWith(`--${name}=`))?.slice(name.length + 3); - if (raw === undefined) { - return fallback; - } - const value = Number(raw); - if (!Number.isSafeInteger(value) || value < 1) { - throw new Error(`--${name} must be a positive integer`); - } - return value; -} - -function readSourceSha(): string { - const value = process.argv - .find((arg) => arg.startsWith("--source-sha=")) - ?.slice("--source-sha=".length); - if (!value || !/^[a-f0-9]{40}$/u.test(value)) { - throw new Error("benchmark requires --source-sha=<40-character commit SHA>"); - } - const checkoutSha = execFileSync("git", ["rev-parse", "HEAD"], { - cwd: path.resolve(import.meta.dirname, ".."), - encoding: "utf8", - }).trim(); - if (checkoutSha !== value) { - throw new Error(`source SHA ${value} does not match checkout HEAD ${checkoutSha}`); - } - return value; -} - -function median(values: readonly number[]): number { - const sorted = values.toSorted((left, right) => left - right); - const upperIndex = Math.floor(sorted.length / 2); - const upper = sorted[upperIndex] ?? 0; - const lower = sorted.length % 2 === 0 ? (sorted[upperIndex - 1] ?? upper) : upper; - return Number(((lower + upper) / 2).toFixed(3)); -} - -function percentile(values: readonly number[], fraction: number): number { - const sorted = values.toSorted((left, right) => left - right); - const index = Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1); - return Number((sorted[Math.max(0, index)] ?? 0).toFixed(3)); -} - -/** Seeds a fully indexed linear transcript without charging setup to measured owner calls. */ -function seedTranscript(params: { - database: ReturnType; - eventCount: number; - payloadText: string; - sessionId: string; -}): void { - const { database, eventCount, payloadText, sessionId } = params; - const insertEvent = database.db.prepare( - `INSERT INTO transcript_events (session_id, seq, event_json, created_at) - VALUES (?, ?, ?, ?)`, - ); - const insertIdentity = database.db.prepare( - `INSERT INTO transcript_event_identities ( - session_id, event_id, seq, event_type, parent_id, message_idempotency_key, created_at - ) VALUES (?, ?, ?, 'message', ?, ?, ?)`, - ); - const insertActive = database.db.prepare( - `INSERT INTO session_transcript_active_events ( - session_id, active_position, event_seq, message_position - ) VALUES (?, ?, ?, ?)`, - ); - const now = Date.now(); - database.db.exec("BEGIN IMMEDIATE"); - try { - for (let seq = 0; seq < eventCount; seq += 1) { - const eventId = `benchmark-event-${seq}`; - const parentId = seq === 0 ? null : `benchmark-event-${seq - 1}`; - const idempotencyKey = `seed:${sessionId}:${seq}`; - const role = seq % 2 === 0 ? "user" : "assistant"; - const event = { - id: eventId, - message: { - content: role === "user" ? payloadText : [{ type: "text", text: payloadText }], - idempotencyKey, - role, - timestamp: now + seq, - }, - parentId, - timestamp: now + seq, - type: "message", - }; - insertEvent.run(sessionId, seq, JSON.stringify(event), now + seq); - const identityValues = [ - sessionId, - eventId, - seq, - parentId, - idempotencyKey, - now + seq, - ] satisfies SQLInputValue[]; - insertIdentity.run(...identityValues); - insertActive.run(sessionId, seq, seq, seq); - } - database.db - .prepare( - `INSERT INTO session_transcript_index_state ( - session_id, indexed_seq, leaf_event_id, needs_rebuild, - active_event_count, active_message_count, updated_at - ) VALUES (?, ?, ?, 0, ?, ?, ?)`, - ) - .run( - sessionId, - eventCount - 1, - `benchmark-event-${eventCount - 1}`, - eventCount, - eventCount, - now + eventCount, - ); - database.db.exec("COMMIT"); - } catch (error) { - database.db.exec("ROLLBACK"); - throw error; - } -} - -function instrumentWork(database: ReturnType): { - counters: WorkCounters; - reset: () => void; - restore: () => void; -} { - const counters: WorkCounters = { - fullTranscriptQueries: 0, - seededEventJsonParses: 0, - selectQueries: 0, - }; - const originalPrepare = database.db.prepare.bind(database.db); - const originalParse = JSON.parse; - Object.defineProperty(database.db, "prepare", { - configurable: true, - value: (sql: string) => { - const normalized = sql.replaceAll(/\s+/gu, " ").trim().toLowerCase(); - if (normalized.startsWith("select ")) { - counters.selectQueries += 1; - } - if ( - /from "?transcript_events"?/u.test(normalized) && - normalized.includes("event_json") && - /order by "?seq"? asc/u.test(normalized) - ) { - counters.fullTranscriptQueries += 1; - } - return originalPrepare(sql); - }, - }); - JSON.parse = ((text: string, reviver?: Parameters[1]) => { - if (text.includes('"id":"benchmark-event-')) { - counters.seededEventJsonParses += 1; - } - return originalParse(text, reviver); - }) as typeof JSON.parse; - return { - counters, - reset: () => { - counters.fullTranscriptQueries = 0; - counters.seededEventJsonParses = 0; - counters.selectQueries = 0; - }, - restore: () => { - Object.defineProperty(database.db, "prepare", { - configurable: true, - value: originalPrepare, - }); - JSON.parse = originalParse; - }, - }; -} - -function buildPromptFinalBatch(ordinal: number): AgentMessage[] { - return [ - attachCodexMirrorIdentity( - { - role: "user", - content: `benchmark prompt ${ordinal}`, - timestamp: 2_000_000_000_000 + ordinal, - } as AgentMessage, - `turn-${ordinal}:prompt`, - ), - attachCodexMirrorIdentity( - { - role: "assistant", - content: [{ type: "text", text: `benchmark final ${ordinal}` }], - timestamp: 2_000_000_100_000 + ordinal, - } as AgentMessage, - `turn-${ordinal}:assistant`, - ), - ]; -} - -async function runMirror(target: MirrorTarget, ordinal: number): Promise { - await codexTranscriptMirrorRuntime.mirror({ - ...target, - idempotencyScope: "codex-app-server:benchmark", - messages: buildPromptFinalBatch(ordinal), - }); -} - -async function main(): Promise { - const sourceSha = readSourceSha(); - const eventCount = readIntegerArg("events", DEFAULT_EVENT_COUNT); - const payloadBytes = readIntegerArg("payload-bytes", DEFAULT_PAYLOAD_BYTES); - const runs = readIntegerArg("runs", DEFAULT_RUNS); - const warmups = readIntegerArg("warmups", DEFAULT_WARMUPS); - const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-codex-mirror-bench-")); - const agentId = "benchmark"; - const sessionId = "codex-mirror-benchmark"; - const sessionKey = `agent:${agentId}:${sessionId}`; - try { - const database = openOpenClawAgentDatabase({ - agentId, - path: path.join(stateDir, "openclaw-agent.sqlite"), - }); - await upsertSessionEntry( - { agentId, sessionKey, storePath: database.path }, - { sessionId, updatedAt: 1 }, - ); - seedTranscript({ - database, - eventCount, - payloadText: "x".repeat(payloadBytes), - sessionId, - }); - const target = { agentId, sessionId, sessionKey, storePath: database.path }; - const instrumentation = instrumentWork(database); - try { - for (let ordinal = 0; ordinal < warmups; ordinal += 1) { - await runMirror(target, ordinal); - } - instrumentation.reset(); - const beforeMaxRssKb = process.resourceUsage().maxRSS; - const durations: number[] = []; - for (let run = 0; run < runs; run += 1) { - const startedAt = performance.now(); - await runMirror(target, warmups + run); - durations.push(performance.now() - startedAt); - } - const afterMaxRssKb = process.resourceUsage().maxRSS; - const measuredWork = { ...instrumentation.counters }; - const lastOrdinal = warmups + runs - 1; - await runMirror(target, lastOrdinal); - const row = database.db - .prepare( - `SELECT COUNT(*) AS count, - SUM(LENGTH(CAST(event_json AS BLOB))) AS bytes - FROM transcript_events - WHERE session_id = ?`, - ) - .get(sessionId) as { bytes: number; count: number }; - const expectedEvents = eventCount + NEW_MESSAGES_PER_OPERATION * (warmups + runs); - if (row.count !== expectedEvents) { - throw new Error(`mirror wrote ${row.count} events; expected ${expectedEvents}`); - } - console.log( - JSON.stringify( - { - sourceSha, - fixture: { - initialMessageEvents: eventCount, - payloadBytes, - sqliteTranscriptBytesAfterOperations: row.bytes, - }, - operation: "real Codex mirror owner with one new prompt and one new final", - runtime: { - arch: process.arch, - node: process.version, - platform: `${os.platform()} ${os.release()}`, - }, - warmups, - runs, - latencyMs: { - median: median(durations), - p95: percentile(durations, 0.95), - raw: durations.map((value) => Number(value.toFixed(3))), - }, - memoryProxy: { - maxRssKbBeforeOperations: beforeMaxRssKb, - maxRssKbAfterOperations: afterMaxRssKb, - maxRssGrowthKb: Math.max(0, afterMaxRssKb - beforeMaxRssKb), - }, - measuredWork: { - ...measuredWork, - perOperation: { - fullTranscriptQueries: Number( - (measuredWork.fullTranscriptQueries / runs).toFixed(3), - ), - seededEventJsonParses: Number( - (measuredWork.seededEventJsonParses / runs).toFixed(3), - ), - selectQueries: Number((measuredWork.selectQueries / runs).toFixed(3)), - }, - }, - correctness: { - idempotentReplayAddedRows: 0, - storedEventCount: row.count, - }, - }, - null, - 2, - ), - ); - } finally { - instrumentation.restore(); - } - } finally { - closeOpenClawAgentDatabasesForTest(); - closeOpenClawStateDatabaseForTest(); - fs.rmSync(stateDir, { force: true, recursive: true }); - } -} - -await main(); diff --git a/scripts/bench-transcript-cursors.ts b/scripts/bench-transcript-cursors.ts deleted file mode 100644 index 51afd3d58978..000000000000 --- a/scripts/bench-transcript-cursors.ts +++ /dev/null @@ -1,193 +0,0 @@ -// Benchmarks generation-aware raw transcript reads against a 100k-event session. -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { - loadTranscriptEventsSync, - readTranscriptRawDelta, -} from "../src/config/sessions/session-accessor.js"; -import { - closeOpenClawAgentDatabasesForTest, - openOpenClawAgentDatabase, -} from "../src/state/openclaw-agent-db.js"; -import { closeOpenClawStateDatabaseForTest } from "../src/state/openclaw-state-db.js"; - -const EVENT_COUNT = 100_000; -const DELTA_COUNT = 20; -const RUNS = 12; - -type Timing = { - p50Ms: number; - p95Ms: number; -}; - -function percentile(values: readonly number[], fraction: number): number { - const sorted = values.toSorted((left, right) => left - right); - const index = Math.min(sorted.length - 1, Math.floor(sorted.length * fraction)); - return Number((sorted[index] ?? 0).toFixed(3)); -} - -function measure(operation: () => void): Timing { - const values: number[] = []; - for (let index = 0; index < RUNS; index += 1) { - const startedAt = performance.now(); - operation(); - values.push(performance.now() - startedAt); - } - return { p50Ms: percentile(values, 0.5), p95Ms: percentile(values, 0.95) }; -} - -function seedTranscript( - database: ReturnType, - sessionId: string, - sessionKey: string, -): void { - database.db.exec("BEGIN IMMEDIATE"); - try { - const now = Date.now(); - database.db - .prepare( - `INSERT INTO session_nodes (session_key, current_session_id, entry_json, updated_at) - VALUES (?, ?, '{}', ?)`, - ) - .run(sessionKey, sessionId, now); - database.db - .prepare( - `INSERT INTO session_windows ( - session_id, session_key, session_scope, created_at, updated_at - ) VALUES (?, ?, 'conversation', ?, ?)`, - ) - .run(sessionId, sessionKey, now, now); - database.db - .prepare( - `INSERT INTO transcript_rewrite_watermarks (session_id, generation, updated_at) - VALUES (?, 'benchmark-generation', ?)`, - ) - .run(sessionId, now); - const insert = database.db.prepare( - `INSERT INTO transcript_events (session_id, seq, event_json, created_at) - VALUES (?, ?, ?, ?)`, - ); - for (let seq = 0; seq < EVENT_COUNT; seq += 1) { - insert.run( - sessionId, - seq, - JSON.stringify({ id: `event-${seq}`, text: "x".repeat(64), type: "custom" }), - now + seq, - ); - } - database.db.exec("COMMIT"); - } catch (error) { - database.db.exec("ROLLBACK"); - throw error; - } -} - -function appendDelta( - database: ReturnType, - sessionId: string, -): void { - const insert = database.db.prepare( - `INSERT INTO transcript_events (session_id, seq, event_json, created_at) - VALUES (?, ?, ?, ?)`, - ); - database.db.exec("BEGIN IMMEDIATE"); - try { - for (let offset = 0; offset < DELTA_COUNT; offset += 1) { - const seq = EVENT_COUNT + offset; - insert.run( - sessionId, - seq, - JSON.stringify({ id: `delta-${offset}`, text: "delta", type: "custom" }), - Date.now() + offset, - ); - } - database.db.exec("COMMIT"); - } catch (error) { - database.db.exec("ROLLBACK"); - throw error; - } -} - -function readFrontierCursor(scope: Parameters[0]): string { - let cursor: string | undefined; - for (;;) { - const page = readTranscriptRawDelta(scope, { - ...(cursor ? { cursor } : {}), - maxBytes: 64 * 1024 * 1024, - maxEvents: 10_000, - }); - if (page.kind !== "page") { - throw new Error(`unexpected cursor bootstrap result: ${page.kind}`); - } - cursor = page.cursor; - if (!page.hasMore) { - return cursor; - } - } -} - -function main(): void { - const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-transcript-bench-")); - const env = { ...process.env, OPENCLAW_STATE_DIR: stateDir }; - const agentId = "benchmark"; - const sessionId = "cursor-benchmark"; - const sessionKey = `agent:${agentId}:cursor-benchmark`; - try { - const database = openOpenClawAgentDatabase({ agentId, env }); - seedTranscript(database, sessionId, sessionKey); - const scope = { agentId, sessionId, sessionKey, storePath: database.path }; - - const bootstrap = measure(() => { - const page = readTranscriptRawDelta(scope, { maxBytes: 1_000_000, maxEvents: 1_000 }); - if (page.kind !== "page" || page.events.length !== 1_000) { - throw new Error("raw bootstrap did not return 1,000 events"); - } - }); - const activeMemoryBounded = measure(() => { - const page = readTranscriptRawDelta(scope, { - maxBytes: 50 * 1024 * 1024, - maxEvents: 2_000, - }); - if (page.kind !== "page" || page.events.length !== 2_000) { - throw new Error("active-memory bounds did not return 2,000 events"); - } - }); - const frontierCursor = readFrontierCursor(scope); - appendDelta(database, sessionId); - const shortDelta = measure(() => { - const page = readTranscriptRawDelta(scope, { - cursor: frontierCursor, - maxBytes: 1_000_000, - maxEvents: 100, - }); - if (page.kind !== "page" || page.events.length !== DELTA_COUNT || page.hasMore) { - throw new Error("raw delta did not return the appended frontier"); - } - }); - const fullRead = measure(() => { - if (loadTranscriptEventsSync(scope).length !== EVENT_COUNT + DELTA_COUNT) { - throw new Error("full transcript read returned the wrong row count"); - } - }); - - console.log( - JSON.stringify( - { - events: EVENT_COUNT, - appendedEvents: DELTA_COUNT, - runs: RUNS, - timings: { activeMemoryBounded, bootstrap, fullRead, shortDelta }, - }, - null, - 2, - ), - ); - } finally { - closeOpenClawAgentDatabasesForTest(); - closeOpenClawStateDatabaseForTest(); - fs.rmSync(stateDir, { force: true, recursive: true }); - } -} - -main(); diff --git a/scripts/check-changed.mjs b/scripts/check-changed.mjs index ff83c8c73ced..d6cedb1cd56d 100644 --- a/scripts/check-changed.mjs +++ b/scripts/check-changed.mjs @@ -23,6 +23,7 @@ import { booleanFlag, parseFlagArgs, stringFlag } from "./lib/arg-utils.mjs"; import { getChangedPathFacts, normalizeChangedPath } from "./lib/changed-path-facts.mjs"; import { printTimingSummary } from "./lib/check-timing-summary.mjs"; import { isDirectRunUrl } from "./lib/direct-run.mjs"; +import { runWithFailedTrailer } from "./lib/failed-trailer.mjs"; import { acquireLocalHeavyCheckLockSync, resolveLocalHeavyCheckEnv, @@ -1080,14 +1081,15 @@ function isDirectRun() { return isDirectRunUrl(process.argv[1], import.meta.url); } -if (isDirectRun()) { +async function main() { const argv = process.argv.slice(2); let args; try { args = parseArgs(argv); } catch (error) { console.error(error instanceof Error ? error.message : String(error)); - process.exit(1); + process.exitCode = 1; + return; } if (args.help) { printUsage(); @@ -1159,3 +1161,7 @@ if (isDirectRun()) { } } } + +if (isDirectRun()) { + await runWithFailedTrailer("check:changed", main); +} diff --git a/scripts/check-deprecated-api-usage.mjs b/scripts/check-deprecated-api-usage.mjs index 60e55388cd06..18b1717c954a 100644 --- a/scripts/check-deprecated-api-usage.mjs +++ b/scripts/check-deprecated-api-usage.mjs @@ -2,7 +2,7 @@ // Scans source files for usage of deprecated API markers. import fs from "node:fs"; import path from "node:path"; -import { collectDeprecatedInternalConfigApiViolations } from "./lib/deprecated-config-api-guard.mjs"; +import { collectDeprecatedInternalConfigApiViolations } from "./lib/config-boundary-guard.mjs"; import { BANNED_INTERNAL_PLUGIN_SDK_FACADE_MODULES, buildDeprecatedPluginSdkModuleSpecifiers, diff --git a/scripts/check-ingress-agent-owner-context.mjs b/scripts/check-ingress-agent-owner-context.mjs index adccc441907f..cd949fd94c9a 100644 --- a/scripts/check-ingress-agent-owner-context.mjs +++ b/scripts/check-ingress-agent-owner-context.mjs @@ -34,7 +34,7 @@ function findLegacyAgentCommandCallLines(content, fileName = "source.ts") { /** * Runs the ingress owner-context guard. */ -export async function main() { +async function main() { await runCallsiteGuard({ importMetaUrl: import.meta.url, sourceRoots, diff --git a/scripts/check-kysely-guardrails.mjs b/scripts/check-kysely-guardrails.mjs index 6b64fc437e4d..de91253a36cf 100644 --- a/scripts/check-kysely-guardrails.mjs +++ b/scripts/check-kysely-guardrails.mjs @@ -493,7 +493,7 @@ async function collectKyselyGuardrails() { /** * Runs the Kysely guardrail check. */ -export async function main() { +async function main() { const violations = await collectKyselyGuardrails(); if (violations.length === 0) { console.log("Kysely guardrails OK"); diff --git a/scripts/check-no-raw-channel-fetch.mjs b/scripts/check-no-raw-channel-fetch.mjs index fc765e80f2b6..1d830bc90507 100644 --- a/scripts/check-no-raw-channel-fetch.mjs +++ b/scripts/check-no-raw-channel-fetch.mjs @@ -94,7 +94,7 @@ function findRawFetchCallLines(content, fileName = "source.ts") { /** * Runs the raw channel/plugin fetch guard. */ -export async function main() { +async function main() { await runCallsiteGuard({ importMetaUrl: import.meta.url, sourceRoots, diff --git a/scripts/check-webhook-auth-body-order.mjs b/scripts/check-webhook-auth-body-order.mjs index fb7dc09a138f..16aac32901d3 100644 --- a/scripts/check-webhook-auth-body-order.mjs +++ b/scripts/check-webhook-auth-body-order.mjs @@ -52,7 +52,7 @@ function findBlockedWebhookBodyReadLines(content, fileName = "source.ts") { /** * Runs the webhook auth/body-order guard. */ -export async function main() { +async function main() { await runCallsiteGuard({ importMetaUrl: import.meta.url, sourceRoots, diff --git a/scripts/debug-claude-usage.ts b/scripts/debug-claude-usage.ts index d31a213ea378..c29ed0cd2236 100644 --- a/scripts/debug-claude-usage.ts +++ b/scripts/debug-claude-usage.ts @@ -7,7 +7,7 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; import { expectDefined } from "../packages/normalization-core/src/expect.js"; import { normalizeOptionalString } from "../packages/normalization-core/src/string-coerce.js"; -import { readBoundedResponseText as readBoundedResponseTextWithLimit } from "./lib/bounded-response.ts"; +import { readBoundedResponseText as readBoundedResponseTextWithLimit } from "./lib/bounded-response.mjs"; import { maskIdentifier, parseStrictIntegerOption, diff --git a/scripts/dev/discord-acp-plain-language-smoke.ts b/scripts/dev/discord-acp-plain-language-smoke.ts index 24223f1567fd..4dbde00c0c2f 100644 --- a/scripts/dev/discord-acp-plain-language-smoke.ts +++ b/scripts/dev/discord-acp-plain-language-smoke.ts @@ -9,7 +9,7 @@ import { pathToFileURL } from "node:url"; import { promisify } from "node:util"; import { formatErrorMessage } from "../../src/infra/errors.ts"; import { createPluginStateKeyedStore } from "../../src/plugin-state/plugin-state-store.ts"; -import { readBoundedResponseText } from "../lib/bounded-response.ts"; +import { readBoundedResponseText } from "../lib/bounded-response.mjs"; import { maskIdentifier, parseStrictIntegerOption, diff --git a/scripts/dev/realtime-talk-live-smoke.ts b/scripts/dev/realtime-talk-live-smoke.ts index d9ad967b76e4..04fc4927791f 100644 --- a/scripts/dev/realtime-talk-live-smoke.ts +++ b/scripts/dev/realtime-talk-live-smoke.ts @@ -3,7 +3,7 @@ import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; -import { readBoundedResponseText } from "../lib/bounded-response.ts"; +import { readBoundedResponseText } from "../lib/bounded-response.mjs"; import { parseStrictIntegerOption, previewForDevToolLog, diff --git a/scripts/docs-link-audit.d.mts b/scripts/docs-link-audit.d.mts index 95ec99aacf0d..56da6638c27b 100644 --- a/scripts/docs-link-audit.d.mts +++ b/scripts/docs-link-audit.d.mts @@ -1,10 +1,3 @@ -export type BrokenDocLink = { - file: string; - line: number; - link: string; - reason: string; -}; - export type ResolveRouteResult = { ok: boolean; terminal: string; @@ -35,15 +28,6 @@ export type ScriptSpawn = ( options: ScriptSpawnOptions, ) => ScriptSpawnResult; -export type ScriptInvocation = { - command: string; - args: string[]; - options?: Partial & { - detached?: boolean; - windowsVerbatimArguments?: boolean; - }; -}; - export function normalizeRoute(route: string): string; export function resolveRoute( route: string, @@ -76,25 +60,6 @@ export function prepareMirroredDocsDir( export function prepareAnchorAuditDocsDir(sourceDir?: string): string; -export function resolveMintlifyAnchorAuditInvocation(params: { - cwd: string; - nodeVersion?: string; - spawnSyncImpl: ScriptSpawn; - env?: NodeJS.ProcessEnv; - nodeExecPath?: string; - npmExecPath?: string; - platform?: NodeJS.Platform; - comSpec?: string; -}): ScriptInvocation; - -export function auditDocsLinks(options?: { - docsDir?: string; - allowExternalClawHubRoutes?: boolean; -}): { - checked: number; - broken: BrokenDocLink[]; -}; - export function runDocsLinkAuditCli(options?: { args?: string[]; comSpec?: string; diff --git a/scripts/docs-link-audit.mjs b/scripts/docs-link-audit.mjs index c24a5484d6b6..00422275d9f8 100644 --- a/scripts/docs-link-audit.mjs +++ b/scripts/docs-link-audit.mjs @@ -610,7 +610,7 @@ function createMintlifyNpmRunnerSpawnSpec(params, options = {}) { * comSpec?: string; * }} params */ -export function resolveMintlifyAnchorAuditInvocation(params) { +function resolveMintlifyAnchorAuditInvocation(params) { const nodeVersion = params.nodeVersion ?? process.versions.node; if (parseNodeMajor(nodeVersion) < NODE_25_UNSUPPORTED_BY_MINTLIFY) { return createMintlifyNpmRunnerSpawnSpec(params); @@ -647,7 +647,7 @@ export function resolveMintlifyAnchorAuditInvocation(params) { /** * Audits local docs links against route, file, and redirect indexes. */ -export function auditDocsLinks(options = {}) { +function auditDocsLinks(options = {}) { const docsDir = options.docsDir ?? DOCS_DIR; const index = buildAuditIndex(docsDir, { allowExternalClawHubRoutes: options.allowExternalClawHubRoutes === true, diff --git a/scripts/e2e/lib/bounded-response-text.d.mts b/scripts/e2e/lib/bounded-response-text.d.mts deleted file mode 100644 index 332b5f403a24..000000000000 --- a/scripts/e2e/lib/bounded-response-text.d.mts +++ /dev/null @@ -1,13 +0,0 @@ -export function readBoundedResponseBytes( - response: unknown, - label: unknown, - byteLimit: unknown, - timeoutPromise?: Promise, -): Promise; - -export function readBoundedResponseText( - response: unknown, - label: unknown, - byteLimit: unknown, - timeoutPromise?: Promise, -): Promise; diff --git a/scripts/e2e/lib/bounded-response-text.mjs b/scripts/e2e/lib/bounded-response-text.mjs deleted file mode 100644 index cf8244be0232..000000000000 --- a/scripts/e2e/lib/bounded-response-text.mjs +++ /dev/null @@ -1,72 +0,0 @@ -// Bounded response body reader used by E2E HTTP fixture clients. -function bodyTooLargeError(label, byteLimit) { - return Object.assign(new Error(`${label} response body exceeded ${byteLimit} bytes`), { - code: "ETOOBIG", - }); -} - -function cancelReaderSoon(reader) { - void Promise.resolve() - .then(() => reader.cancel()) - .catch(() => {}); -} - -function parseContentLengthHeader(headers) { - const raw = headers.get("content-length"); - if (!raw || !/^\d+$/u.test(raw)) { - return undefined; - } - const parsed = Number(raw); - return Number.isSafeInteger(parsed) ? parsed : Number.POSITIVE_INFINITY; -} - -export async function readBoundedResponseBytes(response, label, byteLimit, timeoutPromise) { - const contentLength = parseContentLengthHeader(response.headers); - if (contentLength !== undefined && contentLength > byteLimit) { - await response.body?.cancel().catch(() => {}); - throw bodyTooLargeError(label, byteLimit); - } - if (!response.body) { - return Buffer.alloc(0); - } - - const reader = response.body.getReader(); - const chunks = []; - let byteCount = 0; - let canceled = false; - try { - while (true) { - const read = reader.read(); - const readWithTimeout = timeoutPromise - ? Promise.race([ - read, - timeoutPromise.catch((error) => { - canceled = true; - cancelReaderSoon(reader); - throw error; - }), - ]) - : read; - const { done, value } = await readWithTimeout; - if (done) { - return Buffer.concat(chunks, byteCount); - } - byteCount += value.byteLength; - if (byteCount > byteLimit) { - canceled = true; - await reader.cancel().catch(() => {}); - throw bodyTooLargeError(label, byteLimit); - } - chunks.push(Buffer.from(value)); - } - } finally { - if (!canceled) { - reader.releaseLock(); - } - } -} - -export async function readBoundedResponseText(response, label, byteLimit, timeoutPromise) { - const bytes = await readBoundedResponseBytes(response, label, byteLimit, timeoutPromise); - return new TextDecoder().decode(bytes); -} diff --git a/scripts/e2e/lib/bundled-plugin-install-uninstall/runtime-smoke.mjs b/scripts/e2e/lib/bundled-plugin-install-uninstall/runtime-smoke.mjs index 27fc6b58374f..6913d597f387 100644 --- a/scripts/e2e/lib/bundled-plugin-install-uninstall/runtime-smoke.mjs +++ b/scripts/e2e/lib/bundled-plugin-install-uninstall/runtime-smoke.mjs @@ -6,9 +6,12 @@ import path from "node:path"; import process from "node:process"; import { setTimeout as delay } from "node:timers/promises"; import { fileURLToPath } from "node:url"; +import { + createBoundedResponseTooLargeError, + readBoundedResponseText, +} from "../../../lib/bounded-response.mjs"; import { isRecord } from "../../../lib/record-shared.mjs"; import { resolveWindowsTaskkillPath } from "../../../lib/windows-taskkill.mjs"; -import { readBoundedResponseText } from "../bounded-response-text.mjs"; const TOKEN = "bundled-plugin-runtime-smoke-token"; const RUNTIME_PORT_BASE_ENV = "OPENCLAW_BUNDLED_PLUGIN_RUNTIME_PORT_BASE"; @@ -753,7 +756,7 @@ async function fetchHttpProbeStatus(port, pathName, options = {}) { res, `${pathName} probe`, HTTP_PROBE_BODY_MAX_BYTES, - timeoutPromise, + { createTooLargeError: createBoundedResponseTooLargeError, timeoutPromise }, ); status.bodyText = text; if (text.trim()) { diff --git a/scripts/e2e/lib/plugins/assertions.mjs b/scripts/e2e/lib/plugins/assertions.mjs index 3e3e56dbec47..799bf2ed1861 100644 --- a/scripts/e2e/lib/plugins/assertions.mjs +++ b/scripts/e2e/lib/plugins/assertions.mjs @@ -2,7 +2,10 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { readBoundedResponseText } from "../bounded-response-text.mjs"; +import { + createBoundedResponseTooLargeError, + readBoundedResponseText, +} from "../../../lib/bounded-response.mjs"; import { readPositiveIntEnv } from "../env-limits.mjs"; import { readPluginInstallIndex, @@ -864,7 +867,7 @@ async function assertClawHubPreflight() { response, `ClawHub package preflight response for ${packageName}`, limits.bodyMaxBytes, - timeoutPromise, + { createTooLargeError: createBoundedResponseTooLargeError, timeoutPromise }, ), ); throw new Error( @@ -879,7 +882,7 @@ async function assertClawHubPreflight() { response, `ClawHub package preflight response for ${packageName}`, limits.bodyMaxBytes, - timeoutPromise, + { createTooLargeError: createBoundedResponseTooLargeError, timeoutPromise }, ), ); const detail = await withTimeout( diff --git a/scripts/e2e/lib/plugins/npm-registry-server.mjs b/scripts/e2e/lib/plugins/npm-registry-server.mjs index cf2a0e3c14d6..e10c3f8ce353 100644 --- a/scripts/e2e/lib/plugins/npm-registry-server.mjs +++ b/scripts/e2e/lib/plugins/npm-registry-server.mjs @@ -5,7 +5,10 @@ import { once } from "node:events"; import fs from "node:fs"; import http from "node:http"; import path from "node:path"; -import { readBoundedResponseBytes } from "../bounded-response-text.mjs"; +import { + createBoundedResponseTooLargeError, + readBoundedResponseBytes, +} from "../../../lib/bounded-response.mjs"; const [portFile, ...packageArgs] = process.argv.slice(2); function normalizeUpstreamRegistry(raw) { @@ -159,6 +162,7 @@ async function proxyUpstream(rawRequestUrl, response) { upstreamResponse, "npm registry upstream", UPSTREAM_RESPONSE_MAX_BYTES, + { createTooLargeError: createBoundedResponseTooLargeError }, ); // Fetch decodes compressed bodies but preserves upstream length metadata. // Emit the decoded size so npm clients do not truncate proxied responses. diff --git a/scripts/e2e/lib/release-user-journey/assertions.mjs b/scripts/e2e/lib/release-user-journey/assertions.mjs index 9fca303b777e..dd0acfcf22bc 100644 --- a/scripts/e2e/lib/release-user-journey/assertions.mjs +++ b/scripts/e2e/lib/release-user-journey/assertions.mjs @@ -2,11 +2,14 @@ import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { + createBoundedResponseTooLargeError, + readBoundedResponseText as readBoundedResponseTextWithLimit, +} from "../../../lib/bounded-response.mjs"; import { assertAgentReplyContainsMarker, assertOpenAiRequestLogUsed, } from "../agent-turn-output.mjs"; -import { readBoundedResponseText as readBoundedResponseTextWithLimit } from "../bounded-response-text.mjs"; import { applyMockOpenAiModelConfig, parseMockOpenAiPort, @@ -83,7 +86,10 @@ async function readBoundedResponseText( byteLimit = clickClackHttpBodyMaxBytes(), options = {}, ) { - return await readBoundedResponseTextWithLimit(response, label, byteLimit, options.timeoutPromise); + return await readBoundedResponseTextWithLimit(response, label, byteLimit, { + createTooLargeError: createBoundedResponseTooLargeError, + timeoutPromise: options.timeoutPromise, + }); } async function readBoundedResponseJson(response, label, options = {}) { diff --git a/scripts/e2e/mcp-code-mode-gateway-client.ts b/scripts/e2e/mcp-code-mode-gateway-client.ts index 5eec99b7e681..40c73505ccf9 100644 --- a/scripts/e2e/mcp-code-mode-gateway-client.ts +++ b/scripts/e2e/mcp-code-mode-gateway-client.ts @@ -4,7 +4,7 @@ import { setTimeout as setNodeTimeout, clearTimeout as clearNodeTimeout } from " import { pathToFileURL } from "node:url"; import { getSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; import { readSessionTranscriptEvents } from "openclaw/plugin-sdk/session-transcript-runtime"; -import { readBoundedResponseText } from "../lib/bounded-response.ts"; +import { readBoundedResponseText } from "../lib/bounded-response.mjs"; import { readPositiveIntEnv } from "./lib/env-limits.mjs"; import { extractMcpCodeModePlannedTools, diff --git a/scripts/e2e/npm-telegram-live-docker.sh b/scripts/e2e/npm-telegram-live-docker.sh index d76144d97523..2c222e7e4886 100755 --- a/scripts/e2e/npm-telegram-live-docker.sh +++ b/scripts/e2e/npm-telegram-live-docker.sh @@ -128,7 +128,7 @@ if [ -n "$resolved_package_dir" ]; then package_source_kind="prepared-package-set" package_mount_args=(-v "$resolved_package_dir:/package-under-test:ro") registry_helper_mount_args=( - -v "$ROOT_DIR/scripts/e2e/lib/bounded-response-text.mjs:/tmp/openclaw-e2e/lib/bounded-response-text.mjs:ro" + -v "$ROOT_DIR/scripts/lib/bounded-response.mjs:/tmp/lib/bounded-response.mjs:ro" -v "$ROOT_DIR/scripts/e2e/lib/plugins/npm-registry-server.mjs:/tmp/openclaw-e2e/lib/plugins/npm-registry-server.mjs:ro" ) elif [ -n "$resolved_package_tgz" ]; then diff --git a/scripts/e2e/openwebui-probe.mjs b/scripts/e2e/openwebui-probe.mjs index e5e12e7886a6..086d04301324 100644 --- a/scripts/e2e/openwebui-probe.mjs +++ b/scripts/e2e/openwebui-probe.mjs @@ -1,7 +1,10 @@ // Probe script for OpenWebUI E2E connectivity. import { Agent, setGlobalDispatcher } from "undici"; +import { + createBoundedResponseTooLargeError, + readBoundedResponseText as readBoundedResponseTextWithLimit, +} from "../lib/bounded-response.mjs"; import { escapeRegExp } from "../lib/regexp.mjs"; -import { readBoundedResponseText as readBoundedResponseTextWithLimit } from "./lib/bounded-response-text.mjs"; const baseUrl = process.env.OPENWEBUI_BASE_URL ?? ""; const email = process.env.OPENWEBUI_ADMIN_EMAIL ?? ""; @@ -112,12 +115,10 @@ async function withRequestTimeout(label, timeoutMs, run) { } async function readBoundedResponseText(response, label, timeoutPromise) { - return await readBoundedResponseTextWithLimit( - response, - label, - responseBodyMaxBytes, + return await readBoundedResponseTextWithLimit(response, label, responseBodyMaxBytes, { + createTooLargeError: createBoundedResponseTooLargeError, timeoutPromise, - ); + }); } async function readBoundedResponseJson(response, label, timeoutPromise) { diff --git a/scripts/e2e/telegram-bot-api.ts b/scripts/e2e/telegram-bot-api.ts index ad0849f55790..ecbc19acca6b 100644 --- a/scripts/e2e/telegram-bot-api.ts +++ b/scripts/e2e/telegram-bot-api.ts @@ -1,5 +1,5 @@ // Telegram Bot Api script supports OpenClaw repository automation. -import { readBoundedResponseText } from "../lib/bounded-response.ts"; +import { readBoundedResponseText } from "../lib/bounded-response.mjs"; import { readPositiveIntEnv } from "./lib/env-limits.mjs"; type JsonObject = Record; diff --git a/scripts/e2e/telegram-user-credential-io.ts b/scripts/e2e/telegram-user-credential-io.ts index 29342a9563b4..a29940e34e4f 100644 --- a/scripts/e2e/telegram-user-credential-io.ts +++ b/scripts/e2e/telegram-user-credential-io.ts @@ -1,6 +1,6 @@ // Telegram User Credential Io script supports OpenClaw repository automation. import { spawn, spawnSync } from "node:child_process"; -import { readBoundedResponseText } from "../lib/bounded-response.ts"; +import { readBoundedResponseText } from "../lib/bounded-response.mjs"; import { resolveWindowsTaskkillPath } from "../lib/windows-taskkill.mjs"; export type JsonObject = Record; diff --git a/scripts/firecrawl-compare.ts b/scripts/firecrawl-compare.ts index 8d03d845f121..1d59a104431d 100644 --- a/scripts/firecrawl-compare.ts +++ b/scripts/firecrawl-compare.ts @@ -4,7 +4,7 @@ import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import { fetchFirecrawlContent } from "../extensions/firecrawl/api.ts"; import { formatErrorMessage } from "../src/infra/errors.ts"; import { extractReadableContent } from "../src/web-fetch/content-extractors.runtime.js"; -import { readBoundedResponseText as readBoundedResponseTextWithLimit } from "./lib/bounded-response.ts"; +import { readBoundedResponseText as readBoundedResponseTextWithLimit } from "./lib/bounded-response.mjs"; const DEFAULT_URLS = [ "https://en.wikipedia.org/wiki/Web_scraping", diff --git a/scripts/gh-read.ts b/scripts/gh-read.ts index d5d7db8f2d6c..2dd7c1b94aef 100644 --- a/scripts/gh-read.ts +++ b/scripts/gh-read.ts @@ -5,7 +5,7 @@ import { pathToFileURL } from "node:url"; import { readSecretFileSync } from "@openclaw/fs-safe/secret"; import { expectDefined } from "../packages/normalization-core/src/expect.js"; import { truncateUtf16Safe } from "../packages/normalization-core/src/utf16-slice.js"; -import { readBoundedResponseText } from "./lib/bounded-response.ts"; +import { readBoundedResponseText } from "./lib/bounded-response.mjs"; import { parseStrictIntegerOption } from "./lib/dev-tooling-safety.ts"; import { normalizeGitHubRepo as normalizeRepo, diff --git a/scripts/lib/bounded-response.d.mts b/scripts/lib/bounded-response.d.mts index 0754f69b013d..a945c721e0db 100644 --- a/scripts/lib/bounded-response.d.mts +++ b/scripts/lib/bounded-response.d.mts @@ -1,7 +1,20 @@ -/** Read response text while enforcing max bytes before and during streaming. */ +export type BoundedResponseOptions = { + createTooLargeError?: (message: string) => Error; + formatTooLargeMessage?: (label: string, maxBytes: number) => string; + signal?: AbortSignal; + timeoutPromise?: Promise; +}; + +export function createBoundedResponseTooLargeError(message: string): Error & { code: "ETOOBIG" }; +export function readBoundedResponseBytes( + response: Response, + label: string, + maxBytes: number, + options?: BoundedResponseOptions, +): Promise; export function readBoundedResponseText( - response: unknown, - label: unknown, - maxBytes: unknown, - options?: Record, + response: Response, + label: string, + maxBytes: number, + options?: BoundedResponseOptions, ): Promise; diff --git a/scripts/lib/bounded-response.mjs b/scripts/lib/bounded-response.mjs index 29600386ffd2..12745a04a513 100644 --- a/scripts/lib/bounded-response.mjs +++ b/scripts/lib/bounded-response.mjs @@ -7,6 +7,10 @@ function defaultTooLargeError(message) { return new Error(message); } +export function createBoundedResponseTooLargeError(message) { + return Object.assign(new Error(message), { code: "ETOOBIG" }); +} + function cancelReaderSoon(reader) { void Promise.resolve() .then(() => reader.cancel()) @@ -77,8 +81,8 @@ async function readResponseChunkWithTimeout(reader, label, signal, timeoutPromis } } -/** Read response text while enforcing max bytes before and during streaming. */ -export async function readBoundedResponseText(response, label, maxBytes, options = {}) { +/** Read response bytes while enforcing max bytes before and during streaming. */ +export async function readBoundedResponseBytes(response, label, maxBytes, options = {}) { const formatTooLargeMessage = options.formatTooLargeMessage ?? defaultTooLargeMessage; const createTooLargeError = options.createTooLargeError ?? defaultTooLargeError; const tooLargeError = () => createTooLargeError(formatTooLargeMessage(label, maxBytes)); @@ -89,11 +93,10 @@ export async function readBoundedResponseText(response, label, maxBytes, options } if (!response.body) { - return ""; + return Buffer.alloc(0); } const reader = response.body.getReader(); - const decoder = new TextDecoder(); const chunks = []; let totalBytes = 0; let canceled = false; @@ -110,10 +113,6 @@ export async function readBoundedResponseText(response, label, maxBytes, options }, ); if (done) { - const tail = decoder.decode(); - if (tail) { - chunks.push(tail); - } break; } @@ -123,7 +122,7 @@ export async function readBoundedResponseText(response, label, maxBytes, options await reader.cancel().catch(() => undefined); throw tooLargeError(); } - chunks.push(decoder.decode(value, { stream: true })); + chunks.push(value); } } finally { if (!canceled) { @@ -131,7 +130,13 @@ export async function readBoundedResponseText(response, label, maxBytes, options } } - return chunks.join(""); + return Buffer.concat(chunks, totalBytes); +} + +/** Read response text while enforcing max bytes before and during streaming. */ +export async function readBoundedResponseText(response, label, maxBytes, options = {}) { + const bytes = await readBoundedResponseBytes(response, label, maxBytes, options); + return new TextDecoder().decode(bytes); } function toLintErrorObject(value, fallbackMessage) { diff --git a/scripts/lib/bounded-response.ts b/scripts/lib/bounded-response.ts deleted file mode 100644 index 6d3d3d66553f..000000000000 --- a/scripts/lib/bounded-response.ts +++ /dev/null @@ -1,161 +0,0 @@ -// Bounded Response script supports OpenClaw repository automation. -type BoundedResponseTextOptions = { - createTooLargeError?: (message: string) => Error; - formatTooLargeMessage?: (label: string, maxBytes: number) => string; - signal?: AbortSignal; - timeoutPromise?: Promise; -}; - -const defaultTooLargeMessage = (label: string, maxBytes: number) => - `${label} response body exceeded ${maxBytes} bytes`; - -const defaultTooLargeError = (message: string) => new Error(`${message}.`); - -function cancelReaderSoon(reader: ReadableStreamDefaultReader): void { - void Promise.resolve() - .then(() => reader.cancel()) - .catch(() => undefined); -} - -function parseContentLengthHeader(headers: Headers): number | undefined { - const raw = headers.get("content-length"); - if (!raw || !/^\d+$/u.test(raw)) { - return undefined; - } - const parsed = Number(raw); - return Number.isSafeInteger(parsed) ? parsed : Number.POSITIVE_INFINITY; -} - -async function readResponseChunk( - reader: ReadableStreamDefaultReader, - label: string, - signal: AbortSignal | undefined, - markCanceled: () => void, -): Promise> { - if (!signal) { - return await reader.read(); - } - if (signal.aborted) { - markCanceled(); - await reader.cancel().catch(() => undefined); - throw signal.reason instanceof Error ? signal.reason : new Error(`${label} request aborted`); - } - - let removeAbortListener: (() => void) | undefined; - const abortPromise = new Promise>((_resolve, reject) => { - const onAbort = () => { - markCanceled(); - reject( - signal.reason instanceof Error ? signal.reason : new Error(`${label} request aborted`), - ); - cancelReaderSoon(reader); - }; - signal.addEventListener("abort", onAbort, { once: true }); - removeAbortListener = () => signal.removeEventListener("abort", onAbort); - }); - - try { - return await Promise.race([reader.read(), abortPromise]); - } finally { - removeAbortListener?.(); - } -} - -function toErrorObject(value: unknown, fallbackMessage: string): Error { - if (value instanceof Error) { - return value; - } - if (typeof value === "string") { - return new Error(value); - } - return new Error(fallbackMessage, { cause: value }); -} - -async function readResponseChunkWithTimeout( - reader: ReadableStreamDefaultReader, - label: string, - signal: AbortSignal | undefined, - timeoutPromise: Promise | undefined, - markCanceled: () => void, -): Promise> { - const readPromise = readResponseChunk(reader, label, signal, markCanceled); - if (!timeoutPromise) { - return await readPromise; - } - - let waitingForRead = true; - const timeoutReadPromise = timeoutPromise.catch((error: unknown) => { - if (waitingForRead) { - markCanceled(); - cancelReaderSoon(reader); - } - throw toErrorObject(error, `${label} response body read timed out`); - }); - - try { - return await Promise.race([readPromise, timeoutReadPromise]); - } finally { - waitingForRead = false; - } -} - -export async function readBoundedResponseText( - response: Response, - label: string, - maxBytes: number, - options: BoundedResponseTextOptions = {}, -): Promise { - const formatTooLargeMessage = options.formatTooLargeMessage ?? defaultTooLargeMessage; - const createTooLargeError = options.createTooLargeError ?? defaultTooLargeError; - const tooLargeError = () => createTooLargeError(formatTooLargeMessage(label, maxBytes)); - const contentLength = parseContentLengthHeader(response.headers); - if (contentLength !== undefined && contentLength > maxBytes) { - await response.body?.cancel().catch(() => undefined); - throw tooLargeError(); - } - - if (!response.body) { - return ""; - } - - const reader = response.body.getReader(); - const decoder = new TextDecoder(); - const chunks: string[] = []; - let totalBytes = 0; - let canceled = false; - - try { - for (;;) { - const { done, value } = await readResponseChunkWithTimeout( - reader, - label, - options.signal, - options.timeoutPromise, - () => { - canceled = true; - }, - ); - if (done) { - const tail = decoder.decode(); - if (tail) { - chunks.push(tail); - } - break; - } - - totalBytes += value.byteLength; - if (totalBytes > maxBytes) { - canceled = true; - await reader.cancel().catch(() => undefined); - throw tooLargeError(); - } - chunks.push(decoder.decode(value, { stream: true })); - } - } finally { - if (!canceled) { - reader.releaseLock(); - } - } - - return chunks.join(""); -} diff --git a/scripts/lib/deprecated-config-api-guard.d.mts b/scripts/lib/deprecated-config-api-guard.d.mts deleted file mode 100644 index 21ffd0af02bd..000000000000 --- a/scripts/lib/deprecated-config-api-guard.d.mts +++ /dev/null @@ -1 +0,0 @@ -export { collectDeprecatedInternalConfigApiViolations } from "./config-boundary-guard.mjs"; diff --git a/scripts/lib/deprecated-config-api-guard.mjs b/scripts/lib/deprecated-config-api-guard.mjs deleted file mode 100644 index c6bc7cccea33..000000000000 --- a/scripts/lib/deprecated-config-api-guard.mjs +++ /dev/null @@ -1,2 +0,0 @@ -// Compatibility re-export for deprecated internal config API guard checks. -export { collectDeprecatedInternalConfigApiViolations } from "./config-boundary-guard.mjs"; diff --git a/scripts/lib/failed-trailer.d.mts b/scripts/lib/failed-trailer.d.mts new file mode 100644 index 000000000000..29977f6fe8ea --- /dev/null +++ b/scripts/lib/failed-trailer.d.mts @@ -0,0 +1,10 @@ +export function writeFailedTrailer( + tool: string, + exitCode: number | string | null | undefined, + log?: (value: unknown) => void, +): void; +export function runWithFailedTrailer( + tool: string, + run: () => void | Promise, + log?: (value: unknown) => void, +): Promise; diff --git a/scripts/lib/failed-trailer.mjs b/scripts/lib/failed-trailer.mjs new file mode 100644 index 000000000000..171c04cad94a --- /dev/null +++ b/scripts/lib/failed-trailer.mjs @@ -0,0 +1,16 @@ +// Keeps wrapper failures visible even when preceding diagnostics are truncated. +export function writeFailedTrailer(tool, exitCode, log = console.error) { + if (typeof exitCode === "number" && exitCode !== 0) { + log(`[${tool}] FAILED (exit ${exitCode})`); + } +} + +export async function runWithFailedTrailer(tool, run, log = console.error) { + try { + await run(); + } catch (error) { + log(error); + process.exitCode = 1; + } + writeFailedTrailer(tool, process.exitCode, log); +} diff --git a/scripts/lib/plugin-clawhub-release.ts b/scripts/lib/plugin-clawhub-release.ts index 2c409578372b..b075fa4d470e 100644 --- a/scripts/lib/plugin-clawhub-release.ts +++ b/scripts/lib/plugin-clawhub-release.ts @@ -5,7 +5,7 @@ import { truncateUtf16Safe } from "../../packages/normalization-core/src/utf16-s import { validateExternalCodePluginPackageJson } from "../../packages/plugin-package-contract/src/index.ts"; import { retryClawHubRead } from "../../src/infra/clawhub-retry.js"; import { runTasksWithConcurrency } from "../../src/utils/run-with-concurrency.js"; -import { readBoundedResponseText } from "./bounded-response.ts"; +import { readBoundedResponseText } from "./bounded-response.mjs"; import { assertPluginReleaseDependencyFreshness, collectExtensionPackageJsonCandidates, @@ -115,7 +115,8 @@ const CLAWHUB_SHARED_RELEASE_INPUT_PATHS = [ "package.json", "pnpm-lock.yaml", "packages/plugin-package-contract/src/index.ts", - "scripts/lib/bounded-response.ts", + "scripts/lib/bounded-response.d.mts", + "scripts/lib/bounded-response.mjs", "scripts/lib/npm-publish-plan.mjs", "scripts/lib/release-version.mjs", "scripts/lib/plugin-npm-release.ts", diff --git a/scripts/lib/release-beta-verifier.ts b/scripts/lib/release-beta-verifier.ts index 21d5cfcc5ff6..356c1a17e767 100644 --- a/scripts/lib/release-beta-verifier.ts +++ b/scripts/lib/release-beta-verifier.ts @@ -5,7 +5,7 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { readPublicationArtifactArchive, sha256Digest } from "./actions-artifact-archive.mjs"; -import { readBoundedResponseText } from "./bounded-response.ts"; +import { readBoundedResponseText } from "./bounded-response.mjs"; import { collectClawHubPublishablePluginPackages } from "./plugin-clawhub-release.ts"; import { collectPublishablePluginPackages, diff --git a/scripts/mcp-code-mode-gateway-e2e.ts b/scripts/mcp-code-mode-gateway-e2e.ts index a1b5a2372473..731e4cfe5047 100644 --- a/scripts/mcp-code-mode-gateway-e2e.ts +++ b/scripts/mcp-code-mode-gateway-e2e.ts @@ -23,7 +23,7 @@ import { validateMcpCodeModeResult, } from "./e2e/lib/mcp-code-mode-validation.ts"; import { countSessionLogMentions } from "./e2e/lib/session-log-mentions.ts"; -import { readBoundedResponseText } from "./lib/bounded-response.ts"; +import { readBoundedResponseText } from "./lib/bounded-response.mjs"; async function freePort(): Promise { return await new Promise((resolve, reject) => { diff --git a/scripts/openclaw-npm-postpublish-verify.ts b/scripts/openclaw-npm-postpublish-verify.ts index 9ca0a32008ea..465981ebfec0 100644 --- a/scripts/openclaw-npm-postpublish-verify.ts +++ b/scripts/openclaw-npm-postpublish-verify.ts @@ -27,7 +27,7 @@ import { pathToFileURL } from "node:url"; import { expectDefined } from "../packages/normalization-core/src/expect.js"; import { ALWAYS_ALLOWED_RUNTIME_DIR_NAMES } from "../src/plugin-sdk/facade-activation-contract.ts"; import { BUNDLED_RUNTIME_SIDECAR_PATHS } from "../src/plugins/runtime-sidecar-paths.ts"; -import { readBoundedResponseText } from "./lib/bounded-response.ts"; +import { readBoundedResponseText } from "./lib/bounded-response.mjs"; import { listBundledPluginPackArtifacts } from "./lib/bundled-plugin-build-entries.mjs"; import { formatErrorMessage } from "./lib/error-format.mjs"; import { runNpmVerifyCommand } from "./lib/npm-verify-exec.ts"; diff --git a/scripts/run-oxlint.d.mts b/scripts/run-oxlint.d.mts index 61d5e14e33ea..c334bb14517c 100644 --- a/scripts/run-oxlint.d.mts +++ b/scripts/run-oxlint.d.mts @@ -29,11 +29,3 @@ export function filterSparseMissingOxlintTargets( * Applies wrapper policy and runs oxlint with the final argument list. */ export function main(argv?: string[], runtimeEnv?: NodeJS.ProcessEnv): Promise; -/** - * CLI entry: converts wrapper crashes into exit 1 and ends every failing run - * with a stable `[oxlint] FAILED (exit N)` final line. - */ -export function runOxlintCliEntry( - run?: () => Promise, - log?: (message: unknown) => void, -): Promise; diff --git a/scripts/run-oxlint.mjs b/scripts/run-oxlint.mjs index 021cea0b4144..db7ae6440451 100644 --- a/scripts/run-oxlint.mjs +++ b/scripts/run-oxlint.mjs @@ -3,6 +3,7 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; +import { runWithFailedTrailer } from "./lib/failed-trailer.mjs"; import { acquireLocalHeavyCheckLockSync, applyLocalOxlintPolicy, @@ -281,24 +282,6 @@ export async function main(argv = process.argv.slice(2), runtimeEnv = process.en } } -/** - * CLI entry: converts wrapper crashes into a nonzero exit and ends every - * failing run with one stable final line. That line must survive output - * truncation (`… | tail -N`): without it, a crash or lint failure whose - * diagnostics scrolled away reads as success when only the tail is inspected. - */ -export async function runOxlintCliEntry(run = main, log = console.error) { - try { - await run(); - } catch (error) { - log(error); - process.exitCode = 1; - } - if (typeof process.exitCode === "number" && process.exitCode !== 0) { - log(`[oxlint] FAILED (exit ${process.exitCode})`); - } -} - if (import.meta.main) { - await runOxlintCliEntry(); + await runWithFailedTrailer("oxlint", main); } diff --git a/scripts/run-tsgo.mjs b/scripts/run-tsgo.mjs index 072ac46e1ddf..181fd63f9fac 100644 --- a/scripts/run-tsgo.mjs +++ b/scripts/run-tsgo.mjs @@ -3,6 +3,7 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; import { readFlagValue } from "./lib/arg-utils.mjs"; +import { runWithFailedTrailer } from "./lib/failed-trailer.mjs"; import { acquireLocalHeavyCheckLockSync, applyLocalTsgoPolicy, @@ -17,57 +18,63 @@ import { shouldSkipSparseTsgoGuardError, } from "./lib/tsgo-sparse-guard.mjs"; -const { args: finalArgs, env } = applyLocalTsgoPolicy( - process.argv.slice(2), - resolveLocalHeavyCheckEnv(process.env), -); +function main() { + const { args: finalArgs, env } = applyLocalTsgoPolicy( + process.argv.slice(2), + resolveLocalHeavyCheckEnv(process.env), + ); -const tsgoPath = resolveRepoToolBinPath("tsgo"); -const tsBuildInfoFile = readFlagValue(finalArgs, "--tsBuildInfoFile"); -if (tsBuildInfoFile) { - fs.mkdirSync(path.dirname(path.resolve(tsBuildInfoFile)), { recursive: true }); -} -const sparseGuardError = getSparseTsgoGuardError(finalArgs, { cwd: process.cwd() }); -const releaseLock = - sparseGuardError || - env.OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD === "1" || - !shouldAcquireLocalHeavyCheckLockForTsgo(finalArgs, env) - ? () => {} - : acquireLocalHeavyCheckLockSync({ - cwd: process.cwd(), + const tsgoPath = resolveRepoToolBinPath("tsgo"); + const tsBuildInfoFile = readFlagValue(finalArgs, "--tsBuildInfoFile"); + if (tsBuildInfoFile) { + fs.mkdirSync(path.dirname(path.resolve(tsBuildInfoFile)), { recursive: true }); + } + const sparseGuardError = getSparseTsgoGuardError(finalArgs, { cwd: process.cwd() }); + const releaseLock = + sparseGuardError || + env.OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD === "1" || + !shouldAcquireLocalHeavyCheckLockForTsgo(finalArgs, env) + ? () => {} + : acquireLocalHeavyCheckLockSync({ + cwd: process.cwd(), + env, + toolName: "tsgo", + }); + + try { + if (sparseGuardError) { + console.error(sparseGuardError); + if (shouldSkipSparseTsgoGuardError(env)) { + console.error("[tsgo] skipping sparse-missing project because OPENCLAW_TSGO_SPARSE_SKIP=1"); + process.exitCode = 0; + } else { + process.exitCode = 1; + } + } else { + ensureRepoToolNodeModulesLink(tsgoPath); + const tsgo = createManagedCommandInvocation({ + args: finalArgs, + bin: tsgoPath, env, - toolName: "tsgo", + }); + const result = spawnSync(tsgo.command, tsgo.args, { + stdio: "inherit", + env, + shell: tsgo.shell, + windowsVerbatimArguments: tsgo.windowsVerbatimArguments, }); -try { - if (sparseGuardError) { - console.error(sparseGuardError); - if (shouldSkipSparseTsgoGuardError(env)) { - console.error("[tsgo] skipping sparse-missing project because OPENCLAW_TSGO_SPARSE_SKIP=1"); - process.exitCode = 0; - } else { - process.exitCode = 1; - } - } else { - ensureRepoToolNodeModulesLink(tsgoPath); - const tsgo = createManagedCommandInvocation({ - args: finalArgs, - bin: tsgoPath, - env, - }); - const result = spawnSync(tsgo.command, tsgo.args, { - stdio: "inherit", - env, - shell: tsgo.shell, - windowsVerbatimArguments: tsgo.windowsVerbatimArguments, - }); + if (result.error) { + throw result.error; + } - if (result.error) { - throw result.error; + process.exitCode = result.status ?? 1; } - - process.exitCode = result.status ?? 1; + } finally { + releaseLock(); } -} finally { - releaseLock(); +} + +if (import.meta.main) { + await runWithFailedTrailer("tsgo", main); } diff --git a/scripts/run-vitest.mjs b/scripts/run-vitest.mjs index b7de3f5f0dc6..b1d1f6a0e30c 100644 --- a/scripts/run-vitest.mjs +++ b/scripts/run-vitest.mjs @@ -9,6 +9,8 @@ import { embeddedAgentVitestProjectOwners } from "../test/vitest/vitest.agents-p import { toolingIsolatedTestFiles } from "../test/vitest/vitest.tooling-isolated-paths.mjs"; import { isUiTestTarget } from "../test/vitest/vitest.ui-paths.mjs"; import { boundaryTestFiles } from "../test/vitest/vitest.unit-paths.mjs"; +import { runWithFailedTrailer, writeFailedTrailer } from "./lib/failed-trailer.mjs"; +import { signalExitCode } from "./lib/managed-child-process.mjs"; import { resolveLocalVitestEnv } from "./lib/vitest-local-scheduling.mjs"; import { spawnPnpmRunner } from "./pnpm-runner.mjs"; import { @@ -1107,10 +1109,22 @@ export function runTestProjectsDelegation(argv, env, options = {}) { return child; } -function main(argv = process.argv.slice(2), env = process.env) { +async function finishVitestProcess({ completion, getForwardedSignal }) { + const { code, signal } = await completion; + const exitSignal = getForwardedSignal() ?? signal; + if (exitSignal) { + writeFailedTrailer("vitest", signalExitCode(exitSignal)); + process.kill(process.pid, exitSignal); + return; + } + process.exitCode = code ?? 1; +} + +async function main(argv = process.argv.slice(2), env = process.env) { if (argv.length === 0) { console.error("usage: node scripts/run-vitest.mjs "); - process.exit(1); + process.exitCode = 1; + return; } const missingTestFiles = resolveMissingExplicitTestFiles(argv); @@ -1121,12 +1135,13 @@ function main(argv = process.argv.slice(2), env = process.env) { ...missingTestFiles.map((file) => ` - ${file}`), ].join("\n"), ); - process.exit(1); + process.exitCode = 1; + return; } const delegatedArgs = resolveTestProjectsDelegationArgs(argv); if (delegatedArgs) { - runTestProjectsDelegation(delegatedArgs, env); + await finishVitestProcess(spawnTestProjectsRunner(delegatedArgs, env)); return; } @@ -1139,38 +1154,28 @@ function main(argv = process.argv.slice(2), env = process.env) { } catch (error) { if (error instanceof Error && error.code === "OPENCLAW_MISSING_VITEST") { console.error(error.message); - process.exit(1); + process.exitCode = 1; + return; } throw error; } - const { completion, getForwardedSignal } = spawnWatchedVitestProcess({ - pnpmArgs: ["exec", "node", ...resolveVitestNodeArgs(env), vitestCliEntry, ...guardedVitestArgs], - spawnParams: resolveVitestSpawnParams(spawnEnv), - env: spawnEnv, - label: guardedVitestArgs.join(" "), - }); - - completion.then( - ({ code, signal }) => { - const forwardedSignal = getForwardedSignal(); - if (forwardedSignal) { - process.kill(process.pid, forwardedSignal); - return; - } - if (signal) { - process.kill(process.pid, signal); - return; - } - process.exit(code ?? 1); - }, - /** @param {unknown} error */ (error) => { - console.error(error); - process.exit(1); - }, + await finishVitestProcess( + spawnWatchedVitestProcess({ + pnpmArgs: [ + "exec", + "node", + ...resolveVitestNodeArgs(env), + vitestCliEntry, + ...guardedVitestArgs, + ], + spawnParams: resolveVitestSpawnParams(spawnEnv), + env: spawnEnv, + label: guardedVitestArgs.join(" "), + }), ); } if (import.meta.main) { - main(); + await runWithFailedTrailer("vitest", main); } diff --git a/scripts/test-projects.test-support.mjs b/scripts/test-projects.test-support.mjs index b0b8d7ed1aa7..a0cbb943ba3f 100644 --- a/scripts/test-projects.test-support.mjs +++ b/scripts/test-projects.test-support.mjs @@ -1987,6 +1987,7 @@ const EXACT_TOOLING_TARGETS = new Map([ ], ["scripts/run-vitest.mjs", ["run-vitest", "test-projects", "vitest-local-scheduling"]], ["scripts/run-oxlint-shards.mjs", ["run-oxlint"]], + ["scripts/lib/failed-trailer.mjs", ["run-oxlint", "run-tsgo", "run-vitest", "changed-lanes"]], ["scripts/docker-e2e-rerun.mjs", ["docker-e2e-helper-cli"]], ["scripts/openclaw-postpack.mjs", [TOOLING_VITEST_CONFIG]], ["scripts/openclaw-npm-prepublish-verify.ts", ["test/openclaw-npm-prepublish-verify.test.ts"]], diff --git a/scripts/verify.mjs b/scripts/verify.mjs index e40cdbb5cb2d..4b61524b7099 100644 --- a/scripts/verify.mjs +++ b/scripts/verify.mjs @@ -11,7 +11,7 @@ const stages = [ /** * Renders CLI usage for the verification wrapper. */ -export function usage() { +function usage() { return [ "Usage: node scripts/verify.mjs", "", @@ -55,7 +55,7 @@ async function runStage(stage) { /** * Runs verification stages in order and stops at the first failure. */ -export async function main(argv = process.argv.slice(2)) { +async function main(argv = process.argv.slice(2)) { let args; try { args = parseVerifyArgs(argv); diff --git a/src/plugins/contracts/deprecated-internal-config-api.test.ts b/src/plugins/contracts/deprecated-internal-config-api.test.ts index a2e90a83a277..e1acf519913a 100644 --- a/src/plugins/contracts/deprecated-internal-config-api.test.ts +++ b/src/plugins/contracts/deprecated-internal-config-api.test.ts @@ -1,6 +1,6 @@ // Deprecated internal config API tests cover forbidden legacy plugin config API usage. import { beforeAll, describe, expect, it } from "vitest"; -import { collectDeprecatedInternalConfigApiViolations } from "../../../scripts/lib/deprecated-config-api-guard.mjs"; +import { collectDeprecatedInternalConfigApiViolations } from "../../../scripts/lib/config-boundary-guard.mjs"; describe("deprecated internal config API guardrails", () => { let violations: ReturnType; diff --git a/test/e2e/qa-lab/runtime/openwebui-probe.e2e.test.ts b/test/e2e/qa-lab/runtime/openwebui-probe.e2e.test.ts index 16cb8787b58a..a297f21fe8cd 100644 --- a/test/e2e/qa-lab/runtime/openwebui-probe.e2e.test.ts +++ b/test/e2e/qa-lab/runtime/openwebui-probe.e2e.test.ts @@ -191,7 +191,7 @@ describe("scripts/e2e/openwebui-probe.mjs", () => { expect(script).toContain("run(controller.signal, timeoutPromise)"); expect(script).toMatch( - /readBoundedResponseTextWithLimit\(\s*response,\s*label,\s*responseBodyMaxBytes,\s*timeoutPromise,/u, + /readBoundedResponseTextWithLimit\(\s*response,\s*label,\s*responseBodyMaxBytes,\s*\{\s*createTooLargeError: createBoundedResponseTooLargeError,\s*timeoutPromise,\s*\}\s*\)/u, ); expect(script.match(/async \(signal, timeoutPromise\)/gu)).toHaveLength(3); }); diff --git a/test/scripts/bounded-response-text.test.ts b/test/scripts/bounded-response-text.test.ts deleted file mode 100644 index 30af09237017..000000000000 --- a/test/scripts/bounded-response-text.test.ts +++ /dev/null @@ -1,174 +0,0 @@ -// E2E bounded response tests cover shared HTTP body limits. -import { describe, expect, it } from "vitest"; -import { - readBoundedResponseBytes, - readBoundedResponseText, -} from "../../scripts/e2e/lib/bounded-response-text.mjs"; - -describe("scripts/e2e/lib/bounded-response-text.mjs", () => { - it("preserves binary response bytes", async () => { - const body = Buffer.from([0x00, 0xff, 0x80, 0x7f]); - - await expect( - readBoundedResponseBytes(new Response(body), "fixture", body.length), - ).resolves.toEqual(body); - }); - - it("decodes multibyte text split across chunks", async () => { - const encoded = new TextEncoder().encode("a😀b"); - const response = new Response( - new ReadableStream({ - start(controller) { - controller.enqueue(encoded.subarray(0, 3)); - controller.enqueue(encoded.subarray(3)); - controller.close(); - }, - }), - ); - - await expect(readBoundedResponseText(response, "fixture", encoded.length)).resolves.toBe( - "a😀b", - ); - }); - - it("cancels pending response body reads when the timeout wins", async () => { - let canceled = false; - const response = { - headers: new Headers(), - body: { - getReader() { - return { - read() { - return new Promise>(() => {}); - }, - async cancel() { - canceled = true; - }, - releaseLock() { - throw new Error("releaseLock should not run while a read is pending"); - }, - }; - }, - }, - }; - - await expect( - readBoundedResponseText( - response, - "probe", - 1024, - Promise.reject(new Error("probe timed out")), - ), - ).rejects.toThrow("probe timed out"); - - expect(canceled).toBe(true); - }); - - it("keeps timeout rejection ahead of cancel-unblocked stream reads", async () => { - let canceled = false; - const response = new Response( - new ReadableStream({ - pull() { - return new Promise(() => {}); - }, - cancel() { - canceled = true; - }, - }), - { headers: new Headers() }, - ); - - await expect( - readBoundedResponseText( - response, - "probe", - 1024, - Promise.reject(new Error("probe timed out")), - ), - ).rejects.toThrow("probe timed out"); - - expect(canceled).toBe(true); - }); - - it("cancels oversized streamed response bodies", async () => { - let canceled = false; - const response = new Response( - new ReadableStream({ - start(controller) { - controller.enqueue(new Uint8Array(17)); - }, - cancel() { - canceled = true; - }, - }), - { headers: new Headers() }, - ); - - await expect(readBoundedResponseText(response, "probe", 16)).rejects.toMatchObject({ - code: "ETOOBIG", - message: "probe response body exceeded 16 bytes", - }); - expect(canceled).toBe(true); - }); - - it("streams responses with non-decimal content-length values", async () => { - let readStarted = false; - let canceled = false; - const response = { - headers: new Headers({ "content-length": "1e3" }), - body: { - getReader() { - return { - async read() { - readStarted = true; - return { done: false, value: new Uint8Array(17) }; - }, - async cancel() { - canceled = true; - }, - releaseLock() {}, - }; - }, - }, - }; - - await expect(readBoundedResponseText(response, "probe", 16)).rejects.toMatchObject({ - code: "ETOOBIG", - message: "probe response body exceeded 16 bytes", - }); - expect(readStarted).toBe(true); - expect(canceled).toBe(true); - }); - - it("rejects unsafe decimal content-length values before reading", async () => { - let readStarted = false; - let canceled = false; - const response = { - headers: new Headers({ "content-length": "9007199254740993" }), - body: { - async cancel() { - canceled = true; - }, - getReader() { - return { - async read() { - readStarted = true; - return new Promise>(() => {}); - }, - async cancel() { - canceled = true; - }, - releaseLock() {}, - }; - }, - }, - }; - - await expect(readBoundedResponseText(response, "probe", 16)).rejects.toMatchObject({ - code: "ETOOBIG", - message: "probe response body exceeded 16 bytes", - }); - expect(readStarted).toBe(false); - expect(canceled).toBe(true); - }); -}); diff --git a/test/scripts/bounded-response.test.ts b/test/scripts/bounded-response.test.ts index bd5c92cbc504..0e4f292e3fcd 100644 --- a/test/scripts/bounded-response.test.ts +++ b/test/scripts/bounded-response.test.ts @@ -1,15 +1,38 @@ // Bounded Response tests cover bounded response script behavior. import { describe, expect, it } from "vitest"; -import { readBoundedResponseText as readBoundedResponseTextMjs } from "../../scripts/lib/bounded-response.mjs"; -import { readBoundedResponseText as readBoundedResponseTextTs } from "../../scripts/lib/bounded-response.ts"; - -const helpers = [ - ["ts", readBoundedResponseTextTs], - ["mjs", readBoundedResponseTextMjs], -] as const; +import { + createBoundedResponseTooLargeError, + readBoundedResponseBytes, + readBoundedResponseText, +} from "../../scripts/lib/bounded-response.mjs"; describe("scripts bounded response reader", () => { - it.each(helpers)("cancels response bodies when %s read timeout wins", async (_name, read) => { + it("preserves binary response bytes", async () => { + const body = Buffer.from([0x00, 0xff, 0x80, 0x7f]); + + await expect( + readBoundedResponseBytes(new Response(body), "fixture", body.length), + ).resolves.toEqual(body); + }); + + it("decodes multibyte text split across chunks", async () => { + const encoded = new TextEncoder().encode("a😀b"); + const response = new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoded.subarray(0, 3)); + controller.enqueue(encoded.subarray(3)); + controller.close(); + }, + }), + ); + + await expect(readBoundedResponseText(response, "fixture", encoded.length)).resolves.toBe( + "a😀b", + ); + }); + + it("cancels response bodies when a read timeout wins", async () => { let canceled = false; const response = { headers: new Headers(), @@ -22,115 +45,110 @@ describe("scripts bounded response reader", () => { async cancel() { canceled = true; }, - releaseLock() {}, + releaseLock() { + throw new Error("releaseLock should not run while a read is pending"); + }, }; }, }, } as unknown as Response; await expect( - read(response, "probe", 1024, { - timeoutPromise: new Promise((_resolve, reject) => { - setTimeout(() => reject(new Error("timeout")), 0); - }), + readBoundedResponseText(response, "probe", 1024, { + timeoutPromise: Promise.reject(new Error("timeout")), }), ).rejects.toThrow("timeout"); - expect(canceled).toBe(true); }); - it.each(helpers)( - "rejects when %s timeout cancellation unblocks a real stream read", - async (_name, read) => { - let canceled = false; - const response = { - headers: new Headers(), - body: new ReadableStream({ - pull() { - return new Promise(() => {}); - }, - cancel() { - canceled = true; - }, - }), - } as unknown as Response; - - await expect( - read(response, "probe", 1024, { - timeoutPromise: new Promise((_resolve, reject) => { - setTimeout(() => reject(new Error("timeout")), 0); - }), - }), - ).rejects.toThrow("timeout"); - - expect(canceled).toBe(true); - }, - ); - - it.each(helpers)( - "streams %s responses with non-decimal content-length values", - async (_name, read) => { - let readStarted = false; - let canceled = false; - const response = { - headers: new Headers({ "content-length": "1e3" }), - body: { - getReader() { - return { - async read() { - readStarted = true; - return { done: false, value: new Uint8Array(17) }; - }, - async cancel() { - canceled = true; - }, - releaseLock() {}, - }; - }, + it("keeps timeout rejection ahead of cancel-unblocked stream reads", async () => { + let canceled = false; + const response = new Response( + new ReadableStream({ + pull() { + return new Promise(() => {}); }, - } as unknown as Response; - - await expect(read(response, "probe", 16)).rejects.toThrow( - "probe response body exceeded 16 bytes", - ); - - expect(readStarted).toBe(true); - expect(canceled).toBe(true); - }, - ); - - it.each(helpers)( - "rejects unsafe decimal %s content-length values before reading", - async (_name, read) => { - let readStarted = false; - let canceled = false; - const response = { - headers: new Headers({ "content-length": "9007199254740993" }), - body: { - async cancel() { - canceled = true; - }, - getReader() { - return { - async read() { - readStarted = true; - return new Promise>(() => {}); - }, - async cancel() { - canceled = true; - }, - releaseLock() {}, - }; - }, + cancel() { + canceled = true; }, - } as unknown as Response; + }), + ); - await expect(read(response, "probe", 16)).rejects.toThrow( - "probe response body exceeded 16 bytes", - ); + await expect( + readBoundedResponseText(response, "probe", 1024, { + timeoutPromise: Promise.reject(new Error("timeout")), + }), + ).rejects.toThrow("timeout"); + expect(canceled).toBe(true); + }); - expect(readStarted).toBe(false); - expect(canceled).toBe(true); - }, - ); + it("preserves opt-in ETOOBIG errors for E2E callers", async () => { + await expect( + readBoundedResponseText(new Response(new Uint8Array(17)), "probe", 16, { + createTooLargeError: createBoundedResponseTooLargeError, + }), + ).rejects.toMatchObject({ + code: "ETOOBIG", + message: "probe response body exceeded 16 bytes", + }); + }); + + it("streams responses with non-decimal content-length values", async () => { + let readStarted = false; + let canceled = false; + const response = { + headers: new Headers({ "content-length": "1e3" }), + body: { + getReader() { + return { + async read() { + readStarted = true; + return { done: false, value: new Uint8Array(17) }; + }, + async cancel() { + canceled = true; + }, + releaseLock() {}, + }; + }, + }, + } as unknown as Response; + + await expect(readBoundedResponseText(response, "probe", 16)).rejects.toMatchObject({ + message: "probe response body exceeded 16 bytes", + }); + expect(readStarted).toBe(true); + expect(canceled).toBe(true); + }); + + it("rejects unsafe decimal content-length values before reading", async () => { + let readStarted = false; + let canceled = false; + const response = { + headers: new Headers({ "content-length": "9007199254740993" }), + body: { + async cancel() { + canceled = true; + }, + getReader() { + return { + async read() { + readStarted = true; + return new Promise>(() => {}); + }, + async cancel() { + canceled = true; + }, + releaseLock() {}, + }; + }, + }, + } as unknown as Response; + + await expect(readBoundedResponseText(response, "probe", 16)).rejects.toThrow( + "probe response body exceeded 16 bytes", + ); + expect(readStarted).toBe(false); + expect(canceled).toBe(true); + }); }); diff --git a/test/scripts/changed-lanes.test.ts b/test/scripts/changed-lanes.test.ts index a05d5ce13d85..0c61a3efd0f4 100644 --- a/test/scripts/changed-lanes.test.ts +++ b/test/scripts/changed-lanes.test.ts @@ -348,7 +348,10 @@ describe("scripts/changed-lanes", () => { name: "rejects unknown changed check options before treating them as paths", script: "scripts/check-changed.mjs", option: "--dr-run", - expected: { stderr: "Unknown option: --dr-run", excludes: ["[check:changed]"] }, + expected: { + stderr: "Unknown option: --dr-run\n[check:changed] FAILED (exit 1)", + excludes: [], + }, }, ])("$name", ({ script, option, expected }) => { const result = runRepoScript(script, [option], { diff --git a/test/scripts/npm-telegram-live.test.ts b/test/scripts/npm-telegram-live.test.ts index a2b0c0c14898..12eac62d37e9 100644 --- a/test/scripts/npm-telegram-live.test.ts +++ b/test/scripts/npm-telegram-live.test.ts @@ -163,7 +163,7 @@ describe("package Telegram live Docker E2E", () => { expect(script).toContain('package_install_source="openclaw@$(read_package_version'); expect(script).toContain('-v "$resolved_package_dir:/package-under-test:ro"'); expect(script).toContain( - '-v "$ROOT_DIR/scripts/e2e/lib/bounded-response-text.mjs:/tmp/openclaw-e2e/lib/bounded-response-text.mjs:ro"', + '-v "$ROOT_DIR/scripts/lib/bounded-response.mjs:/tmp/lib/bounded-response.mjs:ro"', ); expect(script).toContain( '-v "$ROOT_DIR/scripts/e2e/lib/plugins/npm-registry-server.mjs:/tmp/openclaw-e2e/lib/plugins/npm-registry-server.mjs:ro"', diff --git a/test/scripts/plugins-assertions.test.ts b/test/scripts/plugins-assertions.test.ts index c3bcf5d6cf6c..eeee3834479a 100644 --- a/test/scripts/plugins-assertions.test.ts +++ b/test/scripts/plugins-assertions.test.ts @@ -227,7 +227,9 @@ describe("plugins Docker assertions", () => { expect(script).toContain("run(controller.signal, timeoutPromise)"); expect( - script.match(/readBoundedResponseText\([\s\S]*?limits\.bodyMaxBytes,\n\s+timeoutPromise,/gu), + script.match( + /readBoundedResponseText\([\s\S]*?limits\.bodyMaxBytes,\n\s+\{ createTooLargeError: createBoundedResponseTooLargeError, timeoutPromise \},/gu, + ), ).toHaveLength(2); }); diff --git a/test/scripts/release-beta-verifier.test.ts b/test/scripts/release-beta-verifier.test.ts index 968c33dd4ce3..4619c728e0e4 100644 --- a/test/scripts/release-beta-verifier.test.ts +++ b/test/scripts/release-beta-verifier.test.ts @@ -735,13 +735,13 @@ describe("readBoundedJsonResponse", () => { "ClawHub package", 64, ), - ).rejects.toThrow("ClawHub package response body exceeded 64 bytes."); + ).rejects.toThrow("ClawHub package response body exceeded 64 bytes"); }); it("rejects oversized streamed JSON bodies", async () => { await expect( readBoundedJsonResponse(new Response('{"padding":"too-large"}'), "ClawHub package", 8), - ).rejects.toThrow("ClawHub package response body exceeded 8 bytes."); + ).rejects.toThrow("ClawHub package response body exceeded 8 bytes"); }); it("keeps ClawHub request timeouts active while reading JSON bodies", async () => { diff --git a/test/scripts/run-oxlint.test.ts b/test/scripts/run-oxlint.test.ts index 95fbf96739b9..c41f6ea488c4 100644 --- a/test/scripts/run-oxlint.test.ts +++ b/test/scripts/run-oxlint.test.ts @@ -5,6 +5,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { pathToFileURL } from "node:url"; import { describe, expect, it } from "vitest"; +import { runWithFailedTrailer } from "../../scripts/lib/failed-trailer.mjs"; import { createOxlintShards, filterOxlintShards, @@ -20,7 +21,6 @@ import { } from "../../scripts/run-oxlint-shards.mjs"; import { filterSparseMissingOxlintTargets, - runOxlintCliEntry, shouldPrepareExtensionPackageBoundaryArtifacts, } from "../../scripts/run-oxlint.mjs"; import { createScriptTestHarness } from "./test-helpers.js"; @@ -55,7 +55,8 @@ describe("run-oxlint", () => { const lines: unknown[] = []; try { process.exitCode = 0; - await runOxlintCliEntry( + await runWithFailedTrailer( + "oxlint", async () => { process.exitCode = 2; }, @@ -74,7 +75,8 @@ describe("run-oxlint", () => { const lines: unknown[] = []; try { process.exitCode = 0; - await runOxlintCliEntry( + await runWithFailedTrailer( + "oxlint", async () => { throw new Error("artifact prep failed"); }, @@ -94,7 +96,8 @@ describe("run-oxlint", () => { const lines: unknown[] = []; try { process.exitCode = 0; - await runOxlintCliEntry( + await runWithFailedTrailer( + "oxlint", async () => {}, (line: unknown) => lines.push(line), ); diff --git a/test/scripts/run-tsgo.test.ts b/test/scripts/run-tsgo.test.ts index fe3e48bb28cc..d64f3bf431d0 100644 --- a/test/scripts/run-tsgo.test.ts +++ b/test/scripts/run-tsgo.test.ts @@ -1,4 +1,5 @@ // Run Tsgo tests cover run tsgo script behavior. +import { spawnSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; import { describe, expect, it } from "vitest"; @@ -12,6 +13,25 @@ import { createScriptTestHarness } from "./test-helpers.js"; const { createTempDir } = createScriptTestHarness(); describe("run-tsgo sparse guard", () => { + it("ends sparse-checkout failures with the stable failure trailer", () => { + const cwd = createTempDir("openclaw-run-tsgo-"); + spawnSync("git", ["init", "-q"], { cwd }); + spawnSync("git", ["config", "core.sparseCheckout", "true"], { cwd }); + + const result = spawnSync( + process.execPath, + [path.resolve("scripts/run-tsgo.mjs"), "-p", "test/tsconfig/tsconfig.core.test.json"], + { + cwd, + encoding: "utf8", + env: { ...process.env, OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" }, + }, + ); + + expect(result.status).toBe(1); + expect(result.stderr.trim().split("\n").at(-1)).toBe("[tsgo] FAILED (exit 1)"); + }); + it("ignores non-core projects", () => { const cwd = createTempDir("openclaw-run-tsgo-"); diff --git a/test/scripts/run-vitest.test.ts b/test/scripts/run-vitest.test.ts index c8563227e866..9efb545e3856 100644 --- a/test/scripts/run-vitest.test.ts +++ b/test/scripts/run-vitest.test.ts @@ -1,5 +1,5 @@ // Run Vitest tests cover run vitest script behavior. -import { spawn } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import { EventEmitter } from "node:events"; import fs from "node:fs"; import os from "node:os"; @@ -38,6 +38,15 @@ const posixIt = process.platform === "win32" ? it.skip : it; const LOAD_SENSITIVE_PROCESS_TIMEOUT_MS = process.env.CI ? 30_000 : 15_000; describe("scripts/run-vitest", () => { + it("ends argument failures with the stable failure trailer", () => { + const result = spawnSync(process.execPath, [nodePath.resolve("scripts/run-vitest.mjs")], { + encoding: "utf8", + }); + + expect(result.status).toBe(1); + expect(result.stderr.trim().split("\n").at(-1)).toBe("[vitest] FAILED (exit 1)"); + }); + it.each([...VITEST_CONFIG_NO_OUTPUT_TIMEOUT_MS.keys(), ...TOOLING_EXCLUDED_TESTS])( "keeps hardcoded Vitest path %s valid", (referencedPath) => { diff --git a/test/scripts/test-projects.test.ts b/test/scripts/test-projects.test.ts index 2e4cb1e190f0..853070035f8c 100644 --- a/test/scripts/test-projects.test.ts +++ b/test/scripts/test-projects.test.ts @@ -1719,10 +1719,6 @@ describe("scripts/test-projects changed-target routing", () => { ], ["scripts/lib/config-boundary-guard.d.mts", "scripts/lib/config-boundary-guard.mjs"], ["scripts/lib/arg-utils.d.mts", "scripts/lib/arg-utils.mjs"], - [ - "scripts/lib/deprecated-config-api-guard.d.mts", - "scripts/lib/deprecated-config-api-guard.mjs", - ], [ "scripts/lib/extension-source-classifier.d.mts", "scripts/lib/extension-source-classifier.mjs", @@ -1848,6 +1844,15 @@ describe("scripts/test-projects changed-target routing", () => { ["test/scripts/local-heavy-check-runtime.test.ts"], ], ["scripts/lib/managed-child-process.mjs", ["test/scripts/managed-child-process.test.ts"]], + [ + "scripts/lib/failed-trailer.mjs", + [ + "test/scripts/run-oxlint.test.ts", + "test/scripts/run-tsgo.test.ts", + "test/scripts/run-vitest.test.ts", + "test/scripts/changed-lanes.test.ts", + ], + ], [ "scripts/lib/windows-taskkill.mjs", ["test/scripts/managed-child-process.test.ts", "test/scripts/run-with-env.test.ts"], @@ -2388,10 +2393,6 @@ describe("scripts/test-projects changed-target routing", () => { "src/plugins/contracts/deprecated-internal-config-api.test.ts", ], ], - [ - "scripts/lib/deprecated-config-api-guard.mjs", - ["src/plugins/contracts/deprecated-internal-config-api.test.ts"], - ], [ "scripts/lib/extension-package-boundary.ts", ["src/plugins/contracts/extension-package-project-boundaries.test.ts"],