diff --git a/scripts/root-dependency-ownership-audit.mjs b/scripts/root-dependency-ownership-audit.mjs index 0e6da3b40d50..5087f08a107a 100644 --- a/scripts/root-dependency-ownership-audit.mjs +++ b/scripts/root-dependency-ownership-audit.mjs @@ -21,9 +21,6 @@ const DYNAMIC_CONSTANT_IMPORT_PATTERNS = [ /\brequire\s*\(\s*([_$A-Za-z][\w$]*)\s*\)/g, /\b(?:require|[_$A-Za-z][\w$]*require[\w$]*)\.resolve\s*\(\s*([_$A-Za-z][\w$]*)\s*\)/gi, ]; -const PACKAGE_FILE_LOOKUP_PATTERNS = [ - /\bresolvePackageFileForCommandExplanation\s*\(\s*["']([^"']+)["']/g, -]; const ROOT_OWNED_EXTENSION_RUNTIME_DEPENDENCIES = new Map([ [ "@homebridge/ciao", @@ -93,13 +90,6 @@ export function collectModuleSpecifiers(source) { } } } - for (const pattern of PACKAGE_FILE_LOOKUP_PATTERNS) { - for (const match of source.matchAll(pattern)) { - if (match[1]) { - specifiers.add(match[1]); - } - } - } const stringConstants = new Map(); for (const match of source.matchAll(STRING_CONSTANT_PATTERN)) { if (match[1] && match[2]) { diff --git a/src/infra/command-explainer/extract.test.ts b/src/infra/command-explainer/extract.test.ts index aa26af0948dd..2e4cdd13cb35 100644 --- a/src/infra/command-explainer/extract.test.ts +++ b/src/infra/command-explainer/extract.test.ts @@ -47,18 +47,18 @@ describe("command explainer tree-sitter runtime", () => { ); }); - it("maps parser byte offsets to JavaScript string spans for Unicode source", async () => { - const source = "echo café && echo ok"; + it("uses native JavaScript string offsets for Unicode source", async () => { + const source = "echo café😀 && echo 雪"; const explanation = await explainShellCommand(source); expect(explanation.topLevelCommands).toHaveLength(2); expect(explanation.topLevelCommands.map((command) => command.argv)).toEqual([ - ["echo", "café"], - ["echo", "ok"], + ["echo", "café😀"], + ["echo", "雪"], ]); expect(explanation.topLevelCommands.map((command) => command.span)).toMatchObject([ - { startIndex: 0, endIndex: 9 }, - { startIndex: 13, endIndex: 20 }, + { startIndex: 0, endIndex: 11 }, + { startIndex: 15, endIndex: 21 }, ]); for (const command of explanation.topLevelCommands) { expect(source.slice(command.span.startIndex, command.span.endIndex)).toBe(command.text); diff --git a/src/infra/command-explainer/extract.ts b/src/infra/command-explainer/extract.ts index 73bb8543716d..27ac4fd7d62c 100644 --- a/src/infra/command-explainer/extract.ts +++ b/src/infra/command-explainer/extract.ts @@ -72,15 +72,10 @@ const PARSEABLE_SHELL_WRAPPERS = new Set(POSIX_SHELL_WRAPPERS); // Span bases map nested wrapper payload offsets back to source command offsets. type SpanBase = { - startIndex: number; - startPosition: SourceSpan["startPosition"]; mapOffset?: (offset: number) => { index: number; position: SourceSpan["startPosition"] }; }; -const ROOT_SPAN_BASE: SpanBase = { - startIndex: 0, - startPosition: { row: 0, column: 0 }, -}; +const ROOT_SPAN_BASE: SpanBase = {}; type CommandTopologyBucket = { context: CommandContext; @@ -95,60 +90,27 @@ type OperatorSource = { spanBase: SpanBase; }; -// Tree-sitter exposes nullable children; normalize once for the walkers below. -function children(node: TreeSitterNode): TreeSitterNode[] { - return Array.from({ length: node.childCount }, (_, index) => node.child(index)).filter( - (child): child is TreeSitterNode => child !== null, - ); -} - -function namedChildren(node: TreeSitterNode): TreeSitterNode[] { - return Array.from({ length: node.namedChildCount }, (_, index) => node.namedChild(index)).filter( - (child): child is TreeSitterNode => child !== null, - ); -} - function hasDirectChildType(node: TreeSitterNode, type: string): boolean { - return children(node).some((child) => child.type === type); -} - -function translatePosition( - position: SourceSpan["startPosition"], - base: SourceSpan["startPosition"], -): SourceSpan["startPosition"] { - return { - row: base.row + position.row, - column: position.row === 0 ? base.column + position.column : position.column, - }; + return node.children.some((child) => child.type === type); } function translateSpan(span: SourceSpan, base: SpanBase): SourceSpan { - if (base.mapOffset) { - const start = base.mapOffset(span.startIndex); - const end = base.mapOffset(span.endIndex); - return { - startIndex: start.index, - endIndex: end.index, - startPosition: start.position, - endPosition: end.position, - }; + if (!base.mapOffset) { + return span; } + const start = base.mapOffset(span.startIndex); + const end = base.mapOffset(span.endIndex); return { - startIndex: base.startIndex + span.startIndex, - endIndex: base.startIndex + span.endIndex, - startPosition: translatePosition(span.startPosition, base.startPosition), - endPosition: translatePosition(span.endPosition, base.startPosition), + startIndex: start.index, + endIndex: end.index, + startPosition: start.position, + endPosition: end.position, }; } function spanFromNode(node: TreeSitterNode, base: SpanBase = ROOT_SPAN_BASE): SourceSpan { - const span = { - startIndex: node.startIndex, - endIndex: node.endIndex, - startPosition: { row: node.startPosition.row, column: node.startPosition.column }, - endPosition: { row: node.endPosition.row, column: node.endPosition.column }, - }; - return translateSpan(span, base); + const { startIndex, endIndex, startPosition, endPosition } = node; + return translateSpan({ startIndex, endIndex, startPosition, endPosition }, base); } function advancePosition( @@ -178,7 +140,7 @@ function advancePosition( } function positionAtSourceIndex(source: string, index: number): SourceSpan["startPosition"] { - return advancePosition(ROOT_SPAN_BASE.startPosition, source.slice(0, index)); + return advancePosition({ row: 0, column: 0 }, source.slice(0, index)); } function spanFromSourceRange(source: string, startIndex: number, endIndex: number): SourceSpan { @@ -190,111 +152,6 @@ function spanFromSourceRange(source: string, startIndex: number, endIndex: numbe }; } -function spanFromSourceRangeWithBase( - source: string, - startIndex: number, - endIndex: number, - base: SpanBase, -): SourceSpan { - if (base.mapOffset) { - const start = base.mapOffset(startIndex); - const end = base.mapOffset(endIndex); - return { - startIndex: start.index, - endIndex: end.index, - startPosition: start.position, - endPosition: end.position, - }; - } - return translateSpan(spanFromSourceRange(source, startIndex, endIndex), base); -} - -function utf8ByteLengthForCodePoint(codePoint: number): number { - if (codePoint <= 0x7f) { - return 1; - } - if (codePoint <= 0x7ff) { - return 2; - } - if (codePoint <= 0xffff) { - return 3; - } - return 4; -} - -function utf8ByteLength(text: string): number { - let length = 0; - for (let index = 0; index < text.length; index += 1) { - const codePoint = text.codePointAt(index); - if (codePoint === undefined) { - continue; - } - length += utf8ByteLengthForCodePoint(codePoint); - if (codePoint > 0xffff) { - index += 1; - } - } - return length; -} - -function utf8ByteOffsetToStringIndex(text: string, byteOffset: number): number { - if (byteOffset <= 0) { - return 0; - } - let currentByteOffset = 0; - for (let index = 0; index < text.length; index += 1) { - const codePoint = text.codePointAt(index); - if (codePoint === undefined) { - return text.length; - } - const codePointLength = utf8ByteLengthForCodePoint(codePoint); - if (currentByteOffset + codePointLength > byteOffset) { - return index; - } - currentByteOffset += codePointLength; - if (currentByteOffset === byteOffset) { - return codePoint > 0xffff ? index + 2 : index + 1; - } - if (codePoint > 0xffff) { - index += 1; - } - } - return text.length; -} - -function parserOffsetToStringIndex( - source: string, - rootNode: TreeSitterNode, -): (offset: number) => number { - const utf8Length = utf8ByteLength(source); - if (utf8Length !== source.length && rootNode.endIndex === utf8Length) { - return (offset) => utf8ByteOffsetToStringIndex(source, offset); - } - return (offset) => offset; -} - -function spanBaseForParserSource( - source: string, - rootNode: TreeSitterNode, - base: SpanBase, -): SpanBase { - const offsetToStringIndex = parserOffsetToStringIndex(source, rootNode); - return { - startIndex: base.startIndex, - startPosition: base.startPosition, - mapOffset(offset) { - const sourceIndex = offsetToStringIndex(offset); - if (base.mapOffset) { - return base.mapOffset(sourceIndex); - } - return { - index: base.startIndex + sourceIndex, - position: advancePosition(base.startPosition, source.slice(0, sourceIndex)), - }; - }, - }; -} - function valuePrefixLength(node: TreeSitterNode): number { if (node.type === "string" || node.type === "raw_string") { return 1; @@ -580,10 +437,7 @@ function decodeAnsiCString(text: string): string { } function hasDynamicWordPart(node: TreeSitterNode): boolean { - return ( - DYNAMIC_WORD_NODE_TYPES.has(node.type) || - namedChildren(node).some((child) => hasDynamicWordPart(child)) - ); + return DYNAMIC_WORD_NODE_TYPES.has(node.type) || node.namedChildren.some(hasDynamicWordPart); } function shellWordValue(node: TreeSitterNode): ShellWordValue { @@ -593,7 +447,7 @@ function shellWordValue(node: TreeSitterNode): ShellWordValue { if ( node.type !== "command_name" && node.type !== "concatenation" && - namedChildren(node).some((child) => hasDynamicWordPart(child)) + node.namedChildren.some((child) => hasDynamicWordPart(child)) ) { return { kind: "dynamic", @@ -603,7 +457,7 @@ function shellWordValue(node: TreeSitterNode): ShellWordValue { switch (node.type) { case "command_name": { - const parts = namedChildren(node); + const parts = node.namedChildren; if (parts.length === 0) { return hasUnescapedDynamicPattern(node.text) ? { kind: "dynamic", value: decodeUnquotedShellText(node.text) } @@ -635,7 +489,7 @@ function shellWordValue(node: TreeSitterNode): ShellWordValue { } let value = ""; let dynamic = false; - for (const child of namedChildren(node)) { + for (const child of node.namedChildren) { const childValue = shellWordValue(child); value += childValue.value; if (childValue.kind !== "literal") { @@ -645,20 +499,12 @@ function shellWordValue(node: TreeSitterNode): ShellWordValue { return dynamic ? { kind: "dynamic", value } : { kind: "literal", value }; } default: - return namedChildren(node).some((child) => shellWordValue(child).kind === "dynamic") + return node.namedChildren.some((child) => shellWordValue(child).kind === "dynamic") ? { kind: "dynamic", value: decodeUnquotedShellText(node.text) } : { kind: "literal", value: decodeUnquotedShellText(node.text) }; } } -function commandNameNode(node: TreeSitterNode): TreeSitterNode | null { - return ( - node.childForFieldName("name") ?? - namedChildren(node).find((child) => child.type === "command_name") ?? - null - ); -} - function argvFromCommand( node: TreeSitterNode, nameNode: TreeSitterNode, @@ -672,17 +518,11 @@ function argvFromCommand( return null; } - const skipped = new Set([nameNode, ...namedChildren(nameNode)]); const argv = [executable.value]; const argumentsList: CommandArgument[] = []; const dynamicArguments: DynamicArgument[] = []; - for (const child of namedChildren(node)) { - if ( - skipped.has(child) || - child.type === "command_name" || - child.type === "variable_assignment" || - !COMMAND_ARGUMENT_NODE_TYPES.has(child.type) - ) { + for (const child of node.childrenForFieldName("argument")) { + if (!COMMAND_ARGUMENT_NODE_TYPES.has(child.type)) { continue; } const value = shellWordValue(child); @@ -713,7 +553,7 @@ function argvFromDeclarationCommand(node: TreeSitterNode, state: WalkState): Com const argv = [executable]; const argumentsList: CommandArgument[] = []; const dynamicArguments: DynamicArgument[] = []; - for (const child of namedChildren(node)) { + for (const child of node.namedChildren) { if (!COMMAND_ARGUMENT_NODE_TYPES.has(child.type) && child.type !== "variable_assignment") { continue; } @@ -755,7 +595,7 @@ function appendTestCommandArguments( argv.push(value.value); return; } - for (const child of namedChildren(node)) { + for (const child of node.namedChildren) { appendTestCommandArguments(child, argv, argumentsList, dynamicArguments, state); } } @@ -769,7 +609,7 @@ function argvFromTestCommand(node: TreeSitterNode, state: WalkState): CommandArg const argv = [executable]; const argumentsList: CommandArgument[] = []; const dynamicArguments: DynamicArgument[] = []; - for (const child of namedChildren(node)) { + for (const child of node.namedChildren) { appendTestCommandArguments(child, argv, argumentsList, dynamicArguments, state); } return { argv, arguments: argumentsList, dynamicArguments }; @@ -784,7 +624,7 @@ function isCommandLikeNode(node: TreeSitterNode): boolean { function recordShape(node: TreeSitterNode, output: MutableExplanation): void { if ( (node.type === "program" || node.type === "list") && - (hasDirectChildType(node, ";") || namedChildren(node).filter(isCommandLikeNode).length > 1) + (hasDirectChildType(node, ";") || node.namedChildren.filter(isCommandLikeNode).length > 1) ) { output.shapes.add("sequence"); } @@ -889,10 +729,7 @@ function payloadBaseFromArgument(argument: CommandArgument, payload: string): Sp if (rawPayloadOffset === undefined) { return null; } - const prefix = argument.text.slice(0, rawPayloadOffset); return { - startIndex: argument.span.startIndex + rawPayloadOffset, - startPosition: advancePosition(argument.span.startPosition, prefix), mapOffset(offset) { const rawOffset = argument.decodedSourceOffsets[payloadOffset + offset]; const mappedRawOffset = rawOffset ?? rawPayloadOffset + offset; @@ -1107,7 +944,7 @@ async function walk( node.type === "declaration_command" || node.type === "test_command" ) { - const nameNode = node.type === "command" ? commandNameNode(node) : null; + const nameNode = node.type === "command" ? node.childForFieldName("name") : null; const parsed = node.type === "command" ? nameNode @@ -1150,11 +987,7 @@ async function walk( ); if (wrapperPayload && state.wrapperPayloadDepth < MAX_WRAPPER_PAYLOAD_DEPTH) { const wrapperTree = await parseBashForCommandExplanation(wrapperPayload.command); - const wrapperSpanBase = spanBaseForParserSource( - wrapperPayload.command, - wrapperTree.rootNode, - wrapperPayload.spanBase, - ); + const wrapperSpanBase = wrapperPayload.spanBase; try { output.operatorSources.push({ context: "wrapper-payload", @@ -1182,7 +1015,7 @@ async function walk( } } } - for (const child of namedChildren(node)) { + for (const child of node.namedChildren) { await walk(child, output, childContext, state); } } @@ -1332,10 +1165,8 @@ function resolveOperators( } const startIndex = separatorStart + operator.offset; const span = separatorBase - ? spanFromSourceRangeWithBase( - separatorSource, - startIndex, - startIndex + operator.text.length, + ? translateSpan( + spanFromSourceRange(separatorSource, startIndex, startIndex + operator.text.length), separatorBase, ) : spanFromSourceRange(source, startIndex, startIndex + operator.text.length); @@ -1361,7 +1192,6 @@ function resolveOperators( export async function explainShellCommand(source: string): Promise { const tree = await parseBashForCommandExplanation(source); try { - const spanBase = spanBaseForParserSource(source, tree.rootNode, ROOT_SPAN_BASE); const output: MutableExplanation = { shapes: new Set(), commands: [], @@ -1372,7 +1202,7 @@ export async function explainShellCommand(source: string): Promise command.context === "top-level"); const operators = resolveOperators(source, output.commands, output.operatorSources); diff --git a/src/infra/command-explainer/tree-sitter-runtime.ts b/src/infra/command-explainer/tree-sitter-runtime.ts index 3ca3a0e58f4f..a3d1aa18a454 100644 --- a/src/infra/command-explainer/tree-sitter-runtime.ts +++ b/src/infra/command-explainer/tree-sitter-runtime.ts @@ -1,69 +1,26 @@ -// Lazy tree-sitter runtime resolves WASM assets, caches the bash parser, and -// enforces source-size/time limits for command explanation. -import fs from "node:fs"; +// Lazy tree-sitter runtime caches the bash parser and enforces source-size/time +// limits for command explanation. import { createRequire } from "node:module"; -import path from "node:path"; import * as TreeSitter from "web-tree-sitter"; const require = createRequire(import.meta.url); let parserPromise: Promise | null = null; -const parserLoader: () => Promise = loadParser; const MAX_COMMAND_EXPLANATION_SOURCE_CHARS = 128 * 1024; const MAX_COMMAND_EXPLANATION_PARSE_MS = 500; -function resolvePackageFileForCommandExplanation(packageName: string, fileName: string): string { - let packageEntry: string; - try { - packageEntry = require.resolve(packageName); - } catch (error) { - throw new Error( - `Unable to resolve ${packageName} while loading the shell command explainer parser`, - { cause: error }, - ); - } - - let directory = path.dirname(packageEntry); - const searched: string[] = []; - for (let depth = 0; depth < 5; depth += 1) { - const candidate = path.join(directory, fileName); - searched.push(candidate); - if (fs.existsSync(candidate)) { - return candidate; - } - const parent = path.dirname(directory); - if (parent === directory) { - break; - } - directory = parent; - } - throw new Error( - `Unable to locate ${fileName} in ${packageName} while loading the shell command explainer parser; searched ${searched.join(", ")}`, - ); -} - -function resolveWebTreeSitterFile(fileName: string): string { - return resolvePackageFileForCommandExplanation("web-tree-sitter", fileName); -} - -function resolveBashWasmPath(): string { - return resolvePackageFileForCommandExplanation("tree-sitter-bash", "tree-sitter-bash.wasm"); -} - async function loadParser(): Promise { - await TreeSitter.Parser.init({ - locateFile: resolveWebTreeSitterFile, - }); - const language = await TreeSitter.Language.load(resolveBashWasmPath()); - const parser = new TreeSitter.Parser(); - parser.setLanguage(language); - return parser; + await TreeSitter.Parser.init(); + const language = await TreeSitter.Language.load( + require.resolve("tree-sitter-bash/tree-sitter-bash.wasm"), + ); + return new TreeSitter.Parser().setLanguage(language); } function getBashParserForCommandExplanation(): Promise { // Reset the cache on load failure so transient filesystem or WASM init errors // do not poison all later command explanations in the process. - parserPromise ??= parserLoader().catch((error: unknown) => { + parserPromise ??= loadParser().catch((error: unknown) => { parserPromise = null; throw error; }); diff --git a/test/scripts/root-dependency-ownership-audit.test.ts b/test/scripts/root-dependency-ownership-audit.test.ts index 9bfbc5bbd2b7..80a4f40c391d 100644 --- a/test/scripts/root-dependency-ownership-audit.test.ts +++ b/test/scripts/root-dependency-ownership-audit.test.ts @@ -38,9 +38,8 @@ describe("collectModuleSpecifiers", () => { const runtimeRequire = createRequire(runtimePackagePath); require.resolve("gaxios"); runtimeRequire.resolve("openshell/package.json"); - resolvePackageFileForCommandExplanation("tree-sitter-bash", "tree-sitter-bash.wasm"); `), - ]).toEqual(["gaxios", "openshell/package.json", "tree-sitter-bash"]); + ]).toEqual(["gaxios", "openshell/package.json"]); }); it("resolves simple string constants used by lazy runtime imports", () => {