diff --git a/scripts/android-app-i18n.ts b/scripts/android-app-i18n.ts index da64d7c1cdcd..f1dc5059b5a9 100644 --- a/scripts/android-app-i18n.ts +++ b/scripts/android-app-i18n.ts @@ -45,6 +45,7 @@ const ARRAY_RE = /]*>([\s\S]*?)<\/strin const ARRAY_ITEM_RE = /([\s\S]*?)<\/item>/gu; const ANDROID_REFERENCE_COMMENT_RE = /("""[\s\S]*?"""|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*')||\/\*[\s\S]*?\*\/|\/\/[^\r\n]*/gu; +const XML_COMMENT_RE = //gu; const FORMAT_RE = /%\d+\$[a-z]/giu; const INVALID_APOSTROPHE_RE = /(?:'|(?"; @@ -277,39 +278,51 @@ async function readAndroidSource( return sources; } -async function readAndroidResourceReferences(root = ANDROID_MAIN_ROOT): Promise { +type AndroidResourceReferenceSource = { + path: string; + source: string; +}; + +async function readAndroidResourceReferences( + root = ANDROID_MAIN_ROOT, +): Promise { const entries = await readdir(root, { withFileTypes: true }); - const sources: string[] = []; + const sources: AndroidResourceReferenceSource[] = []; for (const entry of entries) { const fullPath = path.join(root, entry.name); if (entry.isDirectory()) { // This walk is confined to app/src/main, so Gradle build output is never eligible. - sources.push(await readAndroidResourceReferences(fullPath)); + sources.push(...(await readAndroidResourceReferences(fullPath))); continue; } if (entry.isFile() && /\.(?:kt|kts|xml)$/u.test(entry.name)) { - sources.push(await readFile(fullPath, "utf8")); + sources.push({ + path: path.relative(ROOT, fullPath).split(path.sep).join("/"), + source: await readFile(fullPath, "utf8"), + }); } } - return sources.join("\n"); + return sources; } export function findUnusedAndroidResourceKeys( keys: Iterable, - referenceSource: string, + referenceSources: readonly AndroidResourceReferenceSource[], ): string[] { - const sourceWithoutComments = referenceSource.replace( - ANDROID_REFERENCE_COMMENT_RE, - (_match, quoted: string | undefined) => quoted ?? "", - ); - const references = new Set([ - ...[...sourceWithoutComments.matchAll(/\bR\.string\.([A-Za-z0-9_]+)\b/gu)].flatMap((match) => - match[1] ? [match[1]] : [], - ), - ...[...sourceWithoutComments.matchAll(/@string\/([A-Za-z0-9_]+)\b/gu)].flatMap((match) => - match[1] ? [match[1]] : [], - ), - ]); + const references = new Set(); + for (const reference of referenceSources) { + const isXml = reference.path.endsWith(".xml"); + const source = reference.source.replace( + isXml ? XML_COMMENT_RE : ANDROID_REFERENCE_COMMENT_RE, + (match) => match.replace(/[^\r\n]/gu, " "), + ); + const pattern = isXml ? /@string\/([A-Za-z0-9_]+)\b/gu : /\bR\.string\.([A-Za-z0-9_]+)\b/gu; + for (const match of source.matchAll(pattern)) { + if (match[1]) { + references.add(match[1]); + } + } + } return [...keys].filter((key) => !references.has(key)); } @@ -378,13 +391,26 @@ const DIRECT_UI_LITERAL_PATTERNS = [ /\bset(?:Title|Message|PositiveButton|NegativeButton|NeutralButton)\s*\(\s*"((?:\\.|[^"\\])+)"/gu, /\bnativeString(?:Resource)?\([^)]*\)\s*\+\s*"((?:\\.|[^"\\])+)"/gsu, ] as const; +const DIRECT_UI_RAW_LITERAL_PATTERNS = [ + /\bText\s*\(\s*(?:text\s*=\s*)?"""([\s\S]*?)"""/gu, + /\b(?:ClawPrimaryButton|ClawSecondaryButton|ClawDangerButton|ClawLinkButton)\s*\([^)]*?\btext\s*=\s*"""([\s\S]*?)"""/gsu, + /\b(?:title|subtitle|body|text|label|statusText|confirmLabel|dismissLabel|contentDescription|placeholder|onClickLabel)\s*=\s*"""([\s\S]*?)"""/gu, + /\b(?:title|subtitle|body|text|label|statusText|confirmLabel|dismissLabel|contentDescription|placeholder|onClickLabel)\s*=\s*[^,\n]*\?:\s*"""([\s\S]*?)"""/gu, + /\b(?:_[A-Za-z0-9_]*(?:ErrorText|StatusText)|errorText|statusText)(?:\.value)?\s*=\s*"""([\s\S]*?)"""/gu, + /\bSettingsMetric\s*\(\s*"""([\s\S]*?)"""/gu, + /\bToast\.makeText\s*\([^,]+,\s*"""([\s\S]*?)"""/gu, + /\bset(?:Title|Message|PositiveButton|NegativeButton|NeutralButton)\s*\(\s*"""([\s\S]*?)"""/gu, + /\bnativeString(?:Resource)?\([^)]*\)\s*\+\s*"""([\s\S]*?)"""/gsu, +] as const; const UI_STRING_HELPER_NAME_RE = - /(?:description|detail|error|label|message|nextRun|notice|status|subtitle|summary|text|title)$/iu; + /(?:description|detail|error|label|message|nextRun|notice|report|status|subtitle|summary|text|title)$/iu; const UI_STRING_FUNCTION_RE = /\bfun\s+(?:<[^>{}]*>\s*)?([A-Za-z_][A-Za-z0-9_]*)\s*/gu; const UI_STRING_PROPERTY_RE = /\bval\s+(?:[A-Za-z_][A-Za-z0-9_]*\.)?([A-Za-z_][A-Za-z0-9_]*)\s*:\s*String\s*(?:\n\s*)?get\(\)\s*=\s*/gu; const UI_BRANCH_LITERAL_RE = /(?:->|return|\?:)\s*"((?:\\.|[^"\\])+)"/gu; const UI_ELSE_BRANCH_LITERAL_RE = /\belse\s*\{?\s*"((?:\\.|[^"\\])+)"/gu; +const UI_BRANCH_RAW_LITERAL_RE = /(?:->|return|\?:)\s*"""([\s\S]*?)"""/gu; +const UI_ELSE_BRANCH_RAW_LITERAL_RE = /\belse\s*\{?\s*"""([\s\S]*?)"""/gu; const UI_MODEL_STRING_NAME_RE = /^(?:contentDescription|errorText|helperText|onClickLabel|statusText)$/u; const UI_MODEL_CLASS_FIELDS = new Map>([ @@ -393,7 +419,7 @@ const UI_MODEL_CLASS_FIELDS = new Map>([ ["OverviewMetricCardSpec", new Set(["subtitle", "title", "value"])], ["SettingsToggleRow", new Set(["subtitle", "title"])], ]); -const KOTLIN_STRING_LITERAL_RE = /"((?:\\.|[^"\\])+)"/gu; +const KOTLIN_STRING_LITERAL_RE = /"""([\s\S]*?)"""|"((?:\\.|[^"\\])+?)"/gu; // These literals are either technical data or immutable source tokens localized at a typed render edge. const ALLOWED_UI_LITERALS = new Map>([ @@ -663,10 +689,14 @@ function collectHelperLiteralFindings(source: string, repoPath: string): Android } const body = source.slice(start, end); const direct = body.match(/^\s*"((?:\\.|[^"\\])+)"/u); + const directRaw = body.match(/^\s*"""([\s\S]*?)"""/u); const matches = [ ...(direct ? [direct] : []), + ...(directRaw ? [directRaw] : []), ...body.matchAll(UI_BRANCH_LITERAL_RE), ...body.matchAll(UI_ELSE_BRANCH_LITERAL_RE), + ...body.matchAll(UI_BRANCH_RAW_LITERAL_RE), + ...body.matchAll(UI_ELSE_BRANCH_RAW_LITERAL_RE), ]; const seenOffsets = new Set(); const addLiteral = (literal: string, literalOffset: number) => { @@ -685,7 +715,8 @@ function collectHelperLiteralFindings(source: string, repoPath: string): Android if (!literal) { continue; } - const literalOffset = body.indexOf(`"${literal}"`, match.index ?? 0); + const delimiter = match[0]?.includes('"""') ? '"""' : '"'; + const literalOffset = body.indexOf(`${delimiter}${literal}${delimiter}`, match.index ?? 0); addLiteral(literal, Math.max(0, literalOffset)); } for (const match of body.matchAll(/\bif\s*/gu)) { @@ -707,7 +738,10 @@ function collectHelperLiteralFindings(source: string, repoPath: string): Android cursor += 1; } } - const literal = body.slice(cursor).match(/^"((?:\\.|[^"\\])+)"/u)?.[1]; + const expression = body.slice(cursor); + const literal = + expression.match(/^"""([\s\S]*?)"""/u)?.[1] ?? + expression.match(/^"((?:\\.|[^"\\])+)"/u)?.[1]; if (literal) { addLiteral(literal, cursor); } @@ -849,8 +883,9 @@ function collectTypedModelLiteralFindings( continue; } for (const literal of argument.value.matchAll(KOTLIN_STRING_LITERAL_RE)) { + const literalValue = literal[1] ?? literal[2]; if ( - !literal[1] || + !literalValue || isLocalizedLiteral(argument.value, literal.index ?? 0) || isComparisonLiteral(argument.value, literal.index ?? 0) ) { @@ -859,7 +894,7 @@ function collectTypedModelLiteralFindings( findings.push({ line: lineNumber(source, argument.start + (literal.index ?? 0)), path: repoPath, - source: decodeKotlinLiteral(literal[1]), + source: literal[1] === undefined ? decodeKotlinLiteral(literalValue) : literalValue, }); } } @@ -876,28 +911,35 @@ export function findUnlocalizedAndroidUiLiterals( return []; } const findings = new Map(); - for (const pattern of DIRECT_UI_LITERAL_PATTERNS) { - pattern.lastIndex = 0; - for (const match of source.matchAll(pattern)) { - const literal = match[1]; - if (!literal) { - continue; + for (const [patterns, delimiter] of [ + [DIRECT_UI_LITERAL_PATTERNS, '"'], + [DIRECT_UI_RAW_LITERAL_PATTERNS, '"""'], + ] as const) { + for (const pattern of patterns) { + pattern.lastIndex = 0; + for (const match of source.matchAll(pattern)) { + const literal = match[1]; + if (!literal) { + continue; + } + const findingSource = delimiter === '"' ? decodeKotlinLiteral(literal) : literal; + if ( + isAllowedUiLiteral(repoPath, findingSource) || + literal.startsWith("http") || + literal.startsWith("content://") + ) { + continue; + } + const matchText = match[0] ?? ""; + const offset = + (match.index ?? 0) + matchText.lastIndexOf(`${delimiter}${literal}${delimiter}`); + const finding = { + line: lineNumber(source, Math.max(0, offset)), + path: repoPath, + source: findingSource, + }; + findings.set(`${finding.line}\u0000${finding.source}`, finding); } - const matchText = match[0] ?? ""; - if ( - isAllowedUiLiteral(repoPath, decodeKotlinLiteral(literal)) || - literal.startsWith("http") || - literal.startsWith("content://") - ) { - continue; - } - const offset = (match.index ?? 0) + matchText.lastIndexOf(`"${literal}"`); - const finding = { - line: lineNumber(source, Math.max(0, offset)), - path: repoPath, - source: decodeKotlinLiteral(literal), - }; - findings.set(`${finding.line}\u0000${finding.source}`, finding); } } for (const finding of [ diff --git a/test/scripts/android-app-i18n.test.ts b/test/scripts/android-app-i18n.test.ts index 7d9f3a999b19..4f9921b15950 100644 --- a/test/scripts/android-app-i18n.test.ts +++ b/test/scripts/android-app-i18n.test.ts @@ -27,7 +27,14 @@ describe("Android app i18n resources", () => { expect( findUnusedAndroidResourceKeys( ["kotlin_only", "manifest_only", "values_only", "unused"], - 'R.string.kotlin_only android:label="@string/manifest_only" @string/values_only', + [ + { path: "Example.kt", source: "R.string.kotlin_only" }, + { + path: "AndroidManifest.xml", + source: + 'android:label="@string/manifest_only" @string/values_only', + }, + ], ), ).toEqual(["unused"]); }); @@ -36,7 +43,7 @@ describe("Android app i18n resources", () => { expect( findUnusedAndroidResourceKeys( ["native_status", "native_status_detail", "native_unused"], - "R.string.native_status_detail", + [{ path: "Example.kt", source: "R.string.native_status_detail" }], ), ).toEqual(["native_status", "native_unused"]); }); @@ -45,18 +52,44 @@ describe("Android app i18n resources", () => { expect( findUnusedAndroidResourceKeys( ["kotlin_comment", "block_comment", "xml_comment", "live"], - ` - // R.string.kotlin_comment - /* R.string.block_comment */ - - val endpoint = "https://example.test" - val marker = "/* not a comment */" - R.string.live - `, + [ + { + path: "Example.kt", + source: ` + // R.string.kotlin_comment + /* R.string.block_comment */ + val endpoint = "https://example.test" + val marker = "/* not a comment */" + R.string.live + `, + }, + { + path: "AndroidManifest.xml", + source: "", + }, + ], ), ).toEqual(["kotlin_comment", "block_comment", "xml_comment"]); }); + it("ignores Android resource references inside Kotlin strings", () => { + expect( + findUnusedAndroidResourceKeys( + ["regular_string", "raw_string", "live"], + [ + { + path: "Example.kt", + source: ` + val regular = "R.string.regular_string" + val raw = """R.string.raw_string""" + R.string.live + `, + }, + ], + ), + ).toEqual(["regular_string", "raw_string"]); + }); + it("selects duplicate-source translations by frequency then stable text order", () => { expect(selectDeterministicTranslation("Source", ["Beta", "Alpha", "Beta"])).toBe("Beta"); expect(selectDeterministicTranslation("Source", ["Beta", "Alpha"])).toBe("Alpha"); @@ -196,6 +229,25 @@ describe("Android app i18n resources", () => { expect(findings).not.toContain("Ready"); }); + it("finds raw strings in direct UI and presentation helpers", () => { + const source = ` + Text("""Direct raw copy""") + + fun diagnosticsReport(): String = + """ + Raw helper copy + """.trimIndent() + `; + const findings = findUnlocalizedAndroidUiLiterals( + source, + "apps/android/app/src/main/java/ai/openclaw/app/ui/Example.kt", + ).map((finding) => finding.source); + + expect(findings).toEqual( + expect.arrayContaining(["Direct raw copy", expect.stringContaining("Raw helper copy")]), + ); + }); + it("inventories command, attention, and overview model display literals", () => { const source = ` data class CommandItem(