diff --git a/scripts/native-app-i18n.ts b/scripts/native-app-i18n.ts index b705c23a8e55..b32fbbb478af 100644 --- a/scripts/native-app-i18n.ts +++ b/scripts/native-app-i18n.ts @@ -46,6 +46,19 @@ type NativeTranslationArtifact = { locale: string; version: 1; }; +export type NativeI18nQualityFinding = { + code: + | "adjacent-duplicate-word" + | "android-language-picker-source-equal" + | "same-source-contradiction" + | "source-equal"; + id: string; + locale: string; + relatedIds?: string[]; + source: string; + translated: string; + words?: string[]; +}; type NativeTranslator = typeof translateNativeEntries; type NativeLocaleSyncOptions = { glossary?: Array<{ source: string; target: string }>; @@ -125,18 +138,13 @@ const ANDROID_BUILTIN_UI_CALLS = new Set([ "TextButton", "TopAppBar", ]); -const CONDITIONAL_BRANCHES = [ - /\bif\s*\([^)]*\)\s*"((?:\\.|[^"\\])*)"\s*else\s*"((?:\\.|[^"\\])*)"/gu, - /\?\s*"((?:\\.|[^"\\])*)"\s*:\s*"((?:\\.|[^"\\])*)"/gu, -]; const UI_STRING_NAME_RE = /(?:title|subtitle|body|message|label|text|description|detail|prompt|placeholder|help)$/iu; const APPLE_STRING_PROPERTY = /\bvar\s+([A-Za-z_][A-Za-z0-9_]*)\s*:\s*String\s*\{/gu; -const APPLE_SWITCH_BRANCH = - /(?:\bcase\b[^:\n]+|\bdefault)\s*:\s*(?:return\s+)?"((?:\\.|[^"\\])*)"/gu; +const APPLE_SWITCH_BRANCH_START = /(?:\bcase\b[^:\n]+|\bdefault)\s*:\s*(?:return\s+)?/gu; const ANDROID_STRING_FUNCTION = /\bfun\s+([A-Za-z_][A-Za-z0-9_]*)\s*\([^)]*\)\s*:\s*String\s*(=|\{)/gu; -const ANDROID_WHEN_BRANCH = /(?:[^\n{}]+|\belse)\s*->\s*"((?:\\.|[^"\\])*)"/gu; +const ANDROID_WHEN_BRANCH_START = /(?:[^\n{}]+|\belse)\s*->\s*/gu; const ANDROID_RESOURCE_STRINGS = /]*>([\s\S]*?)<\/string>/gu; const ANDROID_RESOURCE_COLLECTIONS = /<(?:string-array|plurals)\b[^>]*>([\s\S]*?)<\/(?:string-array|plurals)>/gu; @@ -176,6 +184,12 @@ const EXCLUDED_PATH_RE = /(?:^|[\\/])(?:Tests?|UITests?|test|Preview(?:s)?)(?:$| const EXCLUDED_FILE_RE = /(?:Tests?|UITests?|Previews?|Testing)\.(?:swift|kt|kts)$/u; const BUILD_SETTING_RE = /\$\([A-Za-z0-9_.-]+\)/gu; const NATIVE_I18N_LOCALE_SET = new Set(NATIVE_I18N_LOCALES); +const ANDROID_LANGUAGE_PICKER_PATH = + "apps/android/app/src/main/java/ai/openclaw/app/AppLanguage.kt"; +const ANDROID_LANGUAGE_PICKER_SOURCES = new Set([ + "Follow Android · $systemLanguageTag", + "OpenClaw translations · $languageTag", +]); function isAsciiLowercaseLetter(character: string): boolean { return character >= "a" && character <= "z"; @@ -481,26 +495,67 @@ function readKotlinStringLiteral( return null; } -function extractKotlinStringLiterals(source: string, start: number, end: number) { - const values: Array<{ offset: number; value: string }> = []; - let cursor = start; - while (cursor < end) { - const openingQuote = source.indexOf('"', cursor); - if (openingQuote < 0 || openingQuote >= end) { - break; - } - const literal = readKotlinStringLiteral(source, openingQuote); - if (!literal || literal.end > end) { - break; - } - values.push({ offset: openingQuote, value: literal.value }); - cursor = literal.end; +function readMultilineStringLiteral( + source: string, + openingQuote: number, +): { end: number; value: string } | null { + if (!source.startsWith('"""', openingQuote)) { + return null; } - return values; + const closingQuote = source.indexOf('"""', openingQuote + 3); + if (closingQuote < 0) { + return null; + } + return { + end: closingQuote + 3, + value: decodeMultilineLiteral(source.slice(openingQuote + 3, closingQuote)), + }; } -function extractSwiftUiCalls( +function readNativeStringLiteral( + surface: NativeI18nSurface, + source: string, + openingQuote: number, +): { end: number; value: string } | null { + return ( + readMultilineStringLiteral(source, openingQuote) ?? + (surface === "apple" + ? readSwiftStringLiteral(source, openingQuote) + : readKotlinStringLiteral(source, openingQuote)) + ); +} + +function readAdjacentStringLiterals( + surface: NativeI18nSurface, + source: string, + openingQuote: number, +): { end: number; fragments: number; value: string } | null { + const first = readNativeStringLiteral(surface, source, openingQuote); + if (!first) { + return null; + } + const values = [first.value]; + let cursor = first.end; + for (;;) { + const separator = source.slice(cursor).match(/^\s*\+\s*/u)?.[0]; + if (!separator) { + return { end: cursor, fragments: values.length, value: values.join("") }; + } + cursor += separator.length; + const next = readNativeStringLiteral(surface, source, cursor); + // A runtime concatenation is not a compile-time literal. Dropping the whole + // candidate prevents the inventory from preserving a misleading prefix. + if (!next) { + return null; + } + values.push(next.value); + cursor = next.end; + } +} + +function extractUiCalls( entries: Candidate[], + surface: NativeI18nSurface, repoPath: string, source: string, uiCallNames: ReadonlySet, @@ -510,32 +565,13 @@ function extractSwiftUiCalls( continue; } const offset = match.index ?? 0; - let cursor = offset + match[0].length; - const first = readSwiftStringLiteral(source, cursor); - if (!first) { + const openingQuote = offset + match[0].length; + const literal = readAdjacentStringLiterals(surface, source, openingQuote); + if (!literal) { continue; } - const values = [first.value]; - cursor = first.end; - let unsupportedConcatenation = false; - while (true) { - const separator = source.slice(cursor).match(/^\s*\+\s*/u)?.[0]; - if (!separator) { - break; - } - cursor += separator.length; - const next = readSwiftStringLiteral(source, cursor); - if (!next) { - unsupportedConcatenation = true; - break; - } - values.push(next.value); - cursor = next.end; - } - if (!unsupportedConcatenation) { - const kind = values.length > 1 ? "ui-call-concatenated" : "ui-call"; - addCandidate(entries, "apple", repoPath, values.join(""), kind, lineNumber(source, offset)); - } + const kind = literal.fragments > 1 ? "ui-call-concatenated" : "ui-call"; + addCandidate(entries, surface, repoPath, literal.value, kind, lineNumber(source, offset)); } } @@ -652,11 +688,175 @@ function addCandidate( entries.push({ kind, line, path: repoPath, source: normalized, surface }); } -function extractCandidates( +function findCapturedLiteralOffset( + source: string, + match: RegExpMatchArray, + value: string, + searchStart: number, +): number | null { + const matchEnd = (match.index ?? 0) + match[0].length; + const normal = source.indexOf(`"${value}"`, searchStart); + const multiline = source.indexOf(`"""${value}"""`, searchStart); + const offsets = [normal, multiline].filter((offset) => offset >= 0 && offset < matchEnd); + return offsets.length > 0 ? Math.min(...offsets) : null; +} + +function addCapturedLiteralCandidates( + entries: Candidate[], surface: NativeI18nSurface, repoPath: string, source: string, - uiCallNames: ReadonlySet, + match: RegExpMatchArray, + kind: string, +) { + let searchStart = match.index ?? 0; + for (const value of match.slice(1)) { + if (!value) { + continue; + } + const openingQuote = findCapturedLiteralOffset(source, match, value, searchStart); + if (openingQuote === null) { + continue; + } + const literal = readAdjacentStringLiterals(surface, source, openingQuote); + if (literal) { + addCandidate( + entries, + surface, + repoPath, + literal.value, + literal.fragments > 1 ? `${kind}-concatenated` : kind, + lineNumber(source, openingQuote), + ); + searchStart = literal.end; + } else { + searchStart = openingQuote + 1; + } + } +} + +function skipWhitespaceAndBrace(source: string, offset: number): number { + let cursor = offset; + while (cursor < source.length && /\s/u.test(source[cursor])) { + cursor += 1; + } + if (source[cursor] === "{") { + cursor += 1; + while (cursor < source.length && /\s/u.test(source[cursor])) { + cursor += 1; + } + } + if (source.startsWith("return", cursor) && !isAsciiAlphaNumeric(source[cursor + 6] ?? "")) { + cursor += 6; + while (cursor < source.length && /\s/u.test(source[cursor])) { + cursor += 1; + } + } + return cursor; +} + +function addConditionalBranchPair( + entries: Candidate[], + surface: NativeI18nSurface, + repoPath: string, + source: string, + firstOffset: number, + separator: RegExp, +) { + const firstStart = skipWhitespaceAndBrace(source, firstOffset); + const first = readAdjacentStringLiterals(surface, source, firstStart); + if (!first) { + return; + } + const remainder = source.slice(first.end); + const separatorMatch = remainder.match(separator); + if (!separatorMatch) { + return; + } + const secondStart = skipWhitespaceAndBrace(source, first.end + separatorMatch[0].length); + const second = readAdjacentStringLiterals(surface, source, secondStart); + if (!second) { + return; + } + for (const branch of [ + { offset: firstStart, value: first.value }, + { offset: secondStart, value: second.value }, + ]) { + addCandidate( + entries, + surface, + repoPath, + branch.value, + "conditional-branch", + lineNumber(source, branch.offset), + ); + } +} + +function extractConditionalBranches( + entries: Candidate[], + surface: NativeI18nSurface, + repoPath: string, + source: string, +) { + for (const match of source.matchAll(/\bif\s*\([^)]*\)\s*/gu)) { + addConditionalBranchPair( + entries, + surface, + repoPath, + source, + (match.index ?? 0) + match[0].length, + /^\s*\}?\s*else\s*/u, + ); + } + for (const match of source.matchAll(/\?\s*/gu)) { + addConditionalBranchPair( + entries, + surface, + repoPath, + source, + (match.index ?? 0) + match[0].length, + /^\s*:\s*/u, + ); + } +} + +function addBranchCandidates( + entries: Candidate[], + surface: NativeI18nSurface, + repoPath: string, + source: string, + bodyOffset: number, + body: string, + branchStart: RegExp, +) { + for (const branch of body.matchAll(branchStart)) { + const openingQuote = skipWhitespaceAndBrace( + source, + bodyOffset + (branch.index ?? 0) + branch[0].length, + ); + const literal = readAdjacentStringLiterals(surface, source, openingQuote); + if (literal) { + addCandidate( + entries, + surface, + repoPath, + literal.value, + "conditional-branch", + lineNumber(source, openingQuote), + ); + } + } +} + +export function extractNativeI18nCandidates( + surface: NativeI18nSurface, + repoPath: string, + source: string, + uiCallNames: ReadonlySet = new Set([ + ...APPLE_BUILTIN_UI_CALLS, + ...ANDROID_BUILTIN_UI_CALLS, + ]), ): Candidate[] { const entries: Candidate[] = []; const patterns: Array = @@ -667,7 +867,6 @@ function extractCandidates( [APPLE_LOCALIZED_STRING_MULTILINE_CALLS, "ui-localized-call-multiline"], [APPLE_MODIFIER_CALLS, "ui-modifier"], [APPLE_MODIFIER_MULTILINE_CALLS, "ui-modifier-multiline"], - ...CONDITIONAL_BRANCHES.map((pattern) => [pattern, "conditional-branch"] as const), ] : [ [ANDROID_CALLS, "ui-call"], @@ -675,20 +874,15 @@ function extractCandidates( [ANDROID_CHOOSER_ARGS, "ui-chooser"], [ANDROID_DIALOG_CALLS, "ui-dialog"], [ANDROID_UI_STATE_TEXT, "ui-state-text"], - ...CONDITIONAL_BRANCHES.map((pattern) => [pattern, "conditional-branch"] as const), ]; for (const [pattern, kind] of patterns) { for (const match of source.matchAll(pattern)) { - const offset = match.index ?? 0; - for (const value of match.slice(1)) { - if (value) { - addCandidate(entries, surface, repoPath, value, kind, lineNumber(source, offset)); - } - } + addCapturedLiteralCandidates(entries, surface, repoPath, source, match, kind); } } + extractConditionalBranches(entries, surface, repoPath, source); + extractUiCalls(entries, surface, repoPath, source, uiCallNames); if (surface === "apple") { - extractSwiftUiCalls(entries, repoPath, source, uiCallNames); for (const property of source.matchAll(APPLE_STRING_PROPERTY)) { const name = property[1]; const openingBrace = (property.index ?? 0) + property[0].lastIndexOf("{"); @@ -700,18 +894,15 @@ function extractCandidates( if (!/\bswitch\b/u.test(body)) { continue; } - for (const branch of body.matchAll(APPLE_SWITCH_BRANCH)) { - if (branch[1]) { - addCandidate( - entries, - surface, - repoPath, - branch[1], - "conditional-branch", - lineNumber(source, openingBrace + 1 + (branch.index ?? 0)), - ); - } - } + addBranchCandidates( + entries, + surface, + repoPath, + source, + openingBrace + 1, + body, + APPLE_SWITCH_BRANCH_START, + ); } for (const match of source.matchAll(APPLE_NAMED_LITERALS)) { const argumentName = match[1]; @@ -727,13 +918,13 @@ function extractCandidates( const multiline = match[2]; const literal = multiline ?? match[3]; if (literal) { - addCandidate( + addCapturedLiteralCandidates( entries, surface, repoPath, - literal, + source, + match, multiline === undefined ? "ui-named-argument" : "ui-named-argument-multiline", - lineNumber(source, match.index ?? 0), ); } } @@ -753,17 +944,20 @@ function extractCandidates( continue; } const body = source.slice(bodyStart, closingBrace); - for (const returnLine of body.matchAll(/\breturn\b([^\n]*)/gu)) { - const lineStart = bodyStart + (returnLine.index ?? 0); - const lineEnd = lineStart + returnLine[0].length; - for (const literal of extractKotlinStringLiterals(source, lineStart, lineEnd)) { + for (const returnKeyword of body.matchAll(/\breturn\b/gu)) { + const openingQuote = skipWhitespaceAndBrace( + source, + bodyStart + (returnKeyword.index ?? 0) + returnKeyword[0].length, + ); + const literal = readAdjacentStringLiterals(surface, source, openingQuote); + if (literal) { addCandidate( entries, surface, repoPath, literal.value, "conditional-branch", - lineNumber(source, literal.offset), + lineNumber(source, openingQuote), ); } } @@ -778,34 +972,27 @@ function extractCandidates( continue; } const body = source.slice(openingBrace + 1, closingBrace); - for (const branch of body.matchAll(ANDROID_WHEN_BRANCH)) { - if (!branch[1]) { - continue; - } - addCandidate( - entries, - surface, - repoPath, - branch[1], - "conditional-branch", - lineNumber(source, openingBrace + 1 + (branch.index ?? 0)), - ); - } + addBranchCandidates( + entries, + surface, + repoPath, + source, + openingBrace + 1, + body, + ANDROID_WHEN_BRANCH_START, + ); continue; } - const expressionLine = expression.split("\n", 1)[0] ?? ""; - for (const literal of extractKotlinStringLiterals( - source, - bodyStart, - bodyStart + expressionLine.length, - )) { + const openingQuote = skipWhitespaceAndBrace(source, bodyStart); + const literal = readAdjacentStringLiterals(surface, source, openingQuote); + if (literal) { addCandidate( entries, surface, repoPath, literal.value, "conditional-branch", - lineNumber(source, literal.offset), + lineNumber(source, openingQuote), ); } } @@ -821,14 +1008,7 @@ function extractCandidates( ) { continue; } - addCandidate( - entries, - surface, - repoPath, - match[2], - "ui-named-argument", - lineNumber(source, match.index ?? 0), - ); + addCapturedLiteralCandidates(entries, surface, repoPath, source, match, "ui-named-argument"); } } if (surface === "android" && /\/res\/values\/[^/]+\.xml$/u.test(repoPath)) { @@ -878,7 +1058,11 @@ function extractCandidates( } } } - return entries; + return [ + ...new Map( + entries.map((entry) => [[entry.surface, entry.path, entry.source].join("\u0000"), entry]), + ).values(), + ]; } async function walkFiles(root: string, surface: NativeI18nSurface): Promise { @@ -1044,7 +1228,7 @@ export async function collectNativeI18nEntries( } } const entries = typedSources.flatMap(({ repoPath, source, surface }) => - extractCandidates(surface, repoPath, source, uiCallNames), + extractNativeI18nCandidates(surface, repoPath, source, uiCallNames), ); return assignNativeI18nIds(entries, stableEntries); } @@ -1061,10 +1245,19 @@ async function syncNativeI18n(options: { const entries = await collectNativeI18nEntries(currentInventory.entries); const expected = render(entries); const current = currentInventory.raw; - if (current !== expected && options.checkOnly) { - throw new Error( - "native app i18n inventory drift detected. Run `pnpm native:i18n:sync` and commit apps/.i18n/native-source.json.", + if (options.checkOnly) { + const findings = await checkNativeLocaleArtifacts(currentInventory.entries); + for (const finding of findings) { + process.stdout.write(`native-app-i18n: advisory=${JSON.stringify(finding)}\n`); + } + process.stdout.write( + `native-app-i18n: locale-artifacts=${NATIVE_I18N_LOCALES.length} advisories=${findings.length}\n`, ); + if (current !== expected) { + throw new Error( + "native app i18n inventory drift detected. Run `pnpm native:i18n:sync` and commit apps/.i18n/native-source.json.", + ); + } } if (current !== expected && options.write) { await mkdir(path.dirname(OUTPUT_PATH), { recursive: true }); @@ -1088,6 +1281,217 @@ async function loadGlossary(locale: string): Promise match[0]); + const duplicates = new Set(); + for (let index = 1; index < words.length; index += 1) { + if ( + words[index - 1].normalize("NFKC").toLocaleLowerCase(locale) === + words[index].normalize("NFKC").toLocaleLowerCase(locale) + ) { + duplicates.add(words[index]); + } + } + return [...duplicates].toSorted(compareCodePoints); +} + +function collectNativeI18nQualityFindings( + locale: string, + inventory: readonly NativeI18nEntry[], + entries: readonly { id: string; source: string; translated: string }[], +): NativeI18nQualityFinding[] { + const inventoryById = new Map(inventory.map((entry) => [entry.id, entry])); + const translatedBySource = new Map>(); + for (const entry of entries) { + const existing = translatedBySource.get(entry.source) ?? []; + existing.push({ id: entry.id, translated: entry.translated }); + translatedBySource.set(entry.source, existing); + } + + const findings: NativeI18nQualityFinding[] = []; + for (const entry of entries) { + const sourceEqual = entry.translated === entry.source; + if (sourceEqual) { + findings.push({ code: "source-equal", locale, ...entry }); + const relatedIds = (translatedBySource.get(entry.source) ?? []) + .filter((candidate) => candidate.id !== entry.id && candidate.translated !== entry.source) + .map((candidate) => candidate.id) + .toSorted(compareCodePoints); + if (relatedIds.length > 0) { + findings.push({ + code: "same-source-contradiction", + locale, + ...entry, + relatedIds, + }); + } + const inventoryEntry = inventoryById.get(entry.id); + if ( + inventoryEntry?.surface === "android" && + inventoryEntry.path === ANDROID_LANGUAGE_PICKER_PATH && + ANDROID_LANGUAGE_PICKER_SOURCES.has(entry.source) + ) { + findings.push({ + code: "android-language-picker-source-equal", + locale, + ...entry, + }); + } + } + + const words = adjacentDuplicateWords(entry.translated, locale); + if (words.length > 0) { + findings.push({ + code: "adjacent-duplicate-word", + locale, + ...entry, + words, + }); + } + } + return findings.toSorted( + (left, right) => + compareCodePoints(left.locale, right.locale) || + compareCodePoints(left.code, right.code) || + compareCodePoints(left.id, right.id), + ); +} + +function describeArtifactValue(value: unknown): string { + return typeof value === "string" ? JSON.stringify(value) : String(value); +} + +export function validateNativeLocaleArtifact( + locale: string, + inventory: readonly NativeI18nEntry[], + artifactValue: unknown, + glossary: readonly { source: string; target: string }[] = [], +): NativeI18nQualityFinding[] { + const errors: string[] = []; + if (!artifactValue || typeof artifactValue !== "object" || Array.isArray(artifactValue)) { + throw new Error(`invalid native locale artifact ${locale}: expected an object`); + } + const artifact = artifactValue as { + entries?: unknown; + glossaryHash?: unknown; + locale?: unknown; + version?: unknown; + }; + if (artifact.version !== 1) { + errors.push(`version must be 1, got ${describeArtifactValue(artifact.version)}`); + } + if (artifact.locale !== locale) { + errors.push( + `locale must be ${JSON.stringify(locale)}, got ${describeArtifactValue(artifact.locale)}`, + ); + } + const expectedGlossaryHash = glossaryHash(glossary); + if (artifact.glossaryHash !== expectedGlossaryHash) { + errors.push( + `glossaryHash must be ${expectedGlossaryHash}, got ${describeArtifactValue(artifact.glossaryHash)}`, + ); + } + if (!Array.isArray(artifact.entries)) { + errors.push("entries must be an array"); + } + + const rawEntries = Array.isArray(artifact.entries) ? artifact.entries : []; + const entries: Array<{ id: string; source: string; translated: string }> = []; + const seenIds = new Set(); + for (const [index, value] of rawEntries.entries()) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + errors.push(`entries[${index}] must be an object`); + continue; + } + const entry = value as { id?: unknown; source?: unknown; translated?: unknown }; + if ( + typeof entry.id !== "string" || + typeof entry.source !== "string" || + typeof entry.translated !== "string" + ) { + errors.push(`entries[${index}] must contain string id, source, and translated fields`); + continue; + } + if (seenIds.has(entry.id)) { + errors.push(`duplicate id ${JSON.stringify(entry.id)}`); + } + seenIds.add(entry.id); + entries.push({ id: entry.id, source: entry.source, translated: entry.translated }); + } + + if (rawEntries.length !== inventory.length) { + errors.push(`entry count must be ${inventory.length}, got ${rawEntries.length}`); + } + for (let index = 0; index < Math.min(entries.length, inventory.length); index += 1) { + const actual = entries[index]; + const expected = inventory[index]; + if (actual.id !== expected.id) { + errors.push( + `entries[${index}].id must be ${JSON.stringify(expected.id)}, got ${JSON.stringify(actual.id)}`, + ); + } + if (actual.source !== expected.source) { + errors.push(`entries[${index}].source does not match inventory id ${expected.id}`); + } + if (!actual.translated.trim()) { + errors.push(`entries[${index}].translated must be nonempty for ${actual.id}`); + } else if ( + structuralTokenSignature(actual.source) !== structuralTokenSignature(actual.translated) + ) { + errors.push(`translation changed structural tokens or line breaks for ${actual.id}`); + } + } + if (errors.length > 0) { + throw new Error(`invalid native locale artifact ${locale}:\n- ${errors.join("\n- ")}`); + } + return collectNativeI18nQualityFindings(locale, inventory, entries); +} + +export async function checkNativeLocaleArtifacts( + inventory: readonly NativeI18nEntry[], + translationsDir = TRANSLATIONS_DIR, +): Promise { + const expectedFiles = NATIVE_I18N_LOCALES.map((locale) => `${locale}.json`).toSorted( + compareCodePoints, + ); + const actualFiles = (await readdir(translationsDir, { withFileTypes: true })) + .filter((entry) => entry.isFile() && entry.name.endsWith(".json")) + .map((entry) => entry.name) + .toSorted(compareCodePoints); + const errors: string[] = []; + if (JSON.stringify(actualFiles) !== JSON.stringify(expectedFiles)) { + errors.push( + `locale files must be ${JSON.stringify(expectedFiles)}, got ${JSON.stringify(actualFiles)}`, + ); + } + + const findings: NativeI18nQualityFinding[] = []; + for (const locale of NATIVE_I18N_LOCALES) { + const artifactPath = path.join(translationsDir, `${locale}.json`); + try { + const artifact: unknown = JSON.parse(await readFile(artifactPath, "utf8")); + findings.push( + ...validateNativeLocaleArtifact(locale, inventory, artifact, await loadGlossary(locale)), + ); + } catch (error) { + errors.push(error instanceof Error ? error.message : String(error)); + } + } + if (errors.length > 0) { + throw new Error(`native locale artifact validation failed:\n${errors.join("\n")}`); + } + return findings.toSorted( + (left, right) => + compareCodePoints(left.locale, right.locale) || + compareCodePoints(left.code, right.code) || + compareCodePoints(left.id, right.id), + ); +} + export async function syncNativeLocale( locale: string, entries: NativeI18nEntry[], @@ -1097,7 +1501,7 @@ export async function syncNativeLocale( // artifacts keep the shared translation-memory handoff current between them. const artifactPath = path.join(options.translationsDir ?? TRANSLATIONS_DIR, `${locale}.json`); const glossary = options.glossary ?? (await loadGlossary(locale)); - const glossaryHash = createHash("sha256").update(JSON.stringify(glossary)).digest("hex"); + const currentGlossaryHash = glossaryHash(glossary); let previousRaw = ""; let previous: NativeTranslationArtifact = { entries: [], @@ -1112,7 +1516,7 @@ export async function syncNativeLocale( // The first refresh creates the locale artifact. } const previousById = new Map(previous.entries.map((entry) => [entry.id, entry])); - const glossaryChanged = previous.glossaryHash !== glossaryHash; + const glossaryChanged = previous.glossaryHash !== currentGlossaryHash; const pending = entries .filter((entry) => { const current = previousById.get(entry.id); @@ -1131,7 +1535,7 @@ export async function syncNativeLocale( const artifact: NativeTranslationArtifact = { version: 1, locale, - glossaryHash, + glossaryHash: currentGlossaryHash, entries: entries.map((entry) => ({ id: entry.id, source: entry.source, @@ -1139,12 +1543,20 @@ export async function syncNativeLocale( translated.get(entry.id) ?? previousById.get(entry.id)?.translated ?? entry.source, })), }; - for (const entry of artifact.entries) { - if (structuralTokenSignature(entry.source) !== structuralTokenSignature(entry.translated)) { + try { + validateNativeLocaleArtifact(locale, entries, artifact, glossary); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const structural = message.match( + /translation changed structural tokens or line breaks for ([^\s]+)/u, + ); + if (structural?.[1]) { throw new Error( - `native translation changed placeholders or line breaks for ${locale}:${entry.id}`, + `native translation changed placeholders or line breaks for ${locale}:${structural[1]}`, + { cause: error }, ); } + throw error; } const rendered = `${JSON.stringify(artifact, null, 2)}\n`; const changed = previousRaw !== rendered; diff --git a/test/scripts/native-app-i18n.test.ts b/test/scripts/native-app-i18n.test.ts index bcdd69595b8c..5eb041f4a944 100644 --- a/test/scripts/native-app-i18n.test.ts +++ b/test/scripts/native-app-i18n.test.ts @@ -1,17 +1,27 @@ +import { createHash } from "node:crypto"; import { readFile, stat } from "node:fs/promises"; import path from "node:path"; import { describe, expect, it } from "vitest"; import { assignNativeI18nIds, collectNativeI18nEntries, + extractNativeI18nCandidates, isConditionalBranchIdentifier, NATIVE_I18N_LOCALES, parseNativeI18nCommand, syncNativeLocale, type NativeI18nEntry, + validateNativeLocaleArtifact, } from "../../scripts/native-app-i18n.ts"; import { cleanupTempDirs, makeTempDir } from "../helpers/temp-dir.js"; +type NativeTranslationArtifact = { + entries: Array<{ id: string; source: string; translated: string }>; + glossaryHash: string; + locale: string; + version: 1; +}; + describe("native app i18n inventory", () => { it("keeps IDs stable across extractor classification changes", () => { const candidate = { @@ -149,6 +159,89 @@ describe("native app i18n inventory", () => { expect(isConditionalBranchIdentifier(`a${"A".repeat(4_096)}!`)).toBe(false); }); + it("joins adjacent literals across supported Swift and Kotlin UI expressions", () => { + const swift = extractNativeI18nCandidates( + "apple", + "apps/ios/Fixture.swift", + ` + struct Fixture: View { + var body: some View { + SettingsPageHeader( + title: "Settings", + subtitle: "Named " + "argument") + .help("Modifier " + "details") + Button("Swift first " + "argument") {} + Text(enabled ? "Enabled " + "now" : "Disabled " + "now") + } + + var statusText: String { + switch state { + case .ready: + "Switch " + "ready" + default: + return "Switch " + "waiting" + } + } + } + `, + new Set(["Button", "SettingsPageHeader", "Text"]), + ); + const kotlin = extractNativeI18nCandidates( + "android", + "apps/android/Fixture.kt", + ` + @Composable + fun Fixture() { + Text("Kotlin first " + "argument") + Text(text = "Named " + "argument") + } + + fun statusText(state: State): String = when (state) { + State.Ready -> "When " + "ready" + else -> "When " + "waiting" + } + + fun messageText(enabled: Boolean): String { + if (enabled) return "Return " + "enabled" + return "Return " + "disabled" + } + `, + ); + const sources = [...swift, ...kotlin].map((entry) => entry.source); + + expect(sources).toEqual( + expect.arrayContaining([ + "Named argument", + "Modifier details", + "Swift first argument", + "Enabled now", + "Disabled now", + "Switch ready", + "Switch waiting", + "Kotlin first argument", + "When ready", + "When waiting", + "Return enabled", + "Return disabled", + ]), + ); + expect( + sources.some((source) => + [ + "Named ", + "Modifier ", + "Enabled ", + "Disabled ", + "Switch ", + "Swift first ", + "Kotlin first ", + "When ", + "Return ", + ].includes(source), + ), + ).toBe(false); + }); + it("collects stable Android and Apple UI entries", async () => { const entries = await collectNativeI18nEntries(); const surfaces = new Set(entries.map((entry) => entry.surface)); @@ -330,6 +423,52 @@ describe("native app i18n inventory", () => { expect( entries.some((entry) => entry.source === "Some channel status checks did not complete."), ).toBe(true); + expect( + entries.some( + (entry) => + entry.source === + "Your AI-powered setup helper. It can check status, fix config, switch models, and connect channels.", + ), + ).toBe(true); + expect( + entries.some( + (entry) => + entry.source === + "Cron changes require operator.admin. Setup codes intentionally do not grant it. Reconnect with the gateway's shared token or password to request admin access. If this device still lacks it, approve the pending scope upgrade from an existing admin client.", + ), + ).toBe(true); + expect( + entries.some( + (entry) => + entry.source === + "This device needs gateway approval before Talk can use realtime voice. Audio will go directly from this device to the voice provider.", + ), + ).toBe(true); + expect( + entries.some( + (entry) => + entry.source === + "Writes a rotating, local-only log under ~/Library/Logs/OpenClaw/. Enable only while actively debugging.", + ), + ).toBe(true); + expect( + entries.some( + (entry) => + entry.source === + "Paste the token configured on the gateway host. On the gateway host, run `openclaw config get gateway.auth.token`. If the gateway uses an environment variable instead, use `OPENCLAW_GATEWAY_TOKEN`.", + ), + ).toBe(true); + expect( + entries.some((entry) => + [ + "Your AI-powered setup helper. It can check status, fix config, ", + "Cron changes require operator.admin. Setup codes intentionally do not grant it. ", + "This device needs gateway approval before Talk can use realtime voice. Audio will go directly from ", + "Writes a rotating, local-only log under ~/Library/Logs/OpenClaw/. ", + "Paste the token configured on the gateway host. ", + ].includes(entry.source), + ), + ).toBe(false); expect( entries.some( (entry) => @@ -567,6 +706,190 @@ describe("native app i18n inventory", () => { } }); + it("rejects invalid locale artifact metadata, inventory, and translations", () => { + const inventory: NativeI18nEntry[] = [ + { + id: "native.android.greeting", + kind: "ui-call", + line: 1, + path: "apps/android/Greeting.kt", + source: "Hello ${name}\nNext", + surface: "android", + }, + { + id: "native.apple.other", + kind: "ui-call", + line: 1, + path: "apps/ios/Other.swift", + source: "Other", + surface: "apple", + }, + ]; + const emptyGlossaryHash = createHash("sha256").update(JSON.stringify([])).digest("hex"); + const createArtifact = (): NativeTranslationArtifact => ({ + version: 1, + locale: "sv", + glossaryHash: emptyGlossaryHash, + entries: [ + { + id: inventory[0].id, + source: inventory[0].source, + translated: "Hej ${name}\nNästa", + }, + { + id: inventory[1].id, + source: inventory[1].source, + translated: "Annat", + }, + ], + }); + const cases: Array<{ + expected: string; + mutate: (artifact: NativeTranslationArtifact) => unknown; + }> = [ + { + expected: "version must be 1", + mutate: (artifact) => ({ ...artifact, version: 2 }), + }, + { + expected: 'locale must be "sv"', + mutate: (artifact) => ({ ...artifact, locale: "de" }), + }, + { + expected: "glossaryHash must be", + mutate: (artifact) => ({ ...artifact, glossaryHash: "stale" }), + }, + { + expected: "entry count must be 2, got 1", + mutate: (artifact) => ({ ...artifact, entries: artifact.entries.slice(0, 1) }), + }, + { + expected: 'entries[0].id must be "native.android.greeting"', + mutate: (artifact) => ({ ...artifact, entries: artifact.entries.toReversed() }), + }, + { + expected: "entries[0].source does not match inventory", + mutate: (artifact) => ({ + ...artifact, + entries: [{ ...artifact.entries[0], source: "Changed" }, artifact.entries[1]], + }), + }, + { + expected: 'duplicate id "native.android.greeting"', + mutate: (artifact) => ({ + ...artifact, + entries: [artifact.entries[0], { ...artifact.entries[1], id: artifact.entries[0].id }], + }), + }, + { + expected: "entries[1].translated must be nonempty", + mutate: (artifact) => ({ + ...artifact, + entries: [artifact.entries[0], { ...artifact.entries[1], translated: " " }], + }), + }, + { + expected: "translation changed structural tokens or line breaks", + mutate: (artifact) => ({ + ...artifact, + entries: [{ ...artifact.entries[0], translated: "Hej\nNästa" }, artifact.entries[1]], + }), + }, + { + expected: "translation changed structural tokens or line breaks", + mutate: (artifact) => ({ + ...artifact, + entries: [ + { ...artifact.entries[0], translated: "Hej ${name} Nästa" }, + artifact.entries[1], + ], + }), + }, + ]; + + expect(validateNativeLocaleArtifact("sv", inventory, createArtifact())).toEqual([]); + for (const testCase of cases) { + expect(() => + validateNativeLocaleArtifact("sv", inventory, testCase.mutate(createArtifact())), + ).toThrow(testCase.expected); + } + }); + + it("emits deterministic advisory translation-quality findings", () => { + const inventory: NativeI18nEntry[] = [ + { + id: "native.android.language-picker", + kind: "conditional-branch", + line: 89, + path: "apps/android/app/src/main/java/ai/openclaw/app/AppLanguage.kt", + source: "OpenClaw translations · $languageTag", + surface: "android", + }, + { + id: "native.android.inspect", + kind: "ui-call", + line: 1, + path: "apps/android/Workshop.kt", + source: "Inspect", + surface: "android", + }, + { + id: "native.apple.inspect", + kind: "ui-call", + line: 1, + path: "apps/ios/Workshop.swift", + source: "Inspect", + surface: "apple", + }, + { + id: "native.android.voice-note", + kind: "ui-call", + line: 1, + path: "apps/android/Voice.kt", + source: "Record voice note", + surface: "android", + }, + ]; + const artifact: NativeTranslationArtifact = { + version: 1, + locale: "id", + glossaryHash: createHash("sha256").update(JSON.stringify([])).digest("hex"), + entries: [ + { + id: inventory[0].id, + source: inventory[0].source, + translated: inventory[0].source, + }, + { + id: inventory[1].id, + source: inventory[1].source, + translated: inventory[1].source, + }, + { + id: inventory[2].id, + source: inventory[2].source, + translated: "Periksa", + }, + { + id: inventory[3].id, + source: inventory[3].source, + translated: "Ghi ghi chú thoại", + }, + ], + }; + + const findings = validateNativeLocaleArtifact("id", inventory, artifact); + expect(findings.map((finding) => `${finding.code}:${finding.id}`)).toEqual([ + "adjacent-duplicate-word:native.android.voice-note", + "android-language-picker-source-equal:native.android.language-picker", + "same-source-contradiction:native.android.inspect", + "source-equal:native.android.inspect", + "source-equal:native.android.language-picker", + ]); + expect(findings[0]?.words).toEqual(["ghi"]); + expect(findings[2]?.relatedIds).toEqual(["native.apple.inspect"]); + }); + it("validates locale refresh arguments before write paths run", () => { expect(parseNativeI18nCommand(["sync", "--write", "--locale", "sv"])).toEqual({ command: "sync",