diff --git a/.github/workflows/control-ui-locale-refresh.yml b/.github/workflows/control-ui-locale-refresh.yml index 439450aa5cbb..e63f42f191a2 100644 --- a/.github/workflows/control-ui-locale-refresh.yml +++ b/.github/workflows/control-ui-locale-refresh.yml @@ -11,6 +11,9 @@ on: - ui/src/i18n/lib/types.ts - ui/src/i18n/lib/registry.ts - scripts/control-ui-i18n.ts + - scripts/control-ui-i18n-verify.ts + - scripts/lib/control-ui-i18n-config.ts + - scripts/lib/control-ui-i18n-raw-copy.ts - .github/actions/create-generated-pr-tokens/action.yml - .github/actions/publish-generated-pr/action.yml - .github/workflows/control-ui-locale-refresh.yml @@ -227,7 +230,7 @@ jobs: artifact_dir="${RUNNER_TEMP}/control-ui-locale-${LOCALE}" mkdir -p "${artifact_dir}" git add -A ui/src/i18n - git diff --cached --binary --full-index -- ui/src/i18n > "${artifact_dir}/${LOCALE}.patch" + git diff --cached --binary --full-index -- ui/src/i18n ':(exclude)ui/src/i18n/.i18n/catalog-fallbacks.json' > "${artifact_dir}/${LOCALE}.patch" - name: Upload locale artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -301,6 +304,9 @@ jobs: ui/src/i18n/lib/types.ts ui/src/i18n/lib/registry.ts scripts/control-ui-i18n.ts + scripts/control-ui-i18n-verify.ts + scripts/lib/control-ui-i18n-config.ts + scripts/lib/control-ui-i18n-raw-copy.ts .github/actions/create-generated-pr-tokens/action.yml .github/actions/publish-generated-pr/action.yml .github/workflows/control-ui-locale-refresh.yml diff --git a/package.json b/package.json index 491f6a647eef..720ebda3f748 100644 --- a/package.json +++ b/package.json @@ -1669,7 +1669,7 @@ "ios:version:check": "node --import tsx scripts/ios-sync-versioning.ts --check", "ios:version:sync": "node --import tsx scripts/ios-sync-versioning.ts --write", "leak:embedded-run": "node --import tsx --expose-gc scripts/embedded-run-abort-leak.ts", - "lint": "node scripts/run-oxlint-shards.mjs", + "lint": "pnpm lint:ui:i18n && node scripts/run-oxlint-shards.mjs", "lint:agent:ingress-owner": "node scripts/check-ingress-agent-owner-context.mjs", "lint:all": "node scripts/run-oxlint.mjs", "lint:apps": "pnpm lint:swift", @@ -1707,6 +1707,7 @@ "lint:tmp:session-transcript-reader-boundary": "node scripts/check-session-transcript-reader-boundary.mjs", "lint:tmp:sqlite-transaction-boundary": "node scripts/check-sqlite-transaction-boundary.mjs", "lint:tmp:tsgo-core-boundary": "node scripts/check-tsgo-core-boundary.mjs", + "lint:ui:i18n": "pnpm ui:i18n:verify", "lint:ui:no-raw-window-open": "node scripts/check-no-raw-window-open.mjs", "lint:web-fetch-provider-boundaries": "node scripts/check-web-fetch-provider-boundaries.mjs", "lint:web-search-provider-boundaries": "node scripts/check-web-search-provider-boundaries.mjs", @@ -1999,9 +2000,11 @@ "tui:pty:test:watch:local": "node --import tsx scripts/dev/tui-pty-test-watch.ts --mode local", "ui:build": "node scripts/ui.js build && node scripts/check-control-ui-precompressed-assets.mjs && node scripts/check-control-ui-performance.mjs", "ui:dev": "node scripts/ui.js dev", + "ui:i18n:baseline": "node --import tsx scripts/control-ui-i18n-verify.ts baseline", "ui:i18n:check": "node --import tsx scripts/control-ui-i18n.ts check", "ui:i18n:report": "node --import tsx scripts/control-ui-i18n-report.ts", "ui:i18n:sync": "node --import tsx scripts/control-ui-i18n.ts sync --write", + "ui:i18n:verify": "node --import tsx scripts/control-ui-i18n-verify.ts verify", "native:i18n:check": "node --import tsx scripts/native-app-i18n.ts check", "native:i18n:sync": "node --import tsx scripts/native-app-i18n.ts sync --write", "android:i18n:check": "node --import tsx scripts/android-app-i18n.ts check", diff --git a/scripts/check-changed.d.mts b/scripts/check-changed.d.mts index f0c6a0b338f6..82398352b38a 100644 --- a/scripts/check-changed.d.mts +++ b/scripts/check-changed.d.mts @@ -41,6 +41,7 @@ export function buildChangedCheckCrabboxArgs(argv?: string[], options?: { cwd?: export function shouldRunShrinkwrapGuard(paths: string[]): boolean; export function shouldRunPromptSnapshotCheck(paths: string[]): boolean; export function shouldRunPromptSnapshotOwnerTest(paths: string[]): boolean; +export function shouldRunControlUiI18nVerify(paths: string[]): boolean; export function shouldRunRuntimeSidecarBaselineCheck(paths: string[]): boolean; export function shouldRunSqliteSessionSchemaBaselineCheck(paths: string[]): boolean; export function shouldRunPluginSdkApiBaselineCheck(paths: string[]): boolean; diff --git a/scripts/check-changed.mjs b/scripts/check-changed.mjs index 4c6c7b4f999d..9df133323f6c 100644 --- a/scripts/check-changed.mjs +++ b/scripts/check-changed.mjs @@ -45,6 +45,8 @@ const PLUGIN_SDK_SURFACE_PATH_RE = /^(?:package\.json$|src\/plugin-sdk\/|scripts\/(?:plugin-sdk-surface-report\.mjs|sync-plugin-sdk-exports\.mjs|lib\/plugin-sdk-(?:declaration-budget\.mjs|deprecated-barrel-subpaths\.json|deprecated-public-subpaths\.json|entries\.mjs|entrypoints\.json|private-local-only-subpaths\.json)))/u; const CANVAS_A2UI_NATIVE_RESOURCE_PATH_RE = /^(?:pnpm-lock\.yaml$|apps\/shared\/OpenClawKit\/Sources\/OpenClawKit\/Resources\/CanvasA2UI\/|extensions\/canvas\/(?:package\.json$|scripts\/bundle-a2ui\.mjs$|src\/host\/a2ui(?:\/(?:index\.html|a2ui\.bundle\.js|\.bundle\.hash)$|-app\/))|scripts\/(?:bundle-a2ui|sync-native-a2ui)\.mjs$)/u; +const CONTROL_UI_I18N_VERIFY_PATH_RE = + /^(?:package\.json$|ui\/src\/|scripts\/(?:control-ui-i18n(?:-(?:report|verify))?\.ts|lib\/control-ui-i18n-[^/]+\.ts)$|test\/scripts\/control-ui-i18n[^/]*\.test\.ts$)/u; const CORE_OXLINT_TS_CONFIG = "config/tsconfig/oxlint.core.json"; const EXTENSIONS_OXLINT_TS_CONFIG = "config/tsconfig/oxlint.extensions.json"; const SCRIPTS_OXLINT_TS_CONFIG = "config/tsconfig/oxlint.scripts.json"; @@ -245,6 +247,12 @@ export function shouldRunPromptSnapshotOwnerTest(paths) { return paths.some((changedPath) => PROMPT_SNAPSHOT_OWNER_TEST_PATH_RE.test(changedPath)); } +export function shouldRunControlUiI18nVerify(paths) { + return paths.some((changedPath) => + CONTROL_UI_I18N_VERIFY_PATH_RE.test(normalizeChangedPath(changedPath)), + ); +} + export function shouldRunRuntimeSidecarBaselineCheck(paths) { return paths.some((changedPath) => RUNTIME_SIDECAR_BASELINE_PATH_RE.test(changedPath)); } @@ -486,6 +494,10 @@ export function createChangedCheckPlan(result, options = {}) { }; } + if (shouldRunControlUiI18nVerify(result.paths)) { + addLint("Control UI i18n catalog", ["lint:ui:i18n"]); + } + if (lanes.core) { addTypecheck("typecheck core", ["tsgo:core"]); } diff --git a/scripts/ci-changed-scope.mjs b/scripts/ci-changed-scope.mjs index 0323a92e161a..2b8ffa43e4f7 100644 --- a/scripts/ci-changed-scope.mjs +++ b/scripts/ci-changed-scope.mjs @@ -56,7 +56,7 @@ const WINDOWS_TEST_SCOPE_RE = const WINDOWS_DAEMON_SCOPE_RE = /^src\/daemon\/(?:schtasks(?:[-.][^/]+)?|runtime-hints\.windows-paths(?:\.test)?|test-helpers\/schtasks-(?:base-mocks|fixtures))\.ts$/; const CONTROL_UI_I18N_SCOPE_RE = - /^(ui\/src\/i18n\/|scripts\/control-ui-i18n\.ts$|\.github\/workflows\/control-ui-locale-refresh\.yml$)/; + /^(ui\/src\/i18n\/|scripts\/(?:control-ui-i18n(?:-verify)?\.ts|lib\/control-ui-i18n-(?:config|raw-copy)\.ts)$|\.github\/workflows\/control-ui-locale-refresh\.yml$)/; const CONTROL_UI_TEST_SCOPE_RE = /^(ui\/|test\/vitest\/vitest\.shared\.config\.ts$|scripts\/ensure-playwright-chromium\.mjs$)/; const NATIVE_I18N_SCOPE_RE = diff --git a/scripts/control-ui-i18n-verify.ts b/scripts/control-ui-i18n-verify.ts new file mode 100644 index 000000000000..6c872c3a2b4f --- /dev/null +++ b/scripts/control-ui-i18n-verify.ts @@ -0,0 +1,297 @@ +import { createHash } from "node:crypto"; +import { existsSync } from "node:fs"; +import { mkdir, readFile, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { CONTROL_UI_LOCALE_ENTRIES } from "./lib/control-ui-i18n-config.ts"; +import { syncControlUiRawCopyBaseline } from "./lib/control-ui-i18n-raw-copy.ts"; +import type { TranslationMap } from "./lib/control-ui-i18n-sync-plan.ts"; + +export type CatalogFallbackBaseline = { + fallbacks: Record; + sourceHash: string; + version: number; +}; + +function fallbackPairs(baseline: CatalogFallbackBaseline): Set { + return new Set( + Object.entries(baseline.fallbacks).flatMap(([key, locales]) => + locales.map((locale) => `${key}\u0000${locale}`), + ), + ); +} + +export function assertScopedCatalogFallbackUpdate( + current: CatalogFallbackBaseline, + next: CatalogFallbackBaseline, + resolvedLocale: string, +) { + if (current.version !== next.version || current.sourceHash !== next.sourceHash) { + throw new Error("scoped locale sync cannot update a stale catalog fallback baseline"); + } + const currentPairs = fallbackPairs(current); + const nextPairs = fallbackPairs(next); + const added = [...nextPairs].filter((pair) => !currentPairs.has(pair)); + const unrelatedRemovals = [...currentPairs].filter( + (pair) => !nextPairs.has(pair) && !pair.endsWith(`\u0000${resolvedLocale}`), + ); + if (added.length > 0 || unrelatedRemovals.length > 0) { + throw new Error( + `scoped locale sync for ${resolvedLocale} found unrelated catalog fallback drift; run pnpm ui:i18n:baseline first`, + ); + } +} + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const LOCALES_DIR = path.join(ROOT, "ui", "src", "i18n", "locales"); +const I18N_ASSETS_DIR = path.join(ROOT, "ui", "src", "i18n", ".i18n"); +const SOURCE_LOCALE_PATH = path.join(LOCALES_DIR, "en.ts"); +const FALLBACK_BASELINE_PATH = path.join(I18N_ASSETS_DIR, "catalog-fallbacks.json"); +const FALLBACK_BASELINE_VERSION = 1; + +function compareStringArrays(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((value, index) => value === right[index]); +} + +function toRepoPath(filePath: string): string { + return path.relative(ROOT, filePath).split(path.sep).join("/"); +} + +async function importLocaleModule(filePath: string): Promise { + const stats = await stat(filePath); + return (await import(`${pathToFileURL(filePath).href}?ts=${stats.mtimeMs}`)) as T; +} + +async function loadLocaleMap(filePath: string, exportName: string): Promise { + if (!existsSync(filePath)) { + return null; + } + const mod = await importLocaleModule>(filePath); + return mod[exportName] ?? null; +} + +function localeFilePath(fileName: string): string { + return path.join(LOCALES_DIR, fileName); +} + +function extractPlaceholders(text: string): string[] { + return [...new Set([...text.matchAll(/\{(\w+)\}/g)].map((match) => match[1] ?? ""))] + .filter(Boolean) + .toSorted((left, right) => left.localeCompare(right)); +} + +export function flattenControlUiCatalog( + value: unknown, + label: string, + prefix = "", + out = new Map(), +): Map { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${label}${prefix ? `:${prefix}` : ""} must be an object`); + } + for (const [key, nested] of Object.entries(value)) { + const fullKey = prefix ? `${prefix}.${key}` : key; + if (typeof nested === "string") { + out.set(fullKey, nested); + } else if (!nested || typeof nested !== "object" || Array.isArray(nested)) { + throw new Error(`${label}:${fullKey} must be a string or object`); + } else { + flattenControlUiCatalog(nested, label, fullKey, out); + } + } + return out; +} + +export function analyzeControlUiCatalogs( + sourceFlat: ReadonlyMap, + localeFlats: ReadonlyMap>, +): { errors: string[]; fallbacks: Record } { + const errors: string[] = []; + const sourceKeys = [...sourceFlat.keys()]; + const sourceKeySet = new Set(sourceKeys); + const fallbackLocalesByKey = new Map(); + + for (const [locale, localeFlat] of [...localeFlats.entries()].toSorted(([left], [right]) => + left.localeCompare(right), + )) { + const localeKeys = [...localeFlat.keys()]; + const orphanKeys = localeKeys.filter((key) => !sourceKeySet.has(key)); + if (orphanKeys.length > 0) { + errors.push(`${locale}: orphan keys: ${orphanKeys.join(", ")}`); + } + + const expectedPresentOrder = sourceKeys.filter((key) => localeFlat.has(key)); + const actualPresentOrder = localeKeys.filter((key) => sourceKeySet.has(key)); + if (!compareStringArrays(actualPresentOrder, expectedPresentOrder)) { + errors.push(`${locale}: keys are not in English catalog order`); + } + + for (const key of sourceKeys) { + const translated = localeFlat.get(key); + if (translated === undefined) { + const locales = fallbackLocalesByKey.get(key) ?? []; + locales.push(locale); + fallbackLocalesByKey.set(key, locales); + continue; + } + const sourcePlaceholders = extractPlaceholders(sourceFlat.get(key) ?? ""); + const translatedPlaceholders = extractPlaceholders(translated); + if (!compareStringArrays(sourcePlaceholders, translatedPlaceholders)) { + errors.push( + `${locale}:${key} expected {${sourcePlaceholders.join("},{")}} got {${translatedPlaceholders.join("},{")}}`, + ); + } + } + } + + const fallbacks: Record = {}; + for (const [key, locales] of [...fallbackLocalesByKey.entries()].toSorted(([left], [right]) => + left.localeCompare(right), + )) { + fallbacks[key] = locales.toSorted((left, right) => left.localeCompare(right)); + } + return { errors, fallbacks }; +} + +async function buildCatalogFallbackBaseline(): Promise { + const sourceRaw = await readFile(SOURCE_LOCALE_PATH, "utf8"); + const sourceMap = await loadLocaleMap(SOURCE_LOCALE_PATH, "en"); + if (!sourceMap) { + throw new Error("ui/src/i18n/locales/en.ts does not export en"); + } + const sourceFlat = flattenControlUiCatalog(sourceMap, "en"); + const localeFlats = new Map>(); + for (const entry of CONTROL_UI_LOCALE_ENTRIES) { + const filePath = localeFilePath(entry.fileName); + const localeMap = await loadLocaleMap(filePath, entry.exportName); + if (!localeMap) { + throw new Error(`${toRepoPath(filePath)} does not export ${entry.exportName}`); + } + localeFlats.set(entry.locale, flattenControlUiCatalog(localeMap, entry.locale)); + } + + const analysis = analyzeControlUiCatalogs(sourceFlat, localeFlats); + if (analysis.errors.length > 0) { + throw new Error( + [ + "control-ui catalog verification failed.", + analysis.errors.slice(0, 50).join("\n"), + analysis.errors.length > 50 ? `...and ${analysis.errors.length - 50} more` : "", + ] + .filter(Boolean) + .join("\n"), + ); + } + + return { + fallbacks: analysis.fallbacks, + sourceHash: createHash("sha256").update(sourceRaw).digest("hex"), + version: FALLBACK_BASELINE_VERSION, + }; +} + +export async function syncControlUiCatalogFallbackBaseline(options: { + checkOnly: boolean; + resolvedLocale?: string; + write: boolean; +}) { + const baseline = await buildCatalogFallbackBaseline(); + const expected = `${JSON.stringify(baseline, null, 2)}\n`; + const current = existsSync(FALLBACK_BASELINE_PATH) + ? await readFile(FALLBACK_BASELINE_PATH, "utf8") + : ""; + if (!options.checkOnly && options.write && current !== expected) { + if (options.resolvedLocale) { + let currentBaseline: CatalogFallbackBaseline; + try { + currentBaseline = JSON.parse(current) as CatalogFallbackBaseline; + assertScopedCatalogFallbackUpdate(currentBaseline, baseline, options.resolvedLocale); + } catch (error) { + throw new Error( + `cannot refresh catalog fallback metadata after scoped locale sync: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + } + } + await mkdir(I18N_ASSETS_DIR, { recursive: true }); + await writeFile(FALLBACK_BASELINE_PATH, expected, "utf8"); + } + if (options.checkOnly && current !== expected) { + throw new Error( + [ + "control-ui catalog fallback baseline drift detected.", + `Run \`pnpm ui:i18n:baseline\` and commit ${toRepoPath(FALLBACK_BASELINE_PATH)}.`, + ].join("\n"), + ); + } + const fallbackPairCount = Object.values(baseline.fallbacks).reduce( + (total, locales) => total + locales.length, + 0, + ); + process.stdout.write( + `control-ui-i18n: catalog: fallback_keys=${Object.keys(baseline.fallbacks).length} fallback_pairs=${fallbackPairCount}\n`, + ); +} + +export async function verifyRuntimeLocaleConfig() { + const registryRaw = await readFile( + path.join(ROOT, "ui", "src", "i18n", "lib", "registry.ts"), + "utf8", + ); + const typesRaw = await readFile(path.join(ROOT, "ui", "src", "i18n", "lib", "types.ts"), "utf8"); + for (const entry of CONTROL_UI_LOCALE_ENTRIES) { + if (!registryRaw.includes(`"${entry.locale}"`) || !typesRaw.includes(`| "${entry.locale}"`)) { + throw new Error(`runtime locale config is missing ${entry.locale}`); + } + } + + const enMap = (await loadLocaleMap(SOURCE_LOCALE_PATH, "en")) ?? {}; + const languageMap = enMap.languages; + const languageKeys = + languageMap && typeof languageMap === "object" + ? Object.keys(languageMap).toSorted((left, right) => left.localeCompare(right)) + : []; + const expectedLanguageKeys = [ + "en", + ...CONTROL_UI_LOCALE_ENTRIES.map((entry) => entry.languageKey), + ].toSorted((left, right) => left.localeCompare(right)); + if (!compareStringArrays(languageKeys, expectedLanguageKeys)) { + throw new Error( + `ui/src/i18n/locales/en.ts languages block is out of sync: expected ${expectedLanguageKeys.join(", ")}, got ${languageKeys.join(", ")}`, + ); + } +} + +export async function verifyControlUiCatalogs(options: { checkOnly: boolean; write: boolean }) { + await verifyRuntimeLocaleConfig(); + await syncControlUiRawCopyBaseline(options); + await syncControlUiCatalogFallbackBaseline(options); +} + +function usage(): never { + console.error("Usage: node --import tsx scripts/control-ui-i18n-verify.ts "); + process.exit(2); +} + +async function main() { + const [command, ...rest] = process.argv.slice(2); + if ((command !== "verify" && command !== "baseline") || rest.length > 0) { + usage(); + } + await verifyControlUiCatalogs({ + checkOnly: command === "verify", + write: command === "baseline", + }); +} + +function isCliEntrypoint() { + const entrypoint = process.argv[1]; + return Boolean(entrypoint && import.meta.url === pathToFileURL(path.resolve(entrypoint)).href); +} + +if (isCliEntrypoint()) { + await main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }); +} diff --git a/scripts/control-ui-i18n.ts b/scripts/control-ui-i18n.ts index 63161d6a261d..8657704e15e9 100644 --- a/scripts/control-ui-i18n.ts +++ b/scripts/control-ui-i18n.ts @@ -2,14 +2,20 @@ import { spawn, spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { existsSync } from "node:fs"; -import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises"; +import { mkdir, readFile, stat, writeFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { completeSimple, type AssistantMessage, type Model } from "openclaw/plugin-sdk/llm"; -import * as ts from "typescript"; import { expectDefined } from "../packages/normalization-core/src/expect.js"; import { formatErrorMessage } from "../src/infra/errors.ts"; import { formatDurationCompact } from "../src/infra/format-time/format-duration.ts"; +import { + syncControlUiCatalogFallbackBaseline, + verifyControlUiCatalogs, + verifyRuntimeLocaleConfig, +} from "./control-ui-i18n-verify.ts"; +import { CONTROL_UI_LOCALE_ENTRIES } from "./lib/control-ui-i18n-config.ts"; +import { syncControlUiRawCopyBaseline } from "./lib/control-ui-i18n-raw-copy.ts"; import { compareStringArrays, createControlUiLocaleSyncPlan, @@ -45,27 +51,6 @@ type RunProcessParentSignalState = { signal: NodeJS.Signals | null; }; -type RawCopyFinding = { - kind: "html-attribute" | "html-text" | "object-property"; - line: number; - name: string; - path: string; - text: string; -}; - -type RawCopyBaselineEntry = { - count: number; - kind: RawCopyFinding["kind"]; - name: string; - path: string; - text: string; -}; - -type RawCopyBaseline = { - version: number; - entries: RawCopyBaselineEntry[]; -}; - const CONTROL_UI_I18N_WORKFLOW = 1; const DEFAULT_OPENAI_MODEL = "gpt-5.6-sol"; const DEFAULT_ANTHROPIC_MODEL = "claude-opus-4-6"; @@ -76,14 +61,6 @@ const LOCALES_DIR = path.join(ROOT, "ui", "src", "i18n", "locales"); const I18N_ASSETS_DIR = path.join(ROOT, "ui", "src", "i18n", ".i18n"); const SOURCE_LOCALE_PATH = path.join(LOCALES_DIR, "en.ts"); const SOURCE_LOCALE = "en"; -const CONTROL_UI_RAW_COPY_SOURCE_DIRS = [ - path.join(ROOT, "ui", "src", "app"), - path.join(ROOT, "ui", "src", "components"), - path.join(ROOT, "ui", "src", "lib"), - path.join(ROOT, "ui", "src", "pages"), -] as const; -const RAW_COPY_BASELINE_PATH = path.join(I18N_ASSETS_DIR, "raw-copy-baseline.json"); -const RAW_COPY_BASELINE_VERSION = 1; const MAX_BATCH_ITEMS = 20; const DEFAULT_BATCH_CHAR_BUDGET = 2_000; const TRANSLATE_MAX_ATTEMPTS = 2; @@ -126,28 +103,7 @@ const TRANSLATION_PROVIDER_DEFAULTS: Record & { text: string }, -) { - const text = normalizeRawCopyText(params.text); - if (!text || !hasHumanLetters(text)) { - return; - } - findings.push({ - ...params, - text, - }); -} - -async function walkControlUiSourceFiles(dir: string): Promise { - const entries = await readdir(dir, { withFileTypes: true }); - const files: string[] = []; - for (const entry of entries) { - if (entry.name === "test-helpers") { - continue; - } - const fullPath = path.join(dir, entry.name); - if (entry.isDirectory()) { - files.push(...(await walkControlUiSourceFiles(fullPath))); - continue; - } - if (!entry.isFile() || !/\.tsx?$/u.test(entry.name)) { - continue; - } - if (/\.(?:test|browser\.test|node\.test)\.tsx?$/u.test(entry.name)) { - continue; - } - files.push(fullPath); - } - return files; -} - -function collectRawCopyFromSource(params: { - filePath: string; - source: string; - sourceFile: ts.SourceFile; -}): RawCopyFinding[] { - const { filePath, source, sourceFile } = params; - const repoPath = toRepoPath(filePath); - const findings: RawCopyFinding[] = []; - const attrPattern = - /\b(aria-label|placeholder|title)\s*=\s*"((?:(?!\$\{)[^"\\]|\\.)*?\p{L}(?:(?!\$\{)[^"\\]|\\.)*?)"/gu; - for (const match of source.matchAll(attrPattern)) { - const rawText = match[2]; - if (!rawText) { - continue; - } - pushRawCopyFinding(findings, { - kind: "html-attribute", - line: lineNumberForOffset(source, match.index ?? 0), - name: match[1] ?? "attribute", - path: repoPath, - text: parseDoubleQuotedString(rawText), - }); - } - - const propertyPattern = - /\b(label|title|subtitle|description|help|placeholder)\s*:\s*"((?:[^"\\]|\\.)*?\p{L}(?:[^"\\]|\\.)*?)"/gu; - for (const match of source.matchAll(propertyPattern)) { - const rawText = match[2]; - if (!rawText) { - continue; - } - pushRawCopyFinding(findings, { - kind: "object-property", - line: lineNumberForOffset(source, match.index ?? 0), - name: match[1] ?? "property", - path: repoPath, - text: parseDoubleQuotedString(rawText), - }); - } - - const textPattern = />\s*([^<>{}]*?\p{L}[^<>{}]*?)\s* { - if (ts.isTaggedTemplateExpression(node) && node.tag.getText(sourceFile) === "html") { - const template = node.template; - const chunks: Array<{ offset: number; text: string }> = []; - if (ts.isNoSubstitutionTemplateLiteral(template)) { - chunks.push({ - offset: template.getStart(sourceFile) + 1, - text: template.text, - }); - } else { - chunks.push({ - offset: template.head.getStart(sourceFile) + 1, - text: template.head.text, - }); - for (const span of template.templateSpans) { - chunks.push({ - offset: span.literal.getStart(sourceFile) + 1, - text: span.literal.text, - }); - } - } - for (const chunk of chunks) { - for (const match of chunk.text.matchAll(textPattern)) { - const rawText = match[1]; - if (!rawText) { - continue; - } - pushRawCopyFinding(findings, { - kind: "html-text", - line: lineNumberForOffset(source, chunk.offset + (match.index ?? 0)), - name: "text", - path: repoPath, - text: rawText, - }); - } - } - } - ts.forEachChild(node, visit); - }; - visit(sourceFile); - - return findings; -} - -async function collectControlUiRawCopyFindings(): Promise { - const files = ( - await Promise.all(CONTROL_UI_RAW_COPY_SOURCE_DIRS.map((dir) => walkControlUiSourceFiles(dir))) - ).flat(); - const findings: RawCopyFinding[] = []; - for (const filePath of files.toSorted((left, right) => left.localeCompare(right))) { - const source = await readFile(filePath, "utf8"); - const sourceFile = ts.createSourceFile( - filePath, - source, - ts.ScriptTarget.Latest, - true, - filePath.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS, - ); - findings.push(...collectRawCopyFromSource({ filePath, source, sourceFile })); - } - return findings; -} - -function summarizeRawCopyFindings(findings: RawCopyFinding[]): RawCopyBaselineEntry[] { - const counts = new Map(); - for (const finding of findings) { - const key = [finding.path, finding.kind, finding.name, finding.text].join("\u0000"); - const existing = counts.get(key); - if (existing) { - existing.count += 1; - continue; - } - counts.set(key, { - count: 1, - kind: finding.kind, - name: finding.name, - path: finding.path, - text: finding.text, - }); - } - return [...counts.values()].toSorted( - (left, right) => - left.path.localeCompare(right.path) || - left.kind.localeCompare(right.kind) || - left.name.localeCompare(right.name) || - left.text.localeCompare(right.text), - ); -} - -function formatRawCopyBaseline(entries: RawCopyBaselineEntry[]): string { - return `${JSON.stringify( - { - version: RAW_COPY_BASELINE_VERSION, - entries, - } satisfies RawCopyBaseline, - null, - 2, - )}\n`; -} - -function formatRawCopyBaselineDiff( - current: RawCopyBaselineEntry[], - expected: RawCopyBaselineEntry[], -) { - const keyFor = (entry: RawCopyBaselineEntry) => - [entry.path, entry.kind, entry.name, entry.text].join("\u0000"); - const currentByKey = new Map(current.map((entry) => [keyFor(entry), entry])); - const expectedByKey = new Map(expected.map((entry) => [keyFor(entry), entry])); - const added = current.filter((entry) => { - const expectedEntry = expectedByKey.get(keyFor(entry)); - return !expectedEntry || expectedEntry.count !== entry.count; - }); - const removed = expected.filter((entry) => { - const currentEntry = currentByKey.get(keyFor(entry)); - return !currentEntry || currentEntry.count !== entry.count; - }); - const lines: string[] = []; - for (const entry of added.slice(0, 20)) { - lines.push( - `+ ${entry.path} ${entry.kind}:${entry.name} x${entry.count} ${JSON.stringify(entry.text)}`, - ); - } - for (const entry of removed.slice(0, 20)) { - lines.push( - `- ${entry.path} ${entry.kind}:${entry.name} x${entry.count} ${JSON.stringify(entry.text)}`, - ); - } - const extra = added.length + removed.length - lines.length; - if (extra > 0) { - lines.push(`... ${extra} more baseline delta(s)`); - } - return lines.join("\n"); -} - -async function syncControlUiRawCopyBaseline(options: { checkOnly: boolean; write: boolean }) { - const findings = await collectControlUiRawCopyFindings(); - const entries = summarizeRawCopyFindings(findings); - const expected = formatRawCopyBaseline(entries); - const current = existsSync(RAW_COPY_BASELINE_PATH) - ? await readFile(RAW_COPY_BASELINE_PATH, "utf8") - : ""; - if (!options.checkOnly && options.write && current !== expected) { - await mkdir(I18N_ASSETS_DIR, { recursive: true }); - await writeFile(RAW_COPY_BASELINE_PATH, expected, "utf8"); - } - if (options.checkOnly && current !== expected) { - let currentEntries: RawCopyBaselineEntry[]; - try { - const parsed = JSON.parse(current) as Partial; - currentEntries = Array.isArray(parsed.entries) ? parsed.entries : []; - } catch { - currentEntries = []; - } - const diff = formatRawCopyBaselineDiff(entries, currentEntries); - throw new Error( - [ - "control-ui raw-copy baseline drift detected.", - diff, - "Move user-facing strings into ui/src/i18n/locales/en.ts, or update the baseline with `node --import tsx scripts/control-ui-i18n.ts sync --write` when the raw string is intentional.", - ] - .filter(Boolean) - .join("\n"), - ); - } - logProgress(`raw-copy: baseline entries=${entries.length}`); -} - function isPromptTimeoutError(error: Error): boolean { return error.message.toLowerCase().includes("timed out"); } @@ -1450,7 +1127,7 @@ type SyncOutcome = { async function syncLocale( entry: LocaleEntry, - options: { checkOnly: boolean; force: boolean; write: boolean }, + options: { allowTranslate: boolean; checkOnly: boolean; force: boolean; write: boolean }, context: LocaleRunContext, ) { const localeLabel = formatLocaleLabel(entry.locale, context); @@ -1466,7 +1143,7 @@ async function syncLocale( const glossaryFilePath = glossaryPath(entry); const glossary = await loadGlossary(glossaryFilePath); const tm = await loadTranslationMemory(tmPath(entry)); - const allowTranslate = hasTranslationProvider(); + const allowTranslate = options.allowTranslate; const plan = createControlUiLocaleSyncPlan({ allowTranslate, cacheKeyFor: (key, textHash) => cacheKey(key, textHash, entry.locale), @@ -1622,41 +1299,19 @@ async function syncLocale( } satisfies SyncOutcome; } -async function verifyRuntimeLocaleConfig() { - const registryRaw = await readFile( - path.join(ROOT, "ui", "src", "i18n", "lib", "registry.ts"), - "utf8", - ); - const typesRaw = await readFile(path.join(ROOT, "ui", "src", "i18n", "lib", "types.ts"), "utf8"); - const expectedLocaleSnippets = LOCALE_ENTRIES.map((entry) => entry.locale); - for (const locale of expectedLocaleSnippets) { - if (!registryRaw.includes(`"${locale}"`) || !typesRaw.includes(`| "${locale}"`)) { - throw new Error(`runtime locale config is missing ${locale}`); - } - } - - const enMap = (await loadLocaleMap(SOURCE_LOCALE_PATH, "en")) ?? {}; - const languageMap = enMap.languages; - const languageKeys = - languageMap && typeof languageMap === "object" - ? Object.keys(languageMap).toSorted((left, right) => left.localeCompare(right)) - : []; - const expectedLanguageKeys = ["en", ...LOCALE_ENTRIES.map((entry) => entry.languageKey)].toSorted( - (left, right) => left.localeCompare(right), - ); - if (!compareStringArrays(languageKeys, expectedLanguageKeys)) { - throw new Error( - `ui/src/i18n/locales/en.ts languages block is out of sync: expected ${expectedLanguageKeys.join(", ")}, got ${languageKeys.join(", ")}`, - ); - } -} - async function main() { const args = parseArgs(process.argv.slice(2)); - await verifyRuntimeLocaleConfig(); - if (args.command === "check" || (args.command === "sync" && args.write && !args.localeFilter)) { + if (args.command === "check") { + await verifyControlUiCatalogs({ + checkOnly: true, + write: false, + }); + } else { + await verifyRuntimeLocaleConfig(); + } + if (args.command === "sync" && args.write && !args.localeFilter) { await syncControlUiRawCopyBaseline({ - checkOnly: args.command === "check", + checkOnly: false, write: args.write, }); } @@ -1669,14 +1324,16 @@ async function main() { throw new Error(`unknown locale: ${args.localeFilter}`); } + const allowTranslate = args.command === "sync" && hasTranslationProvider(); logProgress( - `command=${args.command} locales=${entries.length} provider=${hasTranslationProvider() ? resolveConfiguredProvider() : "fallback-only"} model=${hasTranslationProvider() ? resolveConfiguredModel() : "n/a"} thinking=${hasTranslationProvider() ? resolveThinkingLevel() : "n/a"} timeout=${formatDuration(resolvePromptTimeoutMs())} batch_chars=${resolveBatchCharBudget()}`, + `command=${args.command} locales=${entries.length} provider=${allowTranslate ? resolveConfiguredProvider() : "disabled"} model=${allowTranslate ? resolveConfiguredModel() : "n/a"} thinking=${allowTranslate ? resolveThinkingLevel() : "n/a"} timeout=${formatDuration(resolvePromptTimeoutMs())} batch_chars=${resolveBatchCharBudget()}`, ); const outcomes: SyncOutcome[] = []; for (const [index, entry] of entries.entries()) { const outcome = await syncLocale( entry, { + allowTranslate, checkOnly: args.command === "check", force: args.force, write: args.write, @@ -1698,6 +1355,14 @@ async function main() { .join("\n"); process.stdout.write(`${summary}\n`); + if (args.command === "sync" && args.write) { + await syncControlUiCatalogFallbackBaseline({ + checkOnly: false, + resolvedLocale: args.localeFilter ?? undefined, + write: true, + }); + } + if (args.command === "check" && changed.length > 0) { throw new Error( [ diff --git a/scripts/lib/control-ui-i18n-config.ts b/scripts/lib/control-ui-i18n-config.ts new file mode 100644 index 000000000000..c36a0f408dfd --- /dev/null +++ b/scripts/lib/control-ui-i18n-config.ts @@ -0,0 +1,24 @@ +import type { LocaleEntry } from "./control-ui-i18n-sync-plan.ts"; + +export const CONTROL_UI_LOCALE_ENTRIES: readonly LocaleEntry[] = [ + { locale: "zh-CN", fileName: "zh-CN.ts", exportName: "zh_CN", languageKey: "zhCN" }, + { locale: "zh-TW", fileName: "zh-TW.ts", exportName: "zh_TW", languageKey: "zhTW" }, + { locale: "pt-BR", fileName: "pt-BR.ts", exportName: "pt_BR", languageKey: "ptBR" }, + { locale: "de", fileName: "de.ts", exportName: "de", languageKey: "de" }, + { locale: "es", fileName: "es.ts", exportName: "es", languageKey: "es" }, + { locale: "ja-JP", fileName: "ja-JP.ts", exportName: "ja_JP", languageKey: "jaJP" }, + { locale: "ko", fileName: "ko.ts", exportName: "ko", languageKey: "ko" }, + { locale: "fr", fileName: "fr.ts", exportName: "fr", languageKey: "fr" }, + { locale: "hi", fileName: "hi.ts", exportName: "hi", languageKey: "hi" }, + { locale: "ar", fileName: "ar.ts", exportName: "ar", languageKey: "ar" }, + { locale: "it", fileName: "it.ts", exportName: "it", languageKey: "it" }, + { locale: "tr", fileName: "tr.ts", exportName: "tr", languageKey: "tr" }, + { locale: "uk", fileName: "uk.ts", exportName: "uk", languageKey: "uk" }, + { locale: "id", fileName: "id.ts", exportName: "id", languageKey: "id" }, + { locale: "pl", fileName: "pl.ts", exportName: "pl", languageKey: "pl" }, + { locale: "th", fileName: "th.ts", exportName: "th", languageKey: "th" }, + { locale: "vi", fileName: "vi.ts", exportName: "vi", languageKey: "vi" }, + { locale: "nl", fileName: "nl.ts", exportName: "nl", languageKey: "nl" }, + { locale: "fa", fileName: "fa.ts", exportName: "fa", languageKey: "fa" }, + { locale: "ru", fileName: "ru.ts", exportName: "ru", languageKey: "ru" }, +]; diff --git a/scripts/lib/control-ui-i18n-raw-copy.ts b/scripts/lib/control-ui-i18n-raw-copy.ts new file mode 100644 index 000000000000..97f2b01b7d7e --- /dev/null +++ b/scripts/lib/control-ui-i18n-raw-copy.ts @@ -0,0 +1,308 @@ +import { existsSync } from "node:fs"; +import { mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import * as ts from "typescript"; + +type RawCopyFinding = { + kind: "html-attribute" | "html-text" | "object-property"; + line: number; + name: string; + path: string; + text: string; +}; + +type RawCopyBaselineEntry = { + count: number; + kind: RawCopyFinding["kind"]; + name: string; + path: string; + text: string; +}; + +type RawCopyBaseline = { + entries: RawCopyBaselineEntry[]; + version: number; +}; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const I18N_ASSETS_DIR = path.join(ROOT, "ui", "src", "i18n", ".i18n"); +const SOURCE_DIRS = [ + path.join(ROOT, "ui", "src", "app"), + path.join(ROOT, "ui", "src", "components"), + path.join(ROOT, "ui", "src", "lib"), + path.join(ROOT, "ui", "src", "pages"), +] as const; +const BASELINE_PATH = path.join(I18N_ASSETS_DIR, "raw-copy-baseline.json"); +const BASELINE_VERSION = 1; +const INTERPOLATION_MARKER = "\u0000"; + +function toRepoPath(filePath: string): string { + return path.relative(ROOT, filePath).split(path.sep).join("/"); +} + +function normalizeRawCopyText(raw: string): string { + return raw + .replace(/\\n/g, " ") + .replace(/\s+/g, " ") + .replace(/·/giu, "·") + .trim(); +} + +function lineNumberForOffset(source: string, offset: number): number { + let line = 1; + for (let index = 0; index < offset && index < source.length; index += 1) { + if (source.charCodeAt(index) === 10) { + line += 1; + } + } + return line; +} + +function parseDoubleQuotedString(raw: string): string { + try { + return JSON.parse(`"${raw}"`) as string; + } catch { + return raw; + } +} + +function pushRawCopyFinding( + findings: RawCopyFinding[], + params: Omit & { text: string }, +) { + const text = normalizeRawCopyText(params.text); + if (!text || !/\p{L}/u.test(text)) { + return; + } + findings.push({ ...params, text }); +} + +function pushRawCopySegments( + findings: RawCopyFinding[], + params: Omit & { text: string }, +) { + for (const text of params.text.split(INTERPOLATION_MARKER)) { + pushRawCopyFinding(findings, { ...params, text }); + } +} + +async function walkSourceFiles(dir: string): Promise { + const entries = await readdir(dir, { withFileTypes: true }); + const files: string[] = []; + for (const entry of entries) { + if (entry.name === "test-helpers") { + continue; + } + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + files.push(...(await walkSourceFiles(fullPath))); + continue; + } + if ( + entry.isFile() && + /\.tsx?$/u.test(entry.name) && + !/\.(?:test|browser\.test|node\.test)\.tsx?$/u.test(entry.name) + ) { + files.push(fullPath); + } + } + return files; +} + +export function collectControlUiRawCopyFromSource(params: { + filePath: string; + source: string; + sourceFile: ts.SourceFile; +}): RawCopyFinding[] { + const { filePath, source, sourceFile } = params; + const repoPath = toRepoPath(filePath); + const findings: RawCopyFinding[] = []; + const staticAttrPattern = + /\b(aria-label|placeholder|title)\s*=\s*"((?:(?!\$\{)[^"\\]|\\.)*?\p{L}(?:(?!\$\{)[^"\\]|\\.)*?)"/gu; + for (const match of source.matchAll(staticAttrPattern)) { + const rawText = match[2]; + if (rawText) { + pushRawCopyFinding(findings, { + kind: "html-attribute", + line: lineNumberForOffset(source, match.index ?? 0), + name: match[1] ?? "attribute", + path: repoPath, + text: parseDoubleQuotedString(rawText), + }); + } + } + + const propertyPattern = + /\b(label|title|subtitle|description|help|placeholder)\s*:\s*"((?:[^"\\]|\\.)*?\p{L}(?:[^"\\]|\\.)*?)"/gu; + for (const match of source.matchAll(propertyPattern)) { + const rawText = match[2]; + if (rawText) { + pushRawCopyFinding(findings, { + kind: "object-property", + line: lineNumberForOffset(source, match.index ?? 0), + name: match[1] ?? "property", + path: repoPath, + text: parseDoubleQuotedString(rawText), + }); + } + } + + const attrPattern = + /\b(aria-label|placeholder|title)\s*=\s*"((?:[^"\\]|\\.)*?\p{L}(?:[^"\\]|\\.)*?)"/gu; + const textPattern = />\s*([^<>{}]*?\p{L}[^<>{}]*?)\s* { + if (ts.isTaggedTemplateExpression(node) && node.tag.getText(sourceFile) === "html") { + let logicalText: string; + if (ts.isNoSubstitutionTemplateLiteral(node.template)) { + logicalText = node.template.text; + } else { + logicalText = [ + node.template.head.text, + ...node.template.templateSpans.map((span) => span.literal.text), + ].join(INTERPOLATION_MARKER); + } + const line = lineNumberForOffset(source, node.template.getStart(sourceFile)); + for (const match of logicalText.matchAll(attrPattern)) { + const rawText = match[2]; + if (rawText?.includes(INTERPOLATION_MARKER)) { + pushRawCopySegments(findings, { + kind: "html-attribute", + line, + name: match[1] ?? "attribute", + path: repoPath, + text: parseDoubleQuotedString(rawText), + }); + } + } + for (const match of logicalText.matchAll(textPattern)) { + const rawText = match[1]; + if (rawText) { + pushRawCopySegments(findings, { + kind: "html-text", + line, + name: "text", + path: repoPath, + text: rawText, + }); + } + } + } + ts.forEachChild(node, visit); + }; + visit(sourceFile); + return findings; +} + +async function collectFindings(): Promise { + const files = (await Promise.all(SOURCE_DIRS.map((dir) => walkSourceFiles(dir)))).flat(); + const findings: RawCopyFinding[] = []; + for (const filePath of files.toSorted((left, right) => left.localeCompare(right))) { + const source = await readFile(filePath, "utf8"); + const sourceFile = ts.createSourceFile( + filePath, + source, + ts.ScriptTarget.Latest, + true, + filePath.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS, + ); + findings.push(...collectControlUiRawCopyFromSource({ filePath, source, sourceFile })); + } + return findings; +} + +function summarize(findings: RawCopyFinding[]): RawCopyBaselineEntry[] { + const counts = new Map(); + for (const finding of findings) { + const key = [finding.path, finding.kind, finding.name, finding.text].join("\u0000"); + const existing = counts.get(key); + if (existing) { + existing.count += 1; + } else { + counts.set(key, { + count: 1, + kind: finding.kind, + name: finding.name, + path: finding.path, + text: finding.text, + }); + } + } + return [...counts.values()].toSorted( + (left, right) => + left.path.localeCompare(right.path) || + left.kind.localeCompare(right.kind) || + left.name.localeCompare(right.name) || + left.text.localeCompare(right.text), + ); +} + +function formatBaseline(entries: RawCopyBaselineEntry[]): string { + return `${JSON.stringify({ version: BASELINE_VERSION, entries } satisfies RawCopyBaseline, null, 2)}\n`; +} + +function formatDiff(current: RawCopyBaselineEntry[], expected: RawCopyBaselineEntry[]): string { + const keyFor = (entry: RawCopyBaselineEntry) => + [entry.path, entry.kind, entry.name, entry.text].join("\u0000"); + const currentByKey = new Map(current.map((entry) => [keyFor(entry), entry])); + const expectedByKey = new Map(expected.map((entry) => [keyFor(entry), entry])); + const added = current.filter((entry) => { + const expectedEntry = expectedByKey.get(keyFor(entry)); + return !expectedEntry || expectedEntry.count !== entry.count; + }); + const removed = expected.filter((entry) => { + const currentEntry = currentByKey.get(keyFor(entry)); + return !currentEntry || currentEntry.count !== entry.count; + }); + const lines = [ + ...added + .slice(0, 20) + .map( + (entry) => + `+ ${entry.path} ${entry.kind}:${entry.name} x${entry.count} ${JSON.stringify(entry.text)}`, + ), + ...removed + .slice(0, 20) + .map( + (entry) => + `- ${entry.path} ${entry.kind}:${entry.name} x${entry.count} ${JSON.stringify(entry.text)}`, + ), + ]; + const extra = added.length + removed.length - lines.length; + if (extra > 0) { + lines.push(`... ${extra} more baseline delta(s)`); + } + return lines.join("\n"); +} + +export async function syncControlUiRawCopyBaseline(options: { + checkOnly: boolean; + write: boolean; +}) { + const entries = summarize(await collectFindings()); + const expected = formatBaseline(entries); + const current = existsSync(BASELINE_PATH) ? await readFile(BASELINE_PATH, "utf8") : ""; + if (!options.checkOnly && options.write && current !== expected) { + await mkdir(I18N_ASSETS_DIR, { recursive: true }); + await writeFile(BASELINE_PATH, expected, "utf8"); + } + if (options.checkOnly && current !== expected) { + let currentEntries: RawCopyBaselineEntry[] = []; + try { + const parsed = JSON.parse(current) as Partial; + currentEntries = Array.isArray(parsed.entries) ? parsed.entries : []; + } catch { + // Invalid baseline reports as a full delta below. + } + throw new Error( + [ + "control-ui raw-copy baseline drift detected.", + formatDiff(entries, currentEntries), + "Move user-facing strings into ui/src/i18n/locales/en.ts, or run `pnpm ui:i18n:baseline` when the raw string is intentional.", + ] + .filter(Boolean) + .join("\n"), + ); + } + process.stdout.write(`control-ui-i18n: raw-copy: baseline entries=${entries.length}\n`); +} diff --git a/src/scripts/ci-changed-scope.test.ts b/src/scripts/ci-changed-scope.test.ts index e92c7891a032..70b3b21ed119 100644 --- a/src/scripts/ci-changed-scope.test.ts +++ b/src/scripts/ci-changed-scope.test.ts @@ -857,17 +857,23 @@ describe("detectChangedScope", () => { runUiTests: true, }); - expect(detectChangedScope(["scripts/control-ui-i18n.ts"])).toEqual({ - runNode: true, - runMacos: false, - runIosBuild: false, - runAndroid: false, - runWindows: false, - runSkillsPython: false, - runChangedSmoke: false, - runControlUiI18n: true, - runUiTests: false, - }); + for (const scriptPath of [ + "scripts/control-ui-i18n.ts", + "scripts/control-ui-i18n-verify.ts", + "scripts/lib/control-ui-i18n-raw-copy.ts", + ]) { + expect(detectChangedScope([scriptPath])).toEqual({ + runNode: true, + runMacos: false, + runIosBuild: false, + runAndroid: false, + runWindows: false, + runSkillsPython: false, + runChangedSmoke: false, + runControlUiI18n: true, + runUiTests: false, + }); + } }); it.each([ diff --git a/test/scripts/changed-lanes.test.ts b/test/scripts/changed-lanes.test.ts index fbd03794004f..7a17e2f4a84a 100644 --- a/test/scripts/changed-lanes.test.ts +++ b/test/scripts/changed-lanes.test.ts @@ -27,6 +27,7 @@ import { shouldDelegateChangedCheckToCrabbox, shouldRunAppcastOwnerTest, shouldRunCanvasA2uiNativeResourceCheck, + shouldRunControlUiI18nVerify, shouldRunPromptSnapshotCheck, shouldRunPromptSnapshotOwnerTest, shouldRunRuntimeSidecarBaselineCheck, @@ -811,6 +812,7 @@ describe("scripts/changed-lanes", () => { }); expect(plan.commands.map((command) => command.args[0])).toContain("tsgo:ui"); expect(plan.commands.map((command) => command.args[0])).toContain("tsgo:core:test"); + expect(plan.commands.map((command) => command.args[0])).toContain("lint:ui:i18n"); expect(plan.commands.map((command) => command.args[0])).not.toContain("tsgo:core"); }); @@ -839,6 +841,15 @@ describe("scripts/changed-lanes", () => { expect(plan.commands.map((command) => command.args[0])).toContain("tsgo:scripts"); }); + it("routes Control UI i18n tooling changes through keyless catalog verification", () => { + const result = detectChangedLanes(["scripts/control-ui-i18n.ts"]); + const plan = createChangedCheckPlan(result); + + expect(shouldRunControlUiI18nVerify(result.paths)).toBe(true); + expect(plan.commands.map((command) => command.args[0])).toContain("lint:ui:i18n"); + expect(shouldRunControlUiI18nVerify(["scripts/lib/example.ts"])).toBe(false); + }); + it.each([ ["test/vitest/foo.config.ts", true, true], ["test/vitest/vitest-runtime-helper.d.mts", true, true], diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index dc0a0665df30..9d75d32ee356 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -930,9 +930,24 @@ describe("ci workflow guards", () => { ); expect(controlUiRefreshStep.env.OPENAI_API_KEY).toBe("${{ secrets.OPENAI_API_KEY }}"); expect(controlUiRefreshStep.env.OPENCLAW_CONTROL_UI_I18N_AUTH_OPTIONAL).toBe("0"); + const controlUiArtifactStep = controlUiWorkflow.jobs.refresh.steps.find( + (step: { name?: string }) => step.name === "Prepare locale artifact", + ); + expect(controlUiArtifactStep.run).toContain( + ":(exclude)ui/src/i18n/.i18n/catalog-fallbacks.json", + ); expect(controlUiAggregateStep.run).toBe( "node --import tsx scripts/control-ui-i18n.ts sync --write", ); + const controlUiPublishStep = controlUiFinalize.steps.find( + (step: { name?: string }) => step.name === "Open or update generated locale PR", + ); + expect(controlUiPublishStep.with["invalidation-paths"]).toContain( + "scripts/control-ui-i18n-verify.ts", + ); + expect(controlUiPublishStep.with["invalidation-paths"]).toContain( + "scripts/lib/control-ui-i18n-raw-copy.ts", + ); expect(controlUiFinalize.steps.indexOf(controlUiAggregateStep)).toBeLessThan( controlUiFinalize.steps.indexOf(controlUiValidationStep), ); diff --git a/test/scripts/control-ui-i18n.test.ts b/test/scripts/control-ui-i18n.test.ts index a3ff77dd84b0..107c22eef940 100644 --- a/test/scripts/control-ui-i18n.test.ts +++ b/test/scripts/control-ui-i18n.test.ts @@ -1,10 +1,16 @@ // Control Ui I18N tests cover control ui i18n script behavior. -import { spawn } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import { readFileSync, readdirSync, writeFileSync } from "node:fs"; import path from "node:path"; import process from "node:process"; import { pathToFileURL } from "node:url"; +import * as ts from "typescript"; import { describe, expect, it } from "vitest"; +import { + analyzeControlUiCatalogs, + assertScopedCatalogFallbackUpdate, + flattenControlUiCatalog, +} from "../../scripts/control-ui-i18n-verify.ts"; import { appendBoundedProcessOutput, buildBatchPrompt, @@ -12,6 +18,7 @@ import { runProcess, shouldReuseExistingTranslation, } from "../../scripts/control-ui-i18n.ts"; +import { collectControlUiRawCopyFromSource } from "../../scripts/lib/control-ui-i18n-raw-copy.ts"; import { createTempDirTracker } from "../helpers/temp-dir.js"; function processIsAlive(pid: number): boolean { @@ -52,6 +59,126 @@ async function waitForChildClose( } describe("control-ui-i18n process runner", () => { + it("builds a deterministic fallback list without accepting catalog drift", () => { + const source = flattenControlUiCatalog( + { group: { first: "First {count}", second: "Second" } }, + "en", + ); + const missingAnalysis = analyzeControlUiCatalogs( + source, + new Map([ + ["de", new Map([["group.first", "Erste {count}"]])], + ["fr", new Map([["group.first", "Premiere {count}"]])], + ]), + ); + + expect(missingAnalysis).toEqual({ + errors: [], + fallbacks: { "group.second": ["de", "fr"] }, + }); + + const driftAnalysis = analyzeControlUiCatalogs( + source, + new Map([ + [ + "fr", + new Map([ + ["group.second", "Deuxieme"], + ["group.first", "Premiere"], + ["group.orphan", "Orpheline"], + ]), + ], + ]), + ); + expect(driftAnalysis.errors).toEqual([ + "fr: orphan keys: group.orphan", + "fr: keys are not in English catalog order", + "fr:group.first expected {count} got {}", + ]); + expect(driftAnalysis.fallbacks).toEqual({}); + }); + + it("rejects invalid catalog leaf values", () => { + expect(() => flattenControlUiCatalog({ group: { title: 42 } }, "fr")).toThrow( + "fr:group.title must be a string or object", + ); + }); + + it("allows scoped sync to remove only that locale's approved fallbacks", () => { + const current = { + fallbacks: { "group.second": ["de", "fr"] }, + sourceHash: "source", + version: 1, + }; + + expect(() => + assertScopedCatalogFallbackUpdate( + current, + { ...current, fallbacks: { "group.second": ["fr"] } }, + "de", + ), + ).not.toThrow(); + expect(() => + assertScopedCatalogFallbackUpdate( + current, + { ...current, fallbacks: { "group.second": ["de"] } }, + "de", + ), + ).toThrow("unrelated catalog fallback drift"); + expect(() => + assertScopedCatalogFallbackUpdate( + current, + { ...current, fallbacks: { "group.second": ["de", "es", "fr"] } }, + "de", + ), + ).toThrow("unrelated catalog fallback drift"); + }); + + it("finds raw text and attributes split by template interpolation", () => { + const source = + 'const jsx = `;'; + const sourceFile = ts.createSourceFile( + "ui/src/pages/example.ts", + source, + ts.ScriptTarget.Latest, + true, + ts.ScriptKind.TSX, + ); + + expect( + collectControlUiRawCopyFromSource({ + filePath: path.resolve("ui/src/pages/example.ts"), + source, + sourceFile, + }).map(({ kind, text }) => ({ kind, text })), + ).toEqual([ + { kind: "html-attribute", text: "Archive" }, + { kind: "html-attribute", text: "Delete" }, + { kind: "html-text", text: "Delete" }, + ]); + }); + + it("keeps verification keyless even when provider credentials exist", () => { + const result = spawnSync( + process.execPath, + ["--import", "tsx", "scripts/control-ui-i18n-verify.ts", "verify"], + { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + ANTHROPIC_API_KEY: "redacted", + OPENAI_API_KEY: "redacted", + }, + }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("catalog:"); + expect(result.stdout).not.toContain("provider=openai"); + expect(result.stdout).not.toContain("provider=anthropic"); + }); + it("rejects placeholder-corrupt batch replies before they leave the retry loop", () => { const items = [ { diff --git a/test/scripts/run-oxlint.test.ts b/test/scripts/run-oxlint.test.ts index ff301f2164ca..4fedd2adb1fb 100644 --- a/test/scripts/run-oxlint.test.ts +++ b/test/scripts/run-oxlint.test.ts @@ -69,7 +69,9 @@ describe("run-oxlint", () => { const shardedLintRunner = readFileSync("scripts/run-oxlint-shards.mjs", "utf8"); expect(packageJson.scripts.check).toBe("node scripts/check.mjs"); - expect(packageJson.scripts.lint).toBe("node scripts/run-oxlint-shards.mjs"); + expect(packageJson.scripts.lint).toBe( + "pnpm lint:ui:i18n && node scripts/run-oxlint-shards.mjs", + ); expect(packageJson.scripts["lint:core"]).toBe( "node scripts/run-oxlint-shards.mjs --only=core --split-core", ); diff --git a/ui/AGENTS.md b/ui/AGENTS.md index ee93336a385b..fde791b9cb25 100644 --- a/ui/AGENTS.md +++ b/ui/AGENTS.md @@ -10,7 +10,9 @@ This directory owns Control UI-specific guidance that should not live in the rep - `scripts/control-ui-i18n.ts` - `ui/src/i18n/lib/types.ts` - `ui/src/i18n/lib/registry.ts` -- Pipeline: update English strings and locale wiring here and commit `en.ts` only; the `control-ui-locale-refresh` workflow translates and opens a generated PR after merge. Do not run `pnpm ui:i18n:sync` without provider auth (`ANTHROPIC_API_KEY`/`OPENAI_API_KEY`): CI rejects recorded English fallbacks, and sync fails closed on new untranslated keys. Run authenticated sync (and commit bundles plus `.i18n` metadata) only when the change needs it immediately, e.g. a raw-copy-baseline refresh after moving files with hardcoded strings. +- Contributor flow: update English strings and locale wiring, run keyless `pnpm ui:i18n:baseline`, and commit `en.ts` plus changed baseline files. The command records intentional runtime fallbacks without rewriting foreign-language bundles. +- `pnpm ui:i18n:verify` is deterministic and keyless. `pnpm lint` and the changed-check UI lane run it. It validates catalog shape, English-key coverage or explicit fallback listing, orphan keys, placeholder parity, canonical ordering, runtime locale wiring, and raw-copy baseline drift. +- Translation flow: the `control-ui-locale-refresh` workflow translates after merge and opens a generated PR. `pnpm ui:i18n:sync` remains the authenticated maintainer path; do not run it without provider auth when new keys exist. `pnpm ui:i18n:check` is the strict generated-output/release gate. - Prioritization report: `pnpm ui:i18n:report [--surface ] [--locale ] [--top ]` shows current hardcoded-copy focus areas and locale fallback metadata. It is not a drift gate; use `pnpm ui:i18n:check` for that. - If locale outputs drift, regenerate them. Do not manually translate or hand-maintain generated locale files by default. diff --git a/ui/src/i18n/.i18n/catalog-fallbacks.json b/ui/src/i18n/.i18n/catalog-fallbacks.json new file mode 100644 index 000000000000..6fea79701039 --- /dev/null +++ b/ui/src/i18n/.i18n/catalog-fallbacks.json @@ -0,0 +1,5 @@ +{ + "fallbacks": {}, + "sourceHash": "622f10085ef94b6f07d88f35cf370b2ce3108dbb36897da543cfc3b82ccd2240", + "version": 1 +} diff --git a/ui/src/i18n/.i18n/raw-copy-baseline.json b/ui/src/i18n/.i18n/raw-copy-baseline.json index 219e494caaca..92dccc9c7196 100644 --- a/ui/src/i18n/.i18n/raw-copy-baseline.json +++ b/ui/src/i18n/.i18n/raw-copy-baseline.json @@ -78,6 +78,13 @@ "path": "ui/src/components/lobster-pet.ts", "text": "Z" }, + { + "count": 1, + "kind": "html-attribute", + "name": "placeholder", + "path": "ui/src/components/login-gate.ts", + "text": "OPENCLAW_GATEWAY_TOKEN (" + }, { "count": 1, "kind": "html-attribute", @@ -99,6 +106,13 @@ "path": "ui/src/components/settings-sidebar.ts", "text": "esc" }, + { + "count": 1, + "kind": "html-text", + "name": "text", + "path": "ui/src/components/sidebar-update-card.ts", + "text": "v" + }, { "count": 1, "kind": "object-property", @@ -288,6 +302,41 @@ "path": "ui/src/lib/chat/commands.ts", "text": "tools" }, + { + "count": 1, + "kind": "html-text", + "name": "text", + "path": "ui/src/pages/about/view.ts", + "text": "v" + }, + { + "count": 1, + "kind": "html-text", + "name": "text", + "path": "ui/src/pages/agents/memory/view.ts", + "text": "assistant" + }, + { + "count": 1, + "kind": "html-text", + "name": "text", + "path": "ui/src/pages/agents/memory/view.ts", + "text": "chats" + }, + { + "count": 1, + "kind": "html-text", + "name": "text", + "path": "ui/src/pages/agents/memory/view.ts", + "text": "Full vault breakdown:" + }, + { + "count": 1, + "kind": "html-text", + "name": "text", + "path": "ui/src/pages/agents/memory/view.ts", + "text": "Imported chats clustered around" + }, { "count": 1, "kind": "html-text", @@ -302,6 +351,34 @@ "path": "ui/src/pages/agents/memory/view.ts", "text": "plugins.entries.memory-wiki.enabled = true" }, + { + "count": 1, + "kind": "html-text", + "name": "text", + "path": "ui/src/pages/agents/memory/view.ts", + "text": "Selected section:" + }, + { + "count": 1, + "kind": "html-text", + "name": "text", + "path": "ui/src/pages/agents/memory/view.ts", + "text": "Showing the first chunk of this page" + }, + { + "count": 1, + "kind": "html-text", + "name": "text", + "path": "ui/src/pages/agents/memory/view.ts", + "text": "user ·" + }, + { + "count": 1, + "kind": "html-text", + "name": "text", + "path": "ui/src/pages/agents/memory/view.ts", + "text": "Vault ·" + }, { "count": 2, "kind": "html-text", @@ -365,6 +442,20 @@ "path": "ui/src/pages/channels/view.nostr.ts", "text": "NIP-05" }, + { + "count": 1, + "kind": "html-text", + "name": "text", + "path": "ui/src/pages/chat/components/chat-composer.ts", + "text": "Compacting context..." + }, + { + "count": 1, + "kind": "html-text", + "name": "text", + "path": "ui/src/pages/chat/components/chat-composer.ts", + "text": "Context compacted" + }, { "count": 2, "kind": "html-text", @@ -379,6 +470,13 @@ "path": "ui/src/pages/chat/components/chat-composer.ts", "text": "Esc" }, + { + "count": 1, + "kind": "html-text", + "name": "text", + "path": "ui/src/pages/chat/components/chat-composer.ts", + "text": "Replying to" + }, { "count": 2, "kind": "html-text", @@ -386,6 +484,13 @@ "path": "ui/src/pages/chat/components/chat-composer.ts", "text": "Tab" }, + { + "count": 1, + "kind": "html-text", + "name": "text", + "path": "ui/src/pages/chat/components/chat-message.ts", + "text": "% ctx" + }, { "count": 2, "kind": "html-text", @@ -393,6 +498,27 @@ "path": "ui/src/pages/chat/components/chat-message.ts", "text": "JSON" }, + { + "count": 1, + "kind": "html-text", + "name": "text", + "path": "ui/src/pages/chat/components/chat-message.ts", + "text": "R" + }, + { + "count": 1, + "kind": "html-text", + "name": "text", + "path": "ui/src/pages/chat/components/chat-message.ts", + "text": "W" + }, + { + "count": 1, + "kind": "html-text", + "name": "text", + "path": "ui/src/pages/chat/components/chat-thread.ts", + "text": "pinned" + }, { "count": 1, "kind": "html-text", @@ -442,6 +568,13 @@ "path": "ui/src/pages/connection/view.ts", "text": "OPENCLAW_GATEWAY_TOKEN" }, + { + "count": 1, + "kind": "html-text", + "name": "text", + "path": "ui/src/pages/cron/view-runs.ts", + "text": "ms" + }, { "count": 1, "kind": "html-attribute", @@ -463,6 +596,13 @@ "path": "ui/src/pages/debug/view.ts", "text": "openclaw security audit --deep" }, + { + "count": 1, + "kind": "html-text", + "name": "text", + "path": "ui/src/pages/memory-import/view.ts", + "text": "/memory/imports/" + }, { "count": 1, "kind": "html-text", @@ -484,6 +624,13 @@ "path": "ui/src/pages/plugins/view.ts", "text": "https://mcp.example.com/mcp · npx some-mcp-server" }, + { + "count": 4, + "kind": "html-text", + "name": "text", + "path": "ui/src/pages/plugins/view.ts", + "text": "v" + }, { "count": 1, "kind": "html-text", @@ -491,6 +638,13 @@ "path": "ui/src/pages/profile/profile-page.ts", "text": "OpenClaw" }, + { + "count": 2, + "kind": "html-text", + "name": "text", + "path": "ui/src/pages/skill-workshop/view.ts", + "text": "v" + }, { "count": 1, "kind": "object-property",