mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
refactor(i18n): extract locale sync planning (#104943)
This commit is contained in:
Vincent Koc
+69
-331
@@ -8,9 +8,22 @@ import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { completeSimple, type AssistantMessage, type Model } from "openclaw/plugin-sdk/llm";
|
||||
import * as ts from "typescript";
|
||||
import { formatErrorMessage } from "../src/infra/errors.ts";
|
||||
import {
|
||||
compareStringArrays,
|
||||
createControlUiLocaleSyncPlan,
|
||||
flattenTranslations,
|
||||
type GlossaryEntry,
|
||||
type LocaleEntry,
|
||||
type LocaleMeta,
|
||||
type TranslationBatchItem,
|
||||
type TranslationMap,
|
||||
type TranslationMemoryEntry,
|
||||
} from "./lib/control-ui-i18n-sync-plan.ts";
|
||||
import { sleep } from "./lib/sleep.mjs";
|
||||
import { resolveWindowsTaskkillPath } from "./lib/windows-taskkill.mjs";
|
||||
|
||||
export { shouldReuseExistingTranslation } from "./lib/control-ui-i18n-sync-plan.ts";
|
||||
|
||||
const { formatGeneratedModule } = (await import(
|
||||
new URL("./lib/format-generated-module.mjs", import.meta.url).href
|
||||
)) as {
|
||||
@@ -24,62 +37,11 @@ const { formatGeneratedModule } = (await import(
|
||||
) => string;
|
||||
};
|
||||
|
||||
interface TranslationMap {
|
||||
[key: string]: string | TranslationMap;
|
||||
}
|
||||
|
||||
type TranslationValue = string | { [key: string]: TranslationValue };
|
||||
|
||||
type LocaleEntry = {
|
||||
exportName: string;
|
||||
fileName: string;
|
||||
languageKey: string;
|
||||
locale: string;
|
||||
};
|
||||
|
||||
type GlossaryEntry = {
|
||||
source: string;
|
||||
target: string;
|
||||
};
|
||||
|
||||
type RunProcessParentSignalState = {
|
||||
done: boolean;
|
||||
signal: NodeJS.Signals | null;
|
||||
};
|
||||
|
||||
type TranslationMemoryEntry = {
|
||||
cache_key: string;
|
||||
model: string;
|
||||
provider: string;
|
||||
segment_id: string;
|
||||
source_path: string;
|
||||
src_lang: string;
|
||||
text: string;
|
||||
text_hash: string;
|
||||
tgt_lang: string;
|
||||
translated: string;
|
||||
updated_at: string;
|
||||
};
|
||||
|
||||
type LocaleMeta = {
|
||||
fallbackKeys: string[];
|
||||
generatedAt: string;
|
||||
locale: string;
|
||||
model: string;
|
||||
provider: string;
|
||||
sourceHash: string;
|
||||
totalKeys: number;
|
||||
translatedKeys: number;
|
||||
workflow: number;
|
||||
};
|
||||
|
||||
type TranslationBatchItem = {
|
||||
cacheKey: string;
|
||||
key: string;
|
||||
text: string;
|
||||
textHash: string;
|
||||
};
|
||||
|
||||
type RawCopyFinding = {
|
||||
kind: "html-attribute" | "html-text" | "object-property";
|
||||
line: number;
|
||||
@@ -390,42 +352,6 @@ async function loadLocaleMap(filePath: string, exportName: string): Promise<Tran
|
||||
return mod[exportName] ?? null;
|
||||
}
|
||||
|
||||
function flattenTranslations(value: TranslationMap, prefix = "", out = new Map<string, string>()) {
|
||||
for (const [key, nested] of Object.entries(value)) {
|
||||
const fullKey = prefix ? `${prefix}.${key}` : key;
|
||||
if (typeof nested === "string") {
|
||||
out.set(fullKey, nested);
|
||||
continue;
|
||||
}
|
||||
flattenTranslations(nested, fullKey, out);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function setNestedValue(root: TranslationMap, dottedKey: string, value: string) {
|
||||
const parts = dottedKey.split(".");
|
||||
let cursor: TranslationMap = root;
|
||||
for (let index = 0; index < parts.length - 1; index += 1) {
|
||||
const key = parts[index];
|
||||
const next = cursor[key];
|
||||
if (!next || typeof next === "string") {
|
||||
const replacement: TranslationMap = {};
|
||||
cursor[key] = replacement;
|
||||
cursor = replacement;
|
||||
continue;
|
||||
}
|
||||
cursor = next;
|
||||
}
|
||||
cursor[parts.at(-1)!] = value;
|
||||
}
|
||||
|
||||
function compareStringArrays(left: string[], right: string[]) {
|
||||
if (left.length !== right.length) {
|
||||
return false;
|
||||
}
|
||||
return left.every((value, index) => value === right[index]);
|
||||
}
|
||||
|
||||
type PlaceholderMismatch = {
|
||||
key: string;
|
||||
locale: string;
|
||||
@@ -488,41 +414,6 @@ function assertPlaceholderParity(
|
||||
);
|
||||
}
|
||||
|
||||
function isIdentifier(value: string): boolean {
|
||||
return /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(value);
|
||||
}
|
||||
|
||||
function renderTranslationValue(value: TranslationValue, indent = 0): string {
|
||||
if (typeof value === "string") {
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
const entries = Object.entries(value);
|
||||
if (entries.length === 0) {
|
||||
return "{}";
|
||||
}
|
||||
|
||||
const pad = " ".repeat(indent);
|
||||
const innerPad = " ".repeat(indent + 1);
|
||||
return `{\n${entries
|
||||
.map(([key, nested]) => {
|
||||
const renderedKey = isIdentifier(key) ? key : JSON.stringify(key);
|
||||
return `${innerPad}${renderedKey}: ${renderTranslationValue(nested, indent + 1)},`;
|
||||
})
|
||||
.join("\n")}\n${pad}}`;
|
||||
}
|
||||
|
||||
function renderLocaleModule(entry: LocaleEntry, value: TranslationMap): string {
|
||||
return [
|
||||
"// Generated locale bundle for Control UI translations.",
|
||||
"// Run `pnpm ui:i18n:sync` instead of editing this file directly.",
|
||||
'import type { TranslationMap } from "../lib/types.ts";',
|
||||
"",
|
||||
`export const ${entry.exportName}: TranslationMap = ${renderTranslationValue(value)};`,
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
async function loadGlossary(filePath: string): Promise<GlossaryEntry[]> {
|
||||
if (!existsSync(filePath)) {
|
||||
return [];
|
||||
@@ -532,10 +423,6 @@ async function loadGlossary(filePath: string): Promise<GlossaryEntry[]> {
|
||||
return Array.isArray(parsed) ? parsed : [];
|
||||
}
|
||||
|
||||
function renderGlossary(entries: readonly GlossaryEntry[]): string {
|
||||
return `${JSON.stringify(entries, null, 2)}\n`;
|
||||
}
|
||||
|
||||
async function loadMeta(filePath: string): Promise<LocaleMeta | null> {
|
||||
if (!existsSync(filePath)) {
|
||||
return null;
|
||||
@@ -544,10 +431,6 @@ async function loadMeta(filePath: string): Promise<LocaleMeta | null> {
|
||||
return JSON.parse(raw) as LocaleMeta;
|
||||
}
|
||||
|
||||
function renderMeta(meta: LocaleMeta): string {
|
||||
return `${JSON.stringify(meta, null, 2)}\n`;
|
||||
}
|
||||
|
||||
async function loadTranslationMemory(
|
||||
filePath: string,
|
||||
): Promise<Map<string, TranslationMemoryEntry>> {
|
||||
@@ -569,30 +452,6 @@ async function loadTranslationMemory(
|
||||
return entries;
|
||||
}
|
||||
|
||||
function renderTranslationMemory(entries: Map<string, TranslationMemoryEntry>): string {
|
||||
const ordered = [...entries.values()].toSorted((left, right) =>
|
||||
left.cache_key.localeCompare(right.cache_key),
|
||||
);
|
||||
if (ordered.length === 0) {
|
||||
return "";
|
||||
}
|
||||
return `${ordered.map((entry) => JSON.stringify(entry)).join("\n")}\n`;
|
||||
}
|
||||
|
||||
function buildTranslationMemoryByTextHash(
|
||||
entries: Map<string, TranslationMemoryEntry>,
|
||||
locale: string,
|
||||
): Map<string, TranslationMemoryEntry> {
|
||||
const byTextHash = new Map<string, TranslationMemoryEntry>();
|
||||
for (const entry of entries.values()) {
|
||||
if (entry.tgt_lang !== locale || !entry.text_hash || !entry.translated.trim()) {
|
||||
continue;
|
||||
}
|
||||
byTextHash.set(entry.text_hash, entry);
|
||||
}
|
||||
return byTextHash;
|
||||
}
|
||||
|
||||
function buildGlossaryPrompt(glossary: readonly GlossaryEntry[]): string {
|
||||
if (glossary.length === 0) {
|
||||
return "";
|
||||
@@ -1301,6 +1160,23 @@ type ClientAccess = {
|
||||
resetClient: () => Promise<void>;
|
||||
};
|
||||
|
||||
function createTranslationClientAccess(
|
||||
targetLocale: string,
|
||||
glossary: readonly GlossaryEntry[],
|
||||
): ClientAccess {
|
||||
let client: TranslationClient | null = null;
|
||||
return {
|
||||
async getClient() {
|
||||
client ??= await TranslationClient.create(buildSystemPrompt(targetLocale, glossary));
|
||||
return client;
|
||||
},
|
||||
async resetClient() {
|
||||
await client?.close();
|
||||
client = null;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function formatLocaleLabel(locale: string, context: LocaleRunContext): string {
|
||||
return `[${context.localeIndex}/${context.localeCount}] ${locale}`;
|
||||
}
|
||||
@@ -1526,22 +1402,7 @@ export async function translateNativeEntries(
|
||||
textHash: hashText(entry.source),
|
||||
}));
|
||||
const batches = buildTranslationBatches(pending);
|
||||
let client: TranslationClient | null = null;
|
||||
const clientAccess: ClientAccess = {
|
||||
async getClient() {
|
||||
if (!client) {
|
||||
client = await TranslationClient.create(buildSystemPrompt(targetLocale, glossary));
|
||||
}
|
||||
return client;
|
||||
},
|
||||
async resetClient() {
|
||||
if (!client) {
|
||||
return;
|
||||
}
|
||||
await client.close();
|
||||
client = null;
|
||||
},
|
||||
};
|
||||
const clientAccess = createTranslationClientAccess(targetLocale, glossary);
|
||||
try {
|
||||
const translated = new Map<string, string>();
|
||||
for (const [batchIndex, batch] of batches.entries()) {
|
||||
@@ -1569,14 +1430,6 @@ type SyncOutcome = {
|
||||
wrote: boolean;
|
||||
};
|
||||
|
||||
export function shouldReuseExistingTranslation(options: {
|
||||
allowTranslate: boolean;
|
||||
force: boolean;
|
||||
isFallback: boolean;
|
||||
}): boolean {
|
||||
return !options.isFallback || (!options.allowTranslate && !options.force);
|
||||
}
|
||||
|
||||
async function syncLocale(
|
||||
entry: LocaleEntry,
|
||||
options: { checkOnly: boolean; force: boolean; write: boolean },
|
||||
@@ -1592,64 +1445,22 @@ async function syncLocale(
|
||||
const existingMap = (await loadLocaleMap(existingPath, entry.exportName)) ?? {};
|
||||
const existingFlat = flattenTranslations(existingMap);
|
||||
const previousMeta = await loadMeta(metaPath(entry));
|
||||
const previousFallbackKeys = new Set(previousMeta?.fallbackKeys ?? []);
|
||||
const glossaryFilePath = glossaryPath(entry);
|
||||
const glossary = await loadGlossary(glossaryFilePath);
|
||||
const tm = await loadTranslationMemory(tmPath(entry));
|
||||
const tmByTextHash = buildTranslationMemoryByTextHash(tm, entry.locale);
|
||||
const allowTranslate = hasTranslationProvider();
|
||||
|
||||
const nextFlat = new Map<string, string>();
|
||||
const pending: TranslationBatchItem[] = [];
|
||||
const fallbackKeys: string[] = [];
|
||||
|
||||
for (const [key, text] of sourceFlat.entries()) {
|
||||
const textHash = hashText(text);
|
||||
const segmentCacheKey = cacheKey(key, textHash, entry.locale);
|
||||
const cached = tm.get(segmentCacheKey);
|
||||
const cachedByText = tmByTextHash.get(textHash);
|
||||
const existing = existingFlat.get(key);
|
||||
const shouldRefreshFallback = previousFallbackKeys.has(key);
|
||||
const shouldReuse = shouldReuseExistingTranslation({
|
||||
allowTranslate,
|
||||
force: options.force,
|
||||
isFallback: shouldRefreshFallback,
|
||||
});
|
||||
|
||||
if (cached && shouldReuse) {
|
||||
nextFlat.set(key, cached.translated);
|
||||
if (shouldRefreshFallback) {
|
||||
fallbackKeys.push(key);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (cachedByText && (shouldRefreshFallback || existing === undefined)) {
|
||||
nextFlat.set(key, cachedByText.translated);
|
||||
tm.set(segmentCacheKey, {
|
||||
...cachedByText,
|
||||
cache_key: segmentCacheKey,
|
||||
segment_id: key,
|
||||
source_path: `ui/src/i18n/locales/${entry.fileName}`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (existing !== undefined && shouldReuse) {
|
||||
nextFlat.set(key, existing);
|
||||
if (shouldRefreshFallback) {
|
||||
fallbackKeys.push(key);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
pending.push({
|
||||
cacheKey: segmentCacheKey,
|
||||
key,
|
||||
text,
|
||||
textHash,
|
||||
});
|
||||
}
|
||||
const plan = createControlUiLocaleSyncPlan({
|
||||
allowTranslate,
|
||||
cacheKeyFor: (key, textHash) => cacheKey(key, textHash, entry.locale),
|
||||
entry,
|
||||
existingFlat,
|
||||
force: options.force,
|
||||
hashText,
|
||||
previousMeta,
|
||||
sourceFlat,
|
||||
sourceHash,
|
||||
translationMemory: tm,
|
||||
});
|
||||
|
||||
// Writing NEW English fallbacks trips the shipped-fallback CI gate
|
||||
// (test/scripts/control-ui-i18n.test.ts), and post-merge translation is owned
|
||||
@@ -1657,10 +1468,9 @@ async function syncLocale(
|
||||
// must fail here instead of silently recording fallback bundles; refreshing
|
||||
// already-recorded fallback copy (force mode) stays allowed.
|
||||
if (!allowTranslate && options.write && !options.checkOnly && !isProviderAuthOptional()) {
|
||||
const newFallbackKeys = pending.filter((item) => !previousFallbackKeys.has(item.key));
|
||||
if (newFallbackKeys.length > 0) {
|
||||
if (plan.newFallbackCount > 0) {
|
||||
throw new Error(
|
||||
`${localeLabel}: ${newFallbackKeys.length} new key(s) need translation but no provider is configured. ` +
|
||||
`${localeLabel}: ${plan.newFallbackCount} new key(s) need translation but no provider is configured. ` +
|
||||
`Commit only locales/en.ts and let the control-ui-locale-refresh workflow translate after merge, ` +
|
||||
`or export ANTHROPIC_API_KEY/OPENAI_API_KEY and rerun. ` +
|
||||
`Set ${ENV_AUTH_OPTIONAL}=1 to record English fallbacks anyway.`,
|
||||
@@ -1668,28 +1478,13 @@ async function syncLocale(
|
||||
}
|
||||
}
|
||||
|
||||
if (allowTranslate && pending.length > 0) {
|
||||
const batches = buildTranslationBatches(pending);
|
||||
if (allowTranslate && plan.pending.length > 0) {
|
||||
const batches = buildTranslationBatches(plan.pending);
|
||||
const batchCount = batches.length;
|
||||
logProgress(
|
||||
`${localeLabel}: start keys=${sourceFlat.size} pending=${pending.length} batches=${batchCount} provider=${resolveConfiguredProvider()} model=${resolveConfiguredModel()} thinking=${resolveThinkingLevel()} timeout=${formatDuration(resolvePromptTimeoutMs())} batch_chars=${resolveBatchCharBudget()}`,
|
||||
`${localeLabel}: start keys=${sourceFlat.size} pending=${plan.pending.length} batches=${batchCount} provider=${resolveConfiguredProvider()} model=${resolveConfiguredModel()} thinking=${resolveThinkingLevel()} timeout=${formatDuration(resolvePromptTimeoutMs())} batch_chars=${resolveBatchCharBudget()}`,
|
||||
);
|
||||
let client: TranslationClient | null = null;
|
||||
const clientAccess: ClientAccess = {
|
||||
async getClient() {
|
||||
if (!client) {
|
||||
client = await TranslationClient.create(buildSystemPrompt(entry.locale, glossary));
|
||||
}
|
||||
return client;
|
||||
},
|
||||
async resetClient() {
|
||||
if (!client) {
|
||||
return;
|
||||
}
|
||||
await client.close();
|
||||
client = null;
|
||||
},
|
||||
};
|
||||
const clientAccess = createTranslationClientAccess(entry.locale, glossary);
|
||||
try {
|
||||
for (const [batchIndex, batch] of batches.entries()) {
|
||||
const translated = await translateBatch(clientAccess, batch, {
|
||||
@@ -1698,26 +1493,12 @@ async function syncLocale(
|
||||
batchIndex: batchIndex + 1,
|
||||
locale: entry.locale,
|
||||
});
|
||||
for (const item of batch) {
|
||||
const value = translated.get(item.key);
|
||||
if (!value) {
|
||||
continue;
|
||||
}
|
||||
nextFlat.set(item.key, value);
|
||||
tm.set(item.cacheKey, {
|
||||
cache_key: item.cacheKey,
|
||||
model: resolveConfiguredModel(),
|
||||
provider: resolveConfiguredProvider(),
|
||||
segment_id: item.key,
|
||||
source_path: `ui/src/i18n/locales/${entry.fileName}`,
|
||||
src_lang: SOURCE_LOCALE,
|
||||
text: item.text,
|
||||
text_hash: item.textHash,
|
||||
tgt_lang: entry.locale,
|
||||
translated: value,
|
||||
updated_at: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
plan.recordTranslations(batch, translated, {
|
||||
model: resolveConfiguredModel(),
|
||||
provider: resolveConfiguredProvider(),
|
||||
sourceLocale: SOURCE_LOCALE,
|
||||
updatedAt: () => new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
const failure = error instanceof Error ? error : new Error(String(error));
|
||||
@@ -1742,72 +1523,29 @@ async function syncLocale(
|
||||
logProgress(`${localeLabel}: no provider configured, using English fallback for pending keys`);
|
||||
}
|
||||
|
||||
for (const item of pending) {
|
||||
if (nextFlat.has(item.key)) {
|
||||
continue;
|
||||
}
|
||||
const existing = existingFlat.get(item.key);
|
||||
if (existing !== undefined && !options.force) {
|
||||
nextFlat.set(item.key, existing);
|
||||
if (previousFallbackKeys.has(item.key)) {
|
||||
fallbackKeys.push(item.key);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
nextFlat.set(item.key, item.text);
|
||||
fallbackKeys.push(item.key);
|
||||
}
|
||||
|
||||
// Do not infer fallback state from source-text equality alone.
|
||||
// Product names, config keys, and other intentional carry-through strings may
|
||||
// legitimately stay identical to English. Track fallback keys from actual
|
||||
// fallback decisions and previous fallback metadata instead.
|
||||
|
||||
assertPlaceholderParity(sourceFlat, nextFlat, entry.locale);
|
||||
|
||||
const nextMap: TranslationMap = {};
|
||||
for (const [key, value] of sourceFlat.entries()) {
|
||||
setNestedValue(nextMap, key, nextFlat.get(key) ?? value);
|
||||
}
|
||||
|
||||
const nextProvider = allowTranslate
|
||||
? resolveConfiguredProvider()
|
||||
: (previousMeta?.provider ?? "");
|
||||
const nextModel = allowTranslate ? resolveConfiguredModel() : (previousMeta?.model ?? "");
|
||||
const sortedFallbackKeys = [...new Set(fallbackKeys)].toSorted((left, right) =>
|
||||
left.localeCompare(right),
|
||||
);
|
||||
const translatedKeys = sourceFlat.size - sortedFallbackKeys.length;
|
||||
const semanticMetaChanged =
|
||||
!previousMeta ||
|
||||
previousMeta.locale !== entry.locale ||
|
||||
previousMeta.sourceHash !== sourceHash ||
|
||||
previousMeta.provider !== nextProvider ||
|
||||
previousMeta.model !== nextModel ||
|
||||
previousMeta.totalKeys !== sourceFlat.size ||
|
||||
previousMeta.translatedKeys !== translatedKeys ||
|
||||
previousMeta.workflow !== CONTROL_UI_I18N_WORKFLOW ||
|
||||
!compareStringArrays(previousMeta.fallbackKeys, sortedFallbackKeys);
|
||||
|
||||
const nextMeta: LocaleMeta = {
|
||||
fallbackKeys: sortedFallbackKeys,
|
||||
generatedAt: semanticMetaChanged ? new Date().toISOString() : previousMeta.generatedAt,
|
||||
locale: entry.locale,
|
||||
const artifacts = plan.render({
|
||||
defaultGlossary: DEFAULT_GLOSSARY,
|
||||
generatedAt: new Date().toISOString(),
|
||||
glossary,
|
||||
model: nextModel,
|
||||
provider: nextProvider,
|
||||
sourceHash,
|
||||
totalKeys: sourceFlat.size,
|
||||
translatedKeys,
|
||||
workflow: CONTROL_UI_I18N_WORKFLOW,
|
||||
};
|
||||
});
|
||||
assertPlaceholderParity(sourceFlat, artifacts.nextFlat, entry.locale);
|
||||
|
||||
const expectedLocale = await formatGeneratedTypeScript(
|
||||
existingPath,
|
||||
renderLocaleModule(entry, nextMap),
|
||||
);
|
||||
const expectedMeta = renderMeta(nextMeta);
|
||||
const expectedGlossary = renderGlossary(glossary.length === 0 ? DEFAULT_GLOSSARY : glossary);
|
||||
const expectedTm = renderTranslationMemory(tm);
|
||||
const expectedLocale = await formatGeneratedTypeScript(existingPath, artifacts.localeModule);
|
||||
const expectedMeta = artifacts.meta;
|
||||
const expectedGlossary = artifacts.glossary;
|
||||
const expectedTm = artifacts.translationMemory;
|
||||
|
||||
const currentLocale = existsSync(existingPath) ? await readFile(existingPath, "utf8") : "";
|
||||
const currentMeta = existsSync(metaPath(entry)) ? await readFile(metaPath(entry), "utf8") : "";
|
||||
@@ -1830,11 +1568,11 @@ async function syncLocale(
|
||||
!options.write)
|
||||
) {
|
||||
logProgress(
|
||||
`${localeLabel}: done changed=${changed} fallbacks=${nextMeta.fallbackKeys.length} elapsed=${formatDuration(Date.now() - localeStartedAt)}`,
|
||||
`${localeLabel}: done changed=${changed} fallbacks=${artifacts.fallbackCount} elapsed=${formatDuration(Date.now() - localeStartedAt)}`,
|
||||
);
|
||||
return {
|
||||
changed,
|
||||
fallbackCount: nextMeta.fallbackKeys.length,
|
||||
fallbackCount: artifacts.fallbackCount,
|
||||
locale: entry.locale,
|
||||
wrote: false,
|
||||
} satisfies SyncOutcome;
|
||||
@@ -1854,11 +1592,11 @@ async function syncLocale(
|
||||
}
|
||||
|
||||
logProgress(
|
||||
`${localeLabel}: done changed=${changed} fallbacks=${nextMeta.fallbackKeys.length} elapsed=${formatDuration(Date.now() - localeStartedAt)}${!options.checkOnly && options.write && changed ? " wrote" : ""}`,
|
||||
`${localeLabel}: done changed=${changed} fallbacks=${artifacts.fallbackCount} elapsed=${formatDuration(Date.now() - localeStartedAt)}${!options.checkOnly && options.write && changed ? " wrote" : ""}`,
|
||||
);
|
||||
return {
|
||||
changed,
|
||||
fallbackCount: nextMeta.fallbackKeys.length,
|
||||
fallbackCount: artifacts.fallbackCount,
|
||||
locale: entry.locale,
|
||||
wrote: !options.checkOnly && options.write && changed,
|
||||
} satisfies SyncOutcome;
|
||||
|
||||
@@ -0,0 +1,307 @@
|
||||
export interface TranslationMap {
|
||||
[key: string]: string | TranslationMap;
|
||||
}
|
||||
|
||||
export type LocaleEntry = {
|
||||
exportName: string;
|
||||
fileName: string;
|
||||
languageKey: string;
|
||||
locale: string;
|
||||
};
|
||||
|
||||
export type GlossaryEntry = {
|
||||
source: string;
|
||||
target: string;
|
||||
};
|
||||
|
||||
export type TranslationMemoryEntry = {
|
||||
cache_key: string;
|
||||
model: string;
|
||||
provider: string;
|
||||
segment_id: string;
|
||||
source_path: string;
|
||||
src_lang: string;
|
||||
text: string;
|
||||
text_hash: string;
|
||||
tgt_lang: string;
|
||||
translated: string;
|
||||
updated_at: string;
|
||||
};
|
||||
|
||||
export type LocaleMeta = {
|
||||
fallbackKeys: string[];
|
||||
generatedAt: string;
|
||||
locale: string;
|
||||
model: string;
|
||||
provider: string;
|
||||
sourceHash: string;
|
||||
totalKeys: number;
|
||||
translatedKeys: number;
|
||||
workflow: number;
|
||||
};
|
||||
|
||||
export type TranslationBatchItem = {
|
||||
cacheKey: string;
|
||||
key: string;
|
||||
text: string;
|
||||
textHash: string;
|
||||
};
|
||||
|
||||
export function flattenTranslations(
|
||||
value: TranslationMap,
|
||||
prefix = "",
|
||||
out = new Map<string, string>(),
|
||||
) {
|
||||
for (const [key, nested] of Object.entries(value)) {
|
||||
const fullKey = prefix ? `${prefix}.${key}` : key;
|
||||
if (typeof nested === "string") {
|
||||
out.set(fullKey, nested);
|
||||
continue;
|
||||
}
|
||||
flattenTranslations(nested, fullKey, out);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function shouldReuseExistingTranslation(options: {
|
||||
allowTranslate: boolean;
|
||||
force: boolean;
|
||||
isFallback: boolean;
|
||||
}): boolean {
|
||||
return !options.isFallback || (!options.allowTranslate && !options.force);
|
||||
}
|
||||
|
||||
export function createControlUiLocaleSyncPlan(input: {
|
||||
allowTranslate: boolean;
|
||||
cacheKeyFor: (key: string, textHash: string) => string;
|
||||
entry: LocaleEntry;
|
||||
existingFlat: ReadonlyMap<string, string>;
|
||||
force: boolean;
|
||||
hashText: (text: string) => string;
|
||||
previousMeta: LocaleMeta | null;
|
||||
sourceFlat: ReadonlyMap<string, string>;
|
||||
sourceHash: string;
|
||||
translationMemory: ReadonlyMap<string, TranslationMemoryEntry>;
|
||||
}) {
|
||||
const previousFallbackKeys = new Set(input.previousMeta?.fallbackKeys ?? []);
|
||||
const translationMemory = new Map(input.translationMemory);
|
||||
const translationMemoryByTextHash = new Map(
|
||||
[...translationMemory.values()]
|
||||
.filter(
|
||||
(entry) =>
|
||||
entry.tgt_lang === input.entry.locale && entry.text_hash && entry.translated.trim(),
|
||||
)
|
||||
.map((entry) => [entry.text_hash, entry]),
|
||||
);
|
||||
const nextFlat = new Map<string, string>();
|
||||
const pending: TranslationBatchItem[] = [];
|
||||
const fallbackKeys: string[] = [];
|
||||
|
||||
for (const [key, text] of input.sourceFlat.entries()) {
|
||||
const textHash = input.hashText(text);
|
||||
const segmentCacheKey = input.cacheKeyFor(key, textHash);
|
||||
const cached = translationMemory.get(segmentCacheKey);
|
||||
const cachedByText = translationMemoryByTextHash.get(textHash);
|
||||
const existing = input.existingFlat.get(key);
|
||||
const shouldRefreshFallback = previousFallbackKeys.has(key);
|
||||
const shouldReuse = shouldReuseExistingTranslation({
|
||||
allowTranslate: input.allowTranslate,
|
||||
force: input.force,
|
||||
isFallback: shouldRefreshFallback,
|
||||
});
|
||||
|
||||
if (cached && shouldReuse) {
|
||||
nextFlat.set(key, cached.translated);
|
||||
if (shouldRefreshFallback) {
|
||||
fallbackKeys.push(key);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (cachedByText && (shouldRefreshFallback || existing === undefined)) {
|
||||
nextFlat.set(key, cachedByText.translated);
|
||||
translationMemory.set(segmentCacheKey, {
|
||||
...cachedByText,
|
||||
cache_key: segmentCacheKey,
|
||||
segment_id: key,
|
||||
source_path: `ui/src/i18n/locales/${input.entry.fileName}`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (existing !== undefined && shouldReuse) {
|
||||
nextFlat.set(key, existing);
|
||||
if (shouldRefreshFallback) {
|
||||
fallbackKeys.push(key);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
pending.push({ cacheKey: segmentCacheKey, key, text, textHash });
|
||||
}
|
||||
|
||||
return {
|
||||
newFallbackCount: pending.filter((item) => !previousFallbackKeys.has(item.key)).length,
|
||||
pending,
|
||||
recordTranslations(
|
||||
batch: readonly TranslationBatchItem[],
|
||||
translated: ReadonlyMap<string, string>,
|
||||
metadata: {
|
||||
model: string;
|
||||
provider: string;
|
||||
sourceLocale: string;
|
||||
updatedAt: () => string;
|
||||
},
|
||||
): void {
|
||||
for (const item of batch) {
|
||||
const value = translated.get(item.key);
|
||||
if (!value) {
|
||||
continue;
|
||||
}
|
||||
nextFlat.set(item.key, value);
|
||||
translationMemory.set(item.cacheKey, {
|
||||
cache_key: item.cacheKey,
|
||||
model: metadata.model,
|
||||
provider: metadata.provider,
|
||||
segment_id: item.key,
|
||||
source_path: `ui/src/i18n/locales/${input.entry.fileName}`,
|
||||
src_lang: metadata.sourceLocale,
|
||||
text: item.text,
|
||||
text_hash: item.textHash,
|
||||
tgt_lang: input.entry.locale,
|
||||
translated: value,
|
||||
updated_at: metadata.updatedAt(),
|
||||
});
|
||||
}
|
||||
},
|
||||
render(options: {
|
||||
defaultGlossary: readonly GlossaryEntry[];
|
||||
generatedAt: string;
|
||||
glossary: readonly GlossaryEntry[];
|
||||
model: string;
|
||||
provider: string;
|
||||
workflow: number;
|
||||
}) {
|
||||
for (const item of pending) {
|
||||
if (nextFlat.has(item.key)) {
|
||||
continue;
|
||||
}
|
||||
const existing = input.existingFlat.get(item.key);
|
||||
if (existing !== undefined && !input.force) {
|
||||
nextFlat.set(item.key, existing);
|
||||
if (previousFallbackKeys.has(item.key)) {
|
||||
fallbackKeys.push(item.key);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
nextFlat.set(item.key, item.text);
|
||||
fallbackKeys.push(item.key);
|
||||
}
|
||||
|
||||
const sortedFallbackKeys = [...new Set(fallbackKeys)].toSorted((left, right) =>
|
||||
left.localeCompare(right),
|
||||
);
|
||||
const translatedKeys = input.sourceFlat.size - sortedFallbackKeys.length;
|
||||
const previousMeta = input.previousMeta;
|
||||
const semanticMetaChanged =
|
||||
!previousMeta ||
|
||||
previousMeta.locale !== input.entry.locale ||
|
||||
previousMeta.sourceHash !== input.sourceHash ||
|
||||
previousMeta.provider !== options.provider ||
|
||||
previousMeta.model !== options.model ||
|
||||
previousMeta.totalKeys !== input.sourceFlat.size ||
|
||||
previousMeta.translatedKeys !== translatedKeys ||
|
||||
previousMeta.workflow !== options.workflow ||
|
||||
!compareStringArrays(previousMeta.fallbackKeys, sortedFallbackKeys);
|
||||
const nextMeta: LocaleMeta = {
|
||||
fallbackKeys: sortedFallbackKeys,
|
||||
generatedAt: semanticMetaChanged ? options.generatedAt : previousMeta.generatedAt,
|
||||
locale: input.entry.locale,
|
||||
model: options.model,
|
||||
provider: options.provider,
|
||||
sourceHash: input.sourceHash,
|
||||
totalKeys: input.sourceFlat.size,
|
||||
translatedKeys,
|
||||
workflow: options.workflow,
|
||||
};
|
||||
const nextMap: TranslationMap = {};
|
||||
for (const [key, value] of input.sourceFlat.entries()) {
|
||||
setNestedValue(nextMap, key, nextFlat.get(key) ?? value);
|
||||
}
|
||||
|
||||
return {
|
||||
fallbackCount: sortedFallbackKeys.length,
|
||||
glossary: renderJson(
|
||||
options.glossary.length === 0 ? options.defaultGlossary : options.glossary,
|
||||
),
|
||||
localeModule: renderLocaleModule(input.entry, nextMap),
|
||||
meta: renderJson(nextMeta),
|
||||
nextFlat,
|
||||
translationMemory: renderTranslationMemory(translationMemory),
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function compareStringArrays(left: string[], right: string[]): boolean {
|
||||
return left.length === right.length && left.every((value, index) => value === right[index]);
|
||||
}
|
||||
|
||||
function setNestedValue(root: TranslationMap, dottedKey: string, value: string): void {
|
||||
const parts = dottedKey.split(".");
|
||||
let cursor: TranslationMap = root;
|
||||
for (let index = 0; index < parts.length - 1; index += 1) {
|
||||
const key = parts[index];
|
||||
const next = cursor[key];
|
||||
if (!next || typeof next === "string") {
|
||||
const replacement: TranslationMap = {};
|
||||
cursor[key] = replacement;
|
||||
cursor = replacement;
|
||||
continue;
|
||||
}
|
||||
cursor = next;
|
||||
}
|
||||
cursor[parts.at(-1)!] = value;
|
||||
}
|
||||
|
||||
function renderTranslationValue(value: string | TranslationMap, indent = 0): string {
|
||||
if (typeof value === "string") {
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
const entries = Object.entries(value);
|
||||
if (entries.length === 0) {
|
||||
return "{}";
|
||||
}
|
||||
|
||||
const pad = " ".repeat(indent);
|
||||
const innerPad = " ".repeat(indent + 1);
|
||||
return `{\n${entries
|
||||
.map(([key, nested]) => {
|
||||
const renderedKey = /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key) ? key : JSON.stringify(key);
|
||||
return `${innerPad}${renderedKey}: ${renderTranslationValue(nested, indent + 1)},`;
|
||||
})
|
||||
.join("\n")}\n${pad}}`;
|
||||
}
|
||||
|
||||
function renderLocaleModule(entry: LocaleEntry, value: TranslationMap): string {
|
||||
return `// Generated locale bundle for Control UI translations.
|
||||
// Run \`pnpm ui:i18n:sync\` instead of editing this file directly.
|
||||
import type { TranslationMap } from "../lib/types.ts";
|
||||
|
||||
export const ${entry.exportName}: TranslationMap = ${renderTranslationValue(value)};
|
||||
`;
|
||||
}
|
||||
|
||||
function renderTranslationMemory(entries: ReadonlyMap<string, TranslationMemoryEntry>): string {
|
||||
const ordered = [...entries.values()].toSorted((left, right) =>
|
||||
left.cache_key.localeCompare(right.cache_key),
|
||||
);
|
||||
return ordered.length === 0
|
||||
? ""
|
||||
: `${ordered.map((entry) => JSON.stringify(entry)).join("\n")}\n`;
|
||||
}
|
||||
|
||||
function renderJson(value: unknown): string {
|
||||
return `${JSON.stringify(value, null, 2)}\n`;
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
createControlUiLocaleSyncPlan,
|
||||
flattenTranslations,
|
||||
type LocaleEntry,
|
||||
type LocaleMeta,
|
||||
type TranslationMemoryEntry,
|
||||
} from "../../scripts/lib/control-ui-i18n-sync-plan.ts";
|
||||
|
||||
const entry: LocaleEntry = {
|
||||
exportName: "fr",
|
||||
fileName: "fr.ts",
|
||||
languageKey: "fr",
|
||||
locale: "fr",
|
||||
};
|
||||
|
||||
const hashText = (text: string) => `hash:${text}`;
|
||||
const cacheKeyFor = (key: string, textHash: string) => `cache:${key}:${textHash}`;
|
||||
|
||||
function memoryEntry(overrides: Partial<TranslationMemoryEntry> = {}): TranslationMemoryEntry {
|
||||
return {
|
||||
cache_key: "legacy-cache",
|
||||
model: "legacy-model",
|
||||
provider: "legacy-provider",
|
||||
segment_id: "legacy.segment",
|
||||
source_path: "ui/src/i18n/locales/fr.ts",
|
||||
src_lang: "en",
|
||||
text: "Shared",
|
||||
text_hash: hashText("Shared"),
|
||||
tgt_lang: "fr",
|
||||
translated: "Partage",
|
||||
updated_at: "2026-01-01T00:00:00.000Z",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function localeMeta(overrides: Partial<LocaleMeta> = {}): LocaleMeta {
|
||||
return {
|
||||
fallbackKeys: [],
|
||||
generatedAt: "2026-01-01T00:00:00.000Z",
|
||||
locale: "fr",
|
||||
model: "legacy-model",
|
||||
provider: "legacy-provider",
|
||||
sourceHash: "old-source",
|
||||
totalKeys: 0,
|
||||
translatedKeys: 0,
|
||||
workflow: 1,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("createControlUiLocaleSyncPlan", () => {
|
||||
it("plans reuse and renders deterministic locale artifacts", () => {
|
||||
const sourceFlat = flattenTranslations({
|
||||
group: {
|
||||
cached: "Cached source",
|
||||
existing: "Existing source",
|
||||
pending: "Pending source",
|
||||
reused: "Shared",
|
||||
},
|
||||
});
|
||||
const exactCacheKey = cacheKeyFor("group.cached", hashText("Cached source"));
|
||||
const exactCache = memoryEntry({
|
||||
cache_key: exactCacheKey,
|
||||
segment_id: "group.cached",
|
||||
text: "Cached source",
|
||||
text_hash: hashText("Cached source"),
|
||||
translated: "En cache",
|
||||
});
|
||||
const sharedCache = memoryEntry();
|
||||
const plan = createControlUiLocaleSyncPlan({
|
||||
allowTranslate: false,
|
||||
cacheKeyFor,
|
||||
entry,
|
||||
existingFlat: new Map([
|
||||
["group.cached", "Ancien cache"],
|
||||
["group.existing", "Existant"],
|
||||
]),
|
||||
force: false,
|
||||
hashText,
|
||||
previousMeta: localeMeta({ fallbackKeys: ["group.cached"] }),
|
||||
sourceFlat,
|
||||
sourceHash: "next-source",
|
||||
translationMemory: new Map([
|
||||
[sharedCache.cache_key, sharedCache],
|
||||
[exactCache.cache_key, exactCache],
|
||||
]),
|
||||
});
|
||||
|
||||
expect(plan.pending.map((item) => item.key)).toEqual(["group.pending"]);
|
||||
expect(plan.newFallbackCount).toBe(1);
|
||||
|
||||
const artifacts = plan.render({
|
||||
defaultGlossary: [{ source: "OpenClaw", target: "OpenClaw" }],
|
||||
generatedAt: "2026-02-02T00:00:00.000Z",
|
||||
glossary: [],
|
||||
model: "legacy-model",
|
||||
provider: "legacy-provider",
|
||||
workflow: 1,
|
||||
});
|
||||
|
||||
expect(artifacts.localeModule).toBe(
|
||||
[
|
||||
"// Generated locale bundle for Control UI translations.",
|
||||
"// Run `pnpm ui:i18n:sync` instead of editing this file directly.",
|
||||
'import type { TranslationMap } from "../lib/types.ts";',
|
||||
"",
|
||||
"export const fr: TranslationMap = {",
|
||||
" group: {",
|
||||
' cached: "En cache",',
|
||||
' existing: "Existant",',
|
||||
' pending: "Pending source",',
|
||||
' reused: "Partage",',
|
||||
" },",
|
||||
"};",
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
expect(artifacts.meta).toBe(
|
||||
`${JSON.stringify(
|
||||
{
|
||||
fallbackKeys: ["group.cached", "group.pending"],
|
||||
generatedAt: "2026-02-02T00:00:00.000Z",
|
||||
locale: "fr",
|
||||
model: "legacy-model",
|
||||
provider: "legacy-provider",
|
||||
sourceHash: "next-source",
|
||||
totalKeys: 4,
|
||||
translatedKeys: 2,
|
||||
workflow: 1,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
expect(artifacts.glossary).toBe(
|
||||
`${JSON.stringify([{ source: "OpenClaw", target: "OpenClaw" }], null, 2)}\n`,
|
||||
);
|
||||
const clonedCache = {
|
||||
...sharedCache,
|
||||
cache_key: cacheKeyFor("group.reused", hashText("Shared")),
|
||||
segment_id: "group.reused",
|
||||
};
|
||||
expect(artifacts.translationMemory).toBe(
|
||||
`${[clonedCache, exactCache, sharedCache]
|
||||
.toSorted((left, right) => left.cache_key.localeCompare(right.cache_key))
|
||||
.map((value) => JSON.stringify(value))
|
||||
.join("\n")}\n`,
|
||||
);
|
||||
});
|
||||
|
||||
it("refreshes recorded fallbacks and records translated replacements", () => {
|
||||
const sourceFlat = flattenTranslations({ title: "New English" });
|
||||
const previousMeta = localeMeta({
|
||||
fallbackKeys: ["title"],
|
||||
sourceHash: "previous-source",
|
||||
totalKeys: 1,
|
||||
translatedKeys: 0,
|
||||
});
|
||||
const plan = createControlUiLocaleSyncPlan({
|
||||
allowTranslate: true,
|
||||
cacheKeyFor,
|
||||
entry,
|
||||
existingFlat: new Map([["title", "Old English"]]),
|
||||
force: true,
|
||||
hashText,
|
||||
previousMeta,
|
||||
sourceFlat,
|
||||
sourceHash: "next-source",
|
||||
translationMemory: new Map(),
|
||||
});
|
||||
|
||||
expect(plan.newFallbackCount).toBe(0);
|
||||
plan.recordTranslations(plan.pending, new Map([["title", "Nouveau"]]), {
|
||||
model: "next-model",
|
||||
provider: "next-provider",
|
||||
sourceLocale: "en",
|
||||
updatedAt: () => "2026-02-02T00:00:00.000Z",
|
||||
});
|
||||
|
||||
const artifacts = plan.render({
|
||||
defaultGlossary: [],
|
||||
generatedAt: "2026-03-03T00:00:00.000Z",
|
||||
glossary: [],
|
||||
model: "next-model",
|
||||
provider: "next-provider",
|
||||
workflow: 1,
|
||||
});
|
||||
|
||||
expect(artifacts.fallbackCount).toBe(0);
|
||||
expect(artifacts.nextFlat.get("title")).toBe("Nouveau");
|
||||
expect(JSON.parse(artifacts.meta)).toMatchObject({
|
||||
fallbackKeys: [],
|
||||
generatedAt: "2026-03-03T00:00:00.000Z",
|
||||
translatedKeys: 1,
|
||||
});
|
||||
expect(artifacts.translationMemory).toBe(
|
||||
`${JSON.stringify(
|
||||
memoryEntry({
|
||||
cache_key: cacheKeyFor("title", hashText("New English")),
|
||||
model: "next-model",
|
||||
provider: "next-provider",
|
||||
segment_id: "title",
|
||||
text: "New English",
|
||||
text_hash: hashText("New English"),
|
||||
translated: "Nouveau",
|
||||
updated_at: "2026-02-02T00:00:00.000Z",
|
||||
}),
|
||||
)}\n`,
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves generatedAt when semantic metadata is unchanged", () => {
|
||||
const sourceFlat = flattenTranslations({ title: "Titre" });
|
||||
const previousMeta = localeMeta({
|
||||
sourceHash: "same-source",
|
||||
totalKeys: 1,
|
||||
translatedKeys: 1,
|
||||
});
|
||||
const plan = createControlUiLocaleSyncPlan({
|
||||
allowTranslate: false,
|
||||
cacheKeyFor,
|
||||
entry,
|
||||
existingFlat: new Map([["title", "Titre"]]),
|
||||
force: false,
|
||||
hashText,
|
||||
previousMeta,
|
||||
sourceFlat,
|
||||
sourceHash: "same-source",
|
||||
translationMemory: new Map(),
|
||||
});
|
||||
|
||||
const artifacts = plan.render({
|
||||
defaultGlossary: [],
|
||||
generatedAt: "2026-03-03T00:00:00.000Z",
|
||||
glossary: [],
|
||||
model: "legacy-model",
|
||||
provider: "legacy-provider",
|
||||
workflow: 1,
|
||||
});
|
||||
|
||||
expect(JSON.parse(artifacts.meta)).toMatchObject({
|
||||
generatedAt: previousMeta.generatedAt,
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user