import fs from 'node:fs/promises' import path from 'node:path' import { pathToFileURL } from 'node:url' import process from 'node:process' // TypeScript 7 is a native CLI; AST consumers still need the legacy JavaScript API. import ts from 'typescript-api' import { canonicalGenericRenderings } from './locale-generic-ui-terms.mjs' import { LOCALIZATION_SURFACES, readLocalizationSurface } from './localization-surfaces.mjs' import { repairTranslatedValue } from './locale-translation-policy.mjs' import { interpolationVariablesMatch, pluralFamilyDefaults, resolveLocaleSourceKey, syncLocalePluralForms } from './locale-plural-forms.mjs' const SOURCE_EXTENSIONS = new Set(['.ts', '.tsx', '.js', '.jsx', '.mts', '.cts']) const SKIP_PATH_PARTS = new Set(['.git', 'dist', 'node_modules', 'out', '__snapshots__', 'assets']) const LOCALIZATION_FUNCTION_NAMES = new Set([ 't', 'translate', 'translateMain', 'translateSearchKeyword' ]) const PLACEHOLDER_RE = /\{\{[^}]+\}\}/g export const LOCALIZATION_SOURCE_ROOTS = LOCALIZATION_SURFACES.desktop.sourceRoots function normalizePath(root, filePath) { return path.relative(root, filePath).split(path.sep).join('/') } function isSkippedFile(root, filePath) { const relative = normalizePath(root, filePath) if ( relative.endsWith('.d.ts') || // Build output (bundled engines, protocol schemas) is not translatable source. relative.includes('.generated.') || relative.includes('.test.') || relative.includes('.spec.') || relative.includes('/__tests__/') ) { return true } return relative.split('/').some((part) => SKIP_PATH_PARTS.has(part)) } export async function collectSourceFiles(root, dir) { const entries = await fs.readdir(dir, { withFileTypes: true }) const files = [] for (const entry of entries) { const fullPath = path.join(dir, entry.name) if (entry.isDirectory()) { if (!SKIP_PATH_PARTS.has(entry.name)) { files.push(...(await collectSourceFiles(root, fullPath))) } continue } if ( entry.isFile() && SOURCE_EXTENSIONS.has(path.extname(entry.name)) && !isSkippedFile(root, fullPath) ) { files.push(fullPath) } } return files } function flattenCatalogKeys(value, prefix = '') { if (typeof value !== 'object' || value === null || Array.isArray(value)) { return prefix ? [prefix] : [] } return Object.entries(value).flatMap(([key, child]) => flattenCatalogKeys(child, prefix ? `${prefix}.${key}` : key) ) } function expressionNameText(node) { if (ts.isIdentifier(node)) { return node.text } if (ts.isPropertyAccessExpression(node)) { return `${expressionNameText(node.expression) ?? ''}.${node.name.text}`.replace(/^\./, '') } return undefined } function reportAt(root, filePath, sourceFile, node, key, fallback) { const position = sourceFile.getLineAndCharacterOfPosition(node.getStart(sourceFile)) return { filePath: normalizePath(root, filePath), line: position.line + 1, column: position.character + 1, key, fallback } } export function collectLocalizationKeyReferences(filePath, sourceText, root = process.cwd()) { const sourceKind = filePath.endsWith('.tsx') || filePath.endsWith('.jsx') ? ts.ScriptKind.TSX : ts.ScriptKind.TS const sourceFile = ts.createSourceFile( filePath, sourceText, ts.ScriptTarget.Latest, true, sourceKind ) const references = [] function visit(node) { if (ts.isCallExpression(node)) { const name = expressionNameText(node.expression) const functionName = name?.split('.').at(-1) const firstArg = node.arguments[0] if ( functionName && LOCALIZATION_FUNCTION_NAMES.has(functionName) && firstArg && ts.isStringLiteralLike(firstArg) ) { const secondArg = node.arguments[1] const fallback = secondArg && ts.isStringLiteralLike(secondArg) ? secondArg.text : undefined const keys = pluralFamilyDefaults(ts, firstArg.text, fallback, node.arguments[2]) ?? [ [firstArg.text, fallback] ] for (const [key, value] of keys) { references.push(reportAt(root, filePath, sourceFile, firstArg, key, value)) } } } ts.forEachChild(node, visit) } visit(sourceFile) return references } function formatMissingReferences(missing) { return missing .map( (reference) => `${reference.filePath}:${reference.line}:${reference.column} ${reference.key}` ) .join('\n') } function formatMissingKeys(label, keys) { return keys.map((key) => `${label}: ${key}`).join('\n') } // Language packs require nested names even though i18next also accepts flat keys. function collectDottedCatalogKeys(value, prefix = '') { if (typeof value !== 'object' || value === null || Array.isArray(value)) { return [] } return Object.entries(value).flatMap(([key, child]) => { const keyPath = prefix ? `${prefix}.${key}` : key return [...(key.includes('.') ? [keyPath] : []), ...collectDottedCatalogKeys(child, keyPath)] }) } function reportDottedCatalogKeys(fileName, catalog) { const dottedKeys = collectDottedCatalogKeys(catalog) if (dottedKeys.length === 0) { return 0 } console.error(`${fileName} has keys with a dot in the name. Write them as nested objects.`) console.error(formatMissingKeys('dotted', dottedKeys.slice(0, 20))) if (dottedKeys.length > 20) { console.error(`...and ${dottedKeys.length - 20} more dotted keys`) } return 1 } function normalizeInterpolationVariables(value) { return collectInterpolationVariables(value) .map((variable) => variable.slice(2, -2)) .join('|') } function formatFallbackReferenceGroups(groups) { return groups .map(({ key, references }) => { const locations = references .map( (reference) => ` ${reference.filePath}:${reference.line}:${reference.column} ${JSON.stringify(reference.fallback)}` ) .join('\n') return `${key}\n${locations}` }) .join('\n\n') } function groupFallbackReferencesByKey(references) { const byKey = new Map() for (const reference of references) { if (typeof reference.fallback !== 'string') { continue } const existing = byKey.get(reference.key) ?? [] existing.push(reference) byKey.set(reference.key, existing) } return byKey } // Why: the catalog holds one value per key, so a second sentence on the same key never renders. function collectReusedFallbackKeys(references) { return [...groupFallbackReferencesByKey(references).entries()] .filter(([, keyReferences]) => new Set(keyReferences.map((r) => r.fallback)).size > 1) .map(([key, keyReferences]) => ({ key, references: keyReferences })) } /** Prints keys reused for different fallback text; returns true when any exist. */ function reportReusedFallbackKeys(references) { const reusedFallbackKeys = collectReusedFallbackKeys(references) if (reusedFallbackKeys.length === 0) { return false } console.error('Localization keys are reused for different fallback text; give each its own key.') console.error('') console.error(formatFallbackReferenceGroups(reusedFallbackKeys)) return true } function collectInconsistentFallbackVariables(references) { return [...groupFallbackReferencesByKey(references).entries()] .map(([key, keyReferences]) => { const uniqueFallbackVariables = new Set( keyReferences.map((reference) => normalizeInterpolationVariables(reference.fallback)) ) return { key, references: keyReferences, uniqueFallbackVariableCount: uniqueFallbackVariables.size } }) .filter(({ uniqueFallbackVariableCount }) => uniqueFallbackVariableCount > 1) } function collectInterpolationVariables(value) { if (typeof value === 'string') { const matches = value.match(PLACEHOLDER_RE) ?? [] return [...matches].sort() } if (typeof value !== 'object' || value === null || Array.isArray(value)) { return [] } return Object.values(value).flatMap((child) => collectInterpolationVariables(child)) } function flattenCatalogEntries(value, prefix = '', entries = new Map()) { if (typeof value === 'string') { entries.set(prefix, value) return entries } if (typeof value !== 'object' || value === null || Array.isArray(value)) { return entries } for (const [key, child] of Object.entries(value)) { flattenCatalogEntries(child, prefix ? `${prefix}.${key}` : key, entries) } return entries } function getCatalogEntry(catalog, key) { return key.split('.').reduce((cursor, part) => cursor?.[part], catalog) } function setCatalogEntry(catalog, key, value) { const parts = key.split('.') let cursor = catalog for (const part of parts.slice(0, -1)) { if (typeof cursor[part] !== 'object' || cursor[part] === null || Array.isArray(cursor[part])) { cursor[part] = {} } cursor = cursor[part] } cursor[parts.at(-1)] = value } function collectLocaleParityIssues(enCatalog, localeCatalog, localeName) { const enEntries = flattenCatalogEntries(enCatalog) const localeEntries = flattenCatalogEntries(localeCatalog) const missingInLocale = [...enEntries.keys()].filter((key) => !localeEntries.has(key)) const extraInLocale = [...localeEntries.keys()].filter( (key) => !resolveLocaleSourceKey(enEntries, localeName, key) ) const interpolationMismatches = [] for (const [key, localeValue] of localeEntries) { const sourceKey = resolveLocaleSourceKey(enEntries, localeName, key) if (!sourceKey) { continue } const enVariables = collectInterpolationVariables(enEntries.get(sourceKey)) const localeVariables = collectInterpolationVariables(localeValue) if (!interpolationVariablesMatch(sourceKey, enVariables, localeVariables)) { interpolationMismatches.push(key) } } return { enEntries, localeEntries, missingInLocale, extraInLocale, interpolationMismatches } } function referencesMissingFallbacks(missing) { return missing.filter((reference) => typeof reference.fallback !== 'string') } function collectMissingCatalogEntries(missing) { const entries = new Map() for (const reference of missing) { if (typeof reference.fallback !== 'string') { continue } if (!entries.has(reference.key)) { entries.set(reference.key, reference.fallback) } } return entries } function applyMissingEnglishEntries(catalog, missing) { const entries = collectMissingCatalogEntries(missing) let changed = 0 for (const [key, fallback] of entries) { if (getCatalogEntry(catalog, key) !== undefined) { continue } setCatalogEntry(catalog, key, fallback) changed += 1 } return changed } // Check repair policy too: parity passed while repair rewrote translated generic terms into English (#12113). export function collectGenericTermRegressions(enEntries, localeEntries, localeName) { const renderings = canonicalGenericRenderings(localeName) if (renderings.length === 0) { return [] } const regressions = [] for (const [key, enValue] of enEntries) { const localeValue = localeEntries.get(key) if (typeof enValue !== 'string' || typeof localeValue !== 'string') { continue } const repaired = repairTranslatedValue({ key, enValue, localeValue, locale: localeName }) if (repaired === localeValue) { continue } // Why: {{agent}} is an interpolation name, not English copy the reader sees. const repairedCopy = repaired.replace(PLACEHOLDER_RE, '') for (const { form, terms } of renderings) { if (!localeValue.includes(form) || repaired.includes(form)) { continue } if ( terms.some((term) => new RegExp(`(^|[^A-Za-z])${term}($|[^A-Za-z])`).test(repairedCopy)) ) { regressions.push({ key, form, localeValue, repaired }) break } } } return regressions } function formatGenericTermRegressions(regressions) { return regressions .map((entry) => `${entry.key}: ${entry.form} -> English (${entry.repaired})`) .join('\n') } // Why: JSON.parse silently keeps the last duplicate key. function collectDuplicateCatalogKeys(fileName, text) { const sourceFile = ts.parseJsonText(fileName, text) const duplicates = [] const visit = (node, prefix) => { if (!ts.isObjectLiteralExpression(node)) { return } const seen = new Set() for (const property of node.properties) { if (!ts.isPropertyAssignment(property)) { continue } const name = property.name.text const key = prefix ? `${prefix}.${name}` : name if (seen.has(name)) { const { line } = sourceFile.getLineAndCharacterOfPosition(property.getStart(sourceFile)) duplicates.push(`${fileName}:${line + 1} ${key}`) } seen.add(name) visit(property.initializer, key) } } for (const statement of sourceFile.statements) { visit(statement.expression, '') } return duplicates } async function readCatalogFile(catalogPath) { const text = await fs.readFile(catalogPath, 'utf8') const duplicates = collectDuplicateCatalogKeys(path.basename(catalogPath), text) if (duplicates.length > 0) { console.error('Localization catalog has duplicate keys; JSON keeps only the last value.') console.error('') console.error(duplicates.map((duplicate) => `- ${duplicate}`).join('\n')) return undefined } return JSON.parse(text) } function verifyLocaleCatalog(enCatalog, localeName, localeCatalog) { const { enEntries, localeEntries, missingInLocale, extraInLocale, interpolationMismatches } = collectLocaleParityIssues(enCatalog, localeCatalog, localeName) const genericTermRegressions = collectGenericTermRegressions(enEntries, localeEntries, localeName) // Feature PRs own English declarations; missing target translations use i18next fallback until localization supplies them. console.log( `${localeName}.json coverage: ${enEntries.size - missingInLocale.length}/${enEntries.size} translated, ${missingInLocale.length} missing.` ) if ( extraInLocale.length > 0 || interpolationMismatches.length > 0 || genericTermRegressions.length > 0 ) { console.error(`Locale catalog validation failed for ${localeName}.json.`) if (extraInLocale.length > 0) { console.error('') console.error(formatMissingKeys('extra', extraInLocale.slice(0, 20))) if (extraInLocale.length > 20) { console.error(`...and ${extraInLocale.length - 20} more extra keys`) } } if (interpolationMismatches.length > 0) { console.error('') console.error( formatMissingKeys('interpolation mismatch', interpolationMismatches.slice(0, 20)) ) if (interpolationMismatches.length > 20) { console.error(`...and ${interpolationMismatches.length - 20} more interpolation mismatches`) } } if (genericTermRegressions.length > 0) { console.error('') console.error( 'repair-locale-catalog would rewrite these translated terms back to English.', 'Treat the term as generic in config/scripts/locale-generic-ui-terms.mjs', 'instead of listing its translation as a mistranslation.' ) console.error(formatGenericTermRegressions(genericTermRegressions.slice(0, 20))) if (genericTermRegressions.length > 20) { console.error(`...and ${genericTermRegressions.length - 20} more generic term regressions`) } } return 1 } console.log(`Verified ${localeEntries.size} existing ${localeName}.json entries.`) return 0 } function parseArgs(argv) { const pluginCatalogs = [] for (let index = 0; index < argv.length; index += 1) { const argument = argv[index] if (argument === '--plugin-catalog') { const catalogPath = argv[index + 1] if (!catalogPath || catalogPath.startsWith('--')) { throw new Error('--plugin-catalog requires a JSON catalog path') } pluginCatalogs.push(catalogPath) index += 1 } else if (argument.startsWith('--plugin-catalog=')) { pluginCatalogs.push(argument.slice('--plugin-catalog='.length)) } } return { fix: argv.includes('--fix'), pluginCatalogs, surface: readLocalizationSurface(argv) } } async function reportPluginCatalog(root, catalog, pluginCatalogPath) { const resolvedPath = path.resolve(root, pluginCatalogPath) let pluginCatalog try { pluginCatalog = JSON.parse(await fs.readFile(resolvedPath, 'utf8')) } catch (error) { console.error( `Could not read plugin catalog ${normalizePath(root, resolvedPath)}: ${error instanceof Error ? error.message : String(error)}` ) return 1 } const { enEntries, localeEntries, missingInLocale, extraInLocale, interpolationMismatches } = collectLocaleParityIssues(catalog, pluginCatalog) const translated = enEntries.size - missingInLocale.length - interpolationMismatches.length const coverage = enEntries.size === 0 ? 100 : (translated / enEntries.size) * 100 console.log( `Plugin catalog ${normalizePath(root, resolvedPath)}: ${translated}/${enEntries.size} core keys (${coverage.toFixed(1)}% coverage), ${localeEntries.size} catalog entries.` ) if (missingInLocale.length > 0) { console.log(formatMissingKeys('missing', missingInLocale.slice(0, 20))) if (missingInLocale.length > 20) { console.log(`...and ${missingInLocale.length - 20} more missing keys`) } } if (extraInLocale.length > 0) { console.log(formatMissingKeys('extra', extraInLocale.slice(0, 20))) } if (interpolationMismatches.length > 0) { console.log(formatMissingKeys('interpolation mismatch', interpolationMismatches.slice(0, 20))) } // Missing plugin translation can fall back; mismatched variables in present value can mislead or break UI. return interpolationMismatches.length > 0 ? 1 : 0 } export async function collectLocalizationReferences(root, roots = LOCALIZATION_SOURCE_ROOTS) { const sourceRoots = roots.map((sourceRoot) => path.join(root, sourceRoot)) const references = [] for (const sourceRoot of sourceRoots) { const files = await collectSourceFiles(root, sourceRoot) for (const filePath of files) { references.push( ...collectLocalizationKeyReferences(filePath, await fs.readFile(filePath, 'utf8'), root) ) } } return references } export async function main( root = process.cwd(), options = parseArgs(process.argv.slice(2)), sharedReferences ) { const surface = options.surface ?? LOCALIZATION_SURFACES.desktop const localesDir = path.join(root, surface.localesDir) const catalogPath = path.join(localesDir, 'en.json') const enCatalogLabel = normalizePath(root, catalogPath) const catalog = await readCatalogFile(catalogPath) if (catalog === undefined) { return 1 } const pluginCatalogs = options.pluginCatalogs ?? [] if (pluginCatalogs.length > 0) { if (options.fix) { console.error('--fix cannot be combined with --plugin-catalog') return 1 } for (const pluginCatalogPath of pluginCatalogs) { const result = await reportPluginCatalog(root, catalog, pluginCatalogPath) if (result !== 0) { return result } } return 0 } if (reportDottedCatalogKeys('en.json', catalog) !== 0) { return 1 } let catalogKeys = new Set(flattenCatalogKeys(catalog)) const references = sharedReferences ?? (await collectLocalizationReferences(root, surface.sourceRoots)) // Why: runs before --fix so an ambiguous key never gets its first sentence written to en.json. if (reportReusedFallbackKeys(references)) { return 1 } const missing = references.filter((reference) => !catalogKeys.has(reference.key)) if (missing.length > 0) { const missingFallbacks = referencesMissingFallbacks(missing) if (options.fix && missingFallbacks.length === 0) { const added = applyMissingEnglishEntries(catalog, missing) await fs.writeFile(catalogPath, `${JSON.stringify(catalog, null, 2)}\n`, 'utf8') catalogKeys = new Set(flattenCatalogKeys(catalog)) console.log(`Added ${added} missing localization key(s) to en.json.`) } else { if (options.fix && missingFallbacks.length > 0) { console.error('Some missing localization keys do not have string fallbacks to bootstrap.') console.error('') console.error(formatMissingReferences(missingFallbacks)) return 1 } console.error(`Localization keys are missing from ${enCatalogLabel}.`) console.error('') console.error(formatMissingReferences(missing)) console.error('') console.error(`Run \`${surface.syncCatalogCommand}\` to add keys with string fallbacks.`) return 1 } } const remainingMissing = references.filter((reference) => !catalogKeys.has(reference.key)) if (remainingMissing.length > 0) { console.error(`Localization keys are missing from ${enCatalogLabel}.`) console.error('') console.error(formatMissingReferences(remainingMissing)) return 1 } const inconsistentFallbackVariables = collectInconsistentFallbackVariables(references) if (inconsistentFallbackVariables.length > 0) { console.error('Localization keys are used with inconsistent interpolation placeholders.') console.error('') console.error(formatFallbackReferenceGroups(inconsistentFallbackVariables)) return 1 } console.log(`Verified ${references.length} localization key references against en.json.`) const localeFiles = (await fs.readdir(localesDir)) .filter( (fileName) => fileName.endsWith('.json') && fileName !== 'en.json' && !fileName.startsWith('.') && !fileName.includes('-catalog-cache') ) .sort() for (const fileName of localeFiles) { const localeName = fileName.replace(/\.json$/, '') const localeCatalogPath = path.join(localesDir, fileName) let localeCatalog = await readCatalogFile(localeCatalogPath) if (localeCatalog === undefined || reportDottedCatalogKeys(fileName, localeCatalog) !== 0) { return 1 } if (surface.fillsPluralForms) { const plural = await syncLocalePluralForms(localeCatalogPath, localeCatalog, options.fix) if (!plural.ok) { console.error( `Run \`${surface.syncCatalogCommand}\` to copy them from each \`_other\` form.` ) return 1 } localeCatalog = plural.catalog } const exitCode = verifyLocaleCatalog(catalog, localeName, localeCatalog) if (exitCode !== 0) { console.error('') console.error('Fix or retire the existing target entry in a localization PR.') return exitCode } } return 0 } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { process.exit(await main()) }