diff --git a/scripts/generateLocales.ts b/scripts/generateLocales.ts index 86704a7c03d..d2e9513e1b0 100644 --- a/scripts/generateLocales.ts +++ b/scripts/generateLocales.ts @@ -210,7 +210,7 @@ async function generateRecursiveModuleIndexes( submodules = removeIndexTs(submodules); for (const submodule of submodules) { const pathModule = resolve(path, submodule); - updateLocaleFile(pathModule); + await updateLocaleFile(pathModule); // Only process sub folders recursively if (lstatSync(pathModule).isDirectory()) { let moduleDefinition = @@ -238,14 +238,12 @@ async function generateRecursiveModuleIndexes( * * @param filePath The full file path to the file. */ -function updateLocaleFile(filePath: string): void { +async function updateLocaleFile(filePath: string): Promise { if (lstatSync(filePath).isFile()) { - const pathParts = filePath + const [locale, moduleKey, entryKey] = filePath .substring(pathLocales.length + 1, filePath.length - 3) .split(/[\\/]/); - const locale = pathParts[0]; - pathParts.splice(0, 1); - updateLocaleFileHook(filePath, locale, pathParts); + await updateLocaleFileHook(filePath, locale, moduleKey, entryKey); } } @@ -255,16 +253,120 @@ function updateLocaleFile(filePath: string): void { * * @param filePath The full file path to the file. * @param locale The locale for that file. - * @param localePath The locale path parts (after the locale). + * @param definitionKey The definition key of the current file (ex. 'location'). + * @param entryName The entry key of the current file (ex. 'state'). Is `undefined` if `definitionKey` is `'metadata'`. */ -function updateLocaleFileHook( +async function updateLocaleFileHook( filePath: string, locale: string, - localePath: string[] -): void { + definitionKey: string, + entryName: string | undefined +): Promise { + // this needs to stay so all arguments are "used" if (filePath === 'never') { - console.log(`${filePath} <-> ${locale} @ ${localePath.join(' -> ')}`); + console.log(`${filePath} <-> ${locale} @ ${definitionKey} -> ${entryName}`); } + + await normalizeLocaleFile(filePath, definitionKey); +} + +/** + * Normalizes the data of a locale file based on a set of rules. + * Those include: + * - filter the entry list for duplicates + * - limiting the maximum entries of a file to 1000 + * - sorting the entries alphabetically + * + * This function mutates the file by reading and writing to it! + * + * @param filePath The full file path to the file. + * @param definitionKey The definition key of the current file (ex. 'location'). + */ +async function normalizeLocaleFile(filePath: string, definitionKey: string) { + function normalizeDataRecursive(localeData: T): T { + if (typeof localeData !== 'object' || localeData === null) { + // we can only traverse object-like structs + return localeData; + } + + if (Array.isArray(localeData)) { + return ( + [...new Set(localeData)] + // limit entries to 1k + .slice(0, 1000) + // sort entries alphabetically + .sort() as T + ); + } + + const result = {} as T; + for (const key of Object.keys(localeData)) { + result[key] = normalizeDataRecursive(localeData[key]); + } + + return result; + } + + const legacyDefinitions = ['app', 'cell_phone', 'team']; + const definitionsToSkip = [ + 'airline', + 'animal', + 'color', + 'commerce', + 'company', + 'database', + 'date', + 'finance', + 'hacker', + 'internet', + 'location', + 'lorem', + 'metadata', + 'music', + 'person', + 'phone_number', + 'science', + 'system', + 'vehicle', + 'word', + ...legacyDefinitions, + ]; + if (definitionsToSkip.includes(definitionKey)) { + return; + } + + console.log(`Running data normalization for:`, filePath); + + const fileContent = readFileSync(filePath).toString(); + const searchString = 'export default '; + const compareIndex = fileContent.indexOf(searchString) + searchString.length; + const compareString = fileContent.substring(compareIndex); + + const isDynamicFile = compareString.startsWith('mergeArrays'); + const isNonApplicable = compareString.startsWith('null'); + const isFrozenData = compareString.startsWith('Object.freeze'); + if (isDynamicFile || isNonApplicable || isFrozenData) { + return; + } + + const validEntryListStartCharacters = ['[', '{']; + const staticFileOpenSyntax = validEntryListStartCharacters.find( + (validStart) => compareString.startsWith(validStart) + ); + if (staticFileOpenSyntax === undefined) { + console.log('Found an unhandled dynamic file:', filePath); + return; + } + + const fileContentPreData = fileContent.substring(0, compareIndex); + // eslint-disable-next-line @typescript-eslint/no-var-requires + const localeData = normalizeDataRecursive(require(filePath).default); + + // We reattach the content before the actual data implementation to keep stuff like comments. + // In the long term we should probably define a whether we want those in the files at all. + const newContent = fileContentPreData + JSON.stringify(localeData); + + writeFileSync(filePath, await format(newContent, prettierTsOptions)); } // Start of actual logic