feat: add a script for fixing missing locales

README.md

@@ -217,6 +217,7 @@ yarn build:watch
 Command:
 ```
 yarn generate:graphql-types
+```
 
 Generates the `types.ts` files separately for `The Core App` and independent modules.
 If independent modules are not installed on `The Platform`, types can still be safely generated.
@@ -243,13 +244,23 @@ yarn generate:dependency-graph client-app/main.ts client-app/shared/account/comp
 ```
 The generated graph will also be saved in the `artifacts` folder.
 
+## Localization
+### Check for missing locale keys
+
 ```
 yarn check-locales -- path/to/locales_folder path/to/**/locales
 ```
 The command is used to ensure that all locale files have consistent keys across different languages. This helps in maintaining uniformity and avoiding missing translations.
 
 The script will output warnings for any missing keys in the locale files. Review these warnings to ensure all necessary translations are present. Also added to the CI pipeline.
 
+### Fix Missing Locales
+```
+yarn fix-locales -- path/to/locales_folder path/to/\*\*/locales
+```
+This command can be run locally to automatically fix missing translations in locale files by using AI translation. It analyzes all locale files, identifies missing keys, and translates the missing content from the source language to the target language and updates locale files accordingly.
+**Note**: This command requires the `APP_GEMINI_API_KEY` environment variable to be set. You can obtain this API key from the [Google AI Studio](https://aistudio.google.com/app/apikey) website.
+
 ### Troubleshooting
 
 If you encounter an error such as `dot command not found` on Windows, ensure that [Graphviz](https://graphviz.gitlab.io/download/) is installed on your system.

package.json

@@ -37,7 +37,8 @@
     "storybook:dev": "yarn precheck && yarn generate:certificates && yarn storybook:styles && storybook dev --port 6006 --no-open --https --ssl-cert ./.certificates/public.pem --ssl-key ./.certificates/private.pem",
     "storybook:build": "yarn precheck && yarn storybook:styles && storybook build",
     "postinstall": "yarn precheck && husky",
-    "check-locales": "yarn precheck && vite-node scripts/check-locales-folder.ts -- locales client-app/ui-kit/locales client-app/modules/**/locales"
+    "check-locales": "yarn precheck && vite-node scripts/check-and-print-missing-locales -- locales client-app/ui-kit/locales client-app/modules/**/locales",
+    "fix-locales": "yarn precheck && vite-node scripts/fix-missing-locales.ts -- locales client-app/ui-kit/locales client-app/modules/**/locales"
   },
   "dependencies": {
     "@apollo/client": "^3.10.5",
@@ -79,6 +80,7 @@
   "devDependencies": {
     "@commitlint/cli": "^19.3.0",
     "@commitlint/config-conventional": "^19.2.2",
+    "@google/generative-ai": "^0.21.0",
     "@graphql-codegen/add": "5.0.3",
     "@graphql-codegen/cli": "5.0.2",
     "@graphql-codegen/introspection": "4.0.3",

scripts/check-and-print-missing-locales.ts

@@ -0,0 +1,14 @@
+import groupBy from "lodash/groupBy";
+import { main as checkLocales } from "./check-locales-folder-for-missing-keys";
+import type { MissingKeyType } from "./check-locales-folder-for-missing-keys";
+
+function printMissingKeys(missingKeys: MissingKeyType[]) {
+  const groupedByLanguages = groupBy(missingKeys, ({ originFile, targetFile }) => `${originFile}|${targetFile}`);
+  Object.entries(groupedByLanguages).forEach(([languages, keys]) => {
+    const [originFile, targetFile] = languages.split("|");
+    console.warn(`Warning: Keys missing in ${targetFile} compared to ${originFile}:`);
+    keys.forEach((key) => console.warn(`  - ${key.key}`));
+  });
+}
+
+printMissingKeys(checkLocales());

scripts/check-locales-folder.ts → scripts/check-locales-folder-for-missing-keys.ts

@@ -2,10 +2,17 @@ import * as fs from "fs";
 import * as path from "path";
 import * as glob from "glob";
 
-type LocaleDataType = {
+export type LocaleDataType = {
   [key: string]: string | LocaleDataType;
 };
 
+export type MissingKeyType = {
+  key: string;
+  originFile: string;
+  targetFile: string;
+  localeFolder: string;
+};
+
 function loadJson(filePath: string): LocaleDataType {
   const data = fs.readFileSync(filePath, "utf-8");
   return JSON.parse(data) as LocaleDataType;
@@ -25,12 +32,21 @@ function getAllKeys(obj: LocaleDataType, parentKey: string = ""): string[] {
   return keys;
 }
 
-function compareKeys(baseKeys: string[], keysToCompare: string[], baseLang: string, compareLang: string): void {
+function compareKeys(
+  baseKeys: string[],
+  keysToCompare: string[],
+  baseLang: string,
+  compareLang: string,
+  localeFolder: string,
+): MissingKeyType[] {
   const missingInCompare = baseKeys.filter((key) => !keysToCompare.includes(key));
-  if (missingInCompare.length > 0) {
-    console.warn(`Warning: Keys missing in ${compareLang} compared to ${baseLang}:`);
-    missingInCompare.forEach((key) => console.warn(`  - ${key}`));
-  }
+
+  return missingInCompare.map((key) => ({
+    key,
+    originFile: baseLang,
+    targetFile: compareLang,
+    localeFolder,
+  }));
 }
 
 function validateLocaleFolder(localeFolder: string): boolean {
@@ -71,13 +87,15 @@ function getLocaleFolders(patterns: string[]): string[] {
 
 // @description: This script checks if all keys in the locales are presented.
 // @usage: yarn check-locales -- path/to/locales_folder path/to/**/locales
-function main(): void {
+export function main(): MissingKeyType[] {
   const args = process.argv.slice(2);
 
   const patterns = args.length > 0 ? args : ["locales"]; // Default to 'locales' if no argument is provided
 
   const localeFolders = getLocaleFolders(patterns);
 
+  const missingKeys: MissingKeyType[] = [];
+
   localeFolders.forEach((localeFolder) => {
     if (!validateLocaleFolder(localeFolder)) {
       return;
@@ -100,11 +118,20 @@ function main(): void {
     Object.keys(localeKeys).forEach((baseFile) => {
       Object.keys(localeKeys).forEach((compareFile) => {
         if (baseFile !== compareFile) {
-          compareKeys(localeKeys[baseFile], localeKeys[compareFile], baseFile, compareFile);
+          const newMissingKeys = compareKeys(
+            localeKeys[baseFile],
+            localeKeys[compareFile],
+            baseFile,
+            compareFile,
+            localeFolder,
+          );
+          missingKeys.push(...newMissingKeys);
         }
       });
     });
   });
+
+  return missingKeys;
 }
 
 main();

scripts/fix-missing-locales.ts

@@ -0,0 +1,83 @@
+import fs from "fs";
+import path from "path";
+import get from "lodash/get";
+import set from "lodash/set";
+import { main as getMissingKeys } from "./check-locales-folder-for-missing-keys";
+import { translate } from "./translator";
+import type { LocaleDataType } from "./check-locales-folder-for-missing-keys";
+
+const PREFIX = "[FIX_LOCALES_UTILITY]";
+const DELAY_BETWEEN_TRANSLATIONS_MS = 4000;
+
+export async function fixLocales() {
+  const missingKeys = getMissingKeys();
+  if (missingKeys.length === 0) {
+    console.log(`${PREFIX} No missing keys found`);
+    return;
+  }
+  const allNeededFiles = missingKeys.reduce((acc, { originFile, targetFile, localeFolder }) => {
+    acc.add(path.join(localeFolder, originFile));
+    acc.add(path.join(localeFolder, targetFile));
+    return acc;
+  }, new Set<string>());
+
+  const fileContents: Record<string, LocaleDataType> = {};
+  try {
+    await Promise.all(
+      [...allNeededFiles].map(async (filename) => {
+        const fileData = await fs.promises.readFile(filename, "utf-8");
+        fileContents[filename] = JSON.parse(fileData) as LocaleDataType;
+      }),
+    );
+  } catch (error) {
+    console.error(`${PREFIX} Error reading files:`, error, "try again..");
+    return;
+  }
+
+  console.log(`\n---\n${PREFIX} Found ${missingKeys.length} missing keys, translating...`);
+
+  for (const [index, { key, originFile, targetFile, localeFolder }] of missingKeys.entries()) {
+    try {
+      const originFilePath = path.join(localeFolder, originFile);
+      const targetFilePath = path.join(localeFolder, targetFile);
+
+      const originFileContent = get(fileContents, originFilePath);
+      const targetFileContent = get(fileContents, targetFilePath);
+
+      const originString = get(originFileContent, key) as string;
+      const originLanguage = originFile.split(".")[0];
+      const targetLanguage = targetFile.split(".")[0];
+
+      const translatedString = await translate(originString, originLanguage, targetLanguage);
+
+      const tableRow = `${key} (${index + 1}/${missingKeys.length})`;
+      console.table({
+        [originLanguage]: { [tableRow]: originString },
+        [targetLanguage]: { [tableRow]: translatedString },
+      });
+
+      set(targetFileContent, key, translatedString);
+      await new Promise((resolve) => setTimeout(resolve, DELAY_BETWEEN_TRANSLATIONS_MS)); // delay to avoid API rate limits
+    } catch (error) {
+      console.error(
+        `${PREFIX} Error translating ${key}:`,
+        error,
+        "try again. Check api limits if restarting doesn't help.",
+      );
+      return;
+    }
+  }
+
+  try {
+    await Promise.all(
+      [...allNeededFiles].map(async (filename) => {
+        await fs.promises.writeFile(filename, JSON.stringify(fileContents[filename], null, 2));
+      }),
+    );
+    console.log(`${PREFIX} Translation completed successfully\n---\n`);
+  } catch (error) {
+    console.error(`${PREFIX} Error writing files:`, error, "check results and run script again if needed.");
+  }
+}
+
+void fixLocales();

scripts/translator.ts

@@ -0,0 +1,24 @@
+import { GoogleGenerativeAI } from "@google/generative-ai";
+
+const MODEL_NAME = "gemini-1.5-flash";
+const SYSTEM_INSTRUCTION =
+  "You are a professional translator specializing in e-commerce localization. Your task is to provide translations that are short, concise, and adhere to commonly accepted e-commerce terminology. Ensure that the original text and formatting (e.g., i18n interpolation) are preserved. Do not translate placeholders such as '@:some_key' or '@:{'some_key'}'. The output should be the translated text only, without any additional comments, explanations, or symbols not present in the original text.";
+
+const genAI = new GoogleGenerativeAI(process.env.APP_GEMINI_API_KEY as string);
+const model = genAI.getGenerativeModel({
+  model: MODEL_NAME,
+  systemInstruction: SYSTEM_INSTRUCTION,
+  generationConfig: {
+    temperature: 0.0, // disable randomness
+  },
+});
+
+function generatePrompt(text: string, originLanguage: string, targetLanguage: string) {
+  return `Translate from ${originLanguage} to ${targetLanguage}: ${text}`;
+}
+
+export async function translate(text: string, originLanguage: string, targetLanguage: string) {
+  const response = await model.generateContent(generatePrompt(text, originLanguage, targetLanguage));
+  // if original text doesn't end with \n, we need to remove \n from the response, because for some reason it's added by the model
+  return text.endsWith("\n") ? response.response.text() : response.response.text().replace(/\n$/, "");
+}

yarn.lock

@@ -2544,6 +2544,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@google/generative-ai@npm:^0.21.0":
+  version: 0.21.0
+  resolution: "@google/generative-ai@npm:0.21.0"
+  checksum: 10c0/cff5946c5964f2380e5097d82bd563d79be27a1a5ac604aaaad3f9ba3382992e4f0a371bd255baabfba4e5bdf296d8ce1410cbd65424afa98e64b2590fe49f3b
+  languageName: node
+  linkType: hard
+
 "@graphql-codegen/add@npm:5.0.3, @graphql-codegen/add@npm:^5.0.3":
   version: 5.0.3
   resolution: "@graphql-codegen/add@npm:5.0.3"
@@ -14910,6 +14917,7 @@ __metadata:
     "@commitlint/cli": "npm:^19.3.0"
     "@commitlint/config-conventional": "npm:^19.2.2"
     "@floating-ui/vue": "npm:^1.1.4"
+    "@google/generative-ai": "npm:^0.21.0"
     "@graphql-codegen/add": "npm:5.0.3"
     "@graphql-codegen/cli": "npm:5.0.2"
     "@graphql-codegen/introspection": "npm:4.0.3"