Internationalisation, i18next, translating all languages (#403)

* add i18next

* welcome

* typescript

* converted files

* Convert all translations to i18next

* Fix previous game

* French

* Remove some translations

* Remove previous translations

* Re-instate language provider

* crowdin config

* Crowdin config

* Original languages before translations

* Downloading MT translations

* Chinese taditional

* Arabic

* Fix language loading

* Setting language

* Setting language

* Rename all files to make plurals work

* Rename all files to make plurals work

* Missing translations

* Migrate default langues

* clean up

* Linting

* readme, fixes, integration tests

* remove uncessary local storage

* integration

* fixes

* fixes

* Making language nullable

* trying to remove vuln

.github/workflows/alpha.yml

@@ -2,7 +2,7 @@ name: 'Alpha Build'
 
 on:
   push:
-    branches: [v4160/docs]
+    branches: [v4160/i18n]
 
 jobs:
   build:

README.md

@@ -87,6 +87,7 @@ This will run a demo version, which you can turn into a fully licenced version b
 
 ### Version 4.16.0
 
+- Complete overhaul of the translations. Switching to [i18next](https://www.i18next.com). Translated all languages using Machine Learning (via Crowdin).
 - Fix the empty file download when logging using Google OAuth
 - Upgrade the documentation to the latest version of Docusaurus
 - Add more integration tests, covering password accounts and account deletion

backend/.gitignore

@@ -1 +1,2 @@
-/ormconfig.json
+/ormconfig.json
+/src/locales

backend/src/auth/passport.ts

@@ -100,7 +100,6 @@ export default () => {
     return {
       name: profile.displayName,
       type: 'twitter',
-      language: 'en',
       photo: profile.photos?.length ? profile.photos[0].value : undefined,
       username: profile.username,
       email,
@@ -124,7 +123,6 @@ export default () => {
     return {
       name: displayName,
       type: 'github',
-      language: 'en',
       photo: profile.photos?.length ? profile.photos[0].value : undefined,
       username: profile.username,
       email: email.value,
@@ -141,7 +139,6 @@ export default () => {
     return {
       name: profile.displayName,
       type: 'google',
-      language: 'en',
       photo: profile.photos?.length ? profile.photos[0].value : undefined,
       username: email,
       email,
@@ -156,7 +153,6 @@ export default () => {
     return {
       name: profile.displayName,
       type: 'slack',
-      language: 'en',
       photo: profile.user.image_192,
       username: email,
       email,
@@ -172,7 +168,6 @@ export default () => {
     return {
       name: profile.displayName,
       type: 'microsoft',
-      language: 'en',
       username: email,
       email,
     };
@@ -184,7 +179,6 @@ export default () => {
     return {
       name: profile.fullName,
       type: 'okta',
-      language: 'en',
       username: email,
       email,
     };

backend/src/common/types.ts

@@ -130,7 +130,7 @@ export interface FullUser extends User {
   username: string | null;
   accountType: AccountType;
   photo: string | null;
-  language: string;
+  language: string | null;
   email: string | null;
   canDeleteSession: boolean;
   stripeId: string | null;

backend/src/db/actions/users.ts

@@ -191,7 +191,7 @@ export async function registerUser(
       UserIdentityRepository
     );
 
-    const identity = await getOrCreateIdentity(
+    const [identity, existing] = await getOrCreateIdentity(
       manager,
       registration.username,
       registration.email,
@@ -211,7 +211,7 @@ export async function registerUser(
     user.photo = registration.photo || user.photo;
     user.email = registration.email;
 
-    if (registration.language) {
+    if (!existing && registration.language) {
       user.language = registration.language;
     }
 
@@ -244,7 +244,6 @@ export async function registerAnonymousUser(
       const identity = new UserIdentityEntity(v4(), user, hashedPassword);
 
       identity.username = username;
-      user.language = 'en';
 
       await userRepository.save(user);
       await identityRepository.save(identity);
@@ -275,7 +274,7 @@ async function getOrCreateIdentity(
   username: string,
   email: string,
   accountType: AccountType
-): Promise<UserIdentityEntity> {
+): Promise<[identity: UserIdentityEntity, existing: boolean]> {
   const identityRepository = manager.getCustomRepository(
     UserIdentityRepository
   );
@@ -287,33 +286,36 @@ async function getOrCreateIdentity(
     // In certain conditions, the user attached to the identity could be wrong if the user didn't have an email
     const identity = identities[0];
     if (identity.user.email !== email) {
-      const user = await getOrCreateUser(manager, email);
+      const [user, existing] = await getOrCreateUser(manager, email);
       identity.user = user;
+      return [identity, existing];
     }
 
-    return identity;
+    return [identity, true];
   }
 
-  const user = await getOrCreateUser(manager, email);
+  const [user, existing] = await getOrCreateUser(manager, email);
   const identity = new UserIdentityEntity(v4(), user);
 
-  return identity;
+  return [identity, existing];
 }
 
 async function getOrCreateUser(
   manager: EntityManager,
   email: string
-): Promise<UserEntity> {
+): Promise<[identity: UserEntity, existing: boolean]> {
   const userRepository = manager.getCustomRepository(UserRepository);
   const existingUser = await userRepository.findOne({
     where: { email },
   });
   if (existingUser) {
-    return existingUser;
+    return [existingUser, true];
   }
   const user = new UserEntity(v4(), '');
   user.email = email;
-  return await userRepository.saveAndReload(user);
+  const savedUser = await userRepository.saveAndReload(user);
+
+  return [savedUser, false];
 }
 
 async function updateUserPassword(

backend/src/db/entities/User.ts

@@ -46,8 +46,8 @@ export default class UserEntity {
   public trial: Date | null;
   @Column({ nullable: false, default: 50 })
   public quota: number;
-  @Column({ nullable: false, type: 'character varying', default: 'en' })
-  public language: string;
+  @Column({ nullable: true, type: 'character varying' })
+  public language: string | null;
   @ManyToOne(() => SessionTemplateEntity, { nullable: true, eager: false })
   @Index()
   public defaultTemplate: SessionTemplateEntity | null | undefined;
@@ -73,7 +73,7 @@ export default class UserEntity {
     this.id = id;
     this.name = name;
     this.email = null;
-    this.language = 'en';
+    this.language = null;
     this.stripeId = null;
     this.currency = null;
     this.trial = null;

backend/src/db/entities/UserView.ts

@@ -50,7 +50,7 @@ export default class UserView {
   @ViewColumn()
   public photo: string | null;
   @ViewColumn()
-  public language: string;
+  public language: string | null;
   @ViewColumn()
   public ownSubscriptionsId: string | null;
   @ViewColumn()
@@ -72,7 +72,7 @@ export default class UserView {
     this.id = id;
     this.identityId = identityId;
     this.name = name;
-    this.language = 'en';
+    this.language = null;
     this.accountType = 'anonymous';
     this.username = null;
     this.photo = null;

backend/src/db/migrations/1652023488716-MigrateLanguages.ts

@@ -0,0 +1,39 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+type Mapping = { from: string; to: string };
+
+const mappings: Mapping[] = [
+  { from: 'ar', to: 'ar-SA' },
+  { from: 'de', to: 'de-DE' },
+  { from: 'en', to: 'en-GB' },
+  { from: 'es', to: 'es-ES' },
+  { from: 'fr', to: 'fr-FR' },
+  { from: 'hu', to: 'hu-HU' },
+  { from: 'it', to: 'it-IT' },
+  { from: 'ja', to: 'ja-JP' },
+  { from: 'nl', to: 'nl-NL' },
+  { from: 'pl', to: 'pl-PL' },
+  { from: 'pt', to: 'pt-BR' },
+  { from: 'ptbr', to: 'pt-PT' },
+  { from: 'ru', to: 'uk-UA' },
+  { from: 'zhcn', to: 'zh-CN' },
+  { from: 'zhtw', to: 'zh-TW' },
+];
+
+export class MigrateLanguages1652023488716 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    for (const mapping of mappings) {
+      await queryRunner.query(
+        `UPDATE users SET language = '${mapping.to}' WHERE language = '${mapping.from}';`
+      );
+    }
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    for (const mapping of mappings) {
+      await queryRunner.query(
+        `UPDATE users SET language = '${mapping.from}' WHERE language = '${mapping.to}';`
+      );
+    }
+  }
+}

backend/src/db/migrations/1652028782451-DefaultLanguage.ts

@@ -0,0 +1,14 @@
+import {MigrationInterface, QueryRunner} from "typeorm";
+
+export class DefaultLanguage1652028782451 implements MigrationInterface {
+    name = 'DefaultLanguage1652028782451'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "users" ALTER COLUMN "language" SET DEFAULT 'en-GB'`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "users" ALTER COLUMN "language" SET DEFAULT 'en'`);
+    }
+
+}

backend/src/db/migrations/1652033290129-NullLanguage.ts

@@ -0,0 +1,16 @@
+import {MigrationInterface, QueryRunner} from "typeorm";
+
+export class NullLanguage1652033290129 implements MigrationInterface {
+    name = 'NullLanguage1652033290129'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "users" ALTER COLUMN "language" DROP NOT NULL`);
+        await queryRunner.query(`ALTER TABLE "users" ALTER COLUMN "language" DROP DEFAULT`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "users" ALTER COLUMN "language" SET DEFAULT 'en-GB'`);
+        await queryRunner.query(`ALTER TABLE "users" ALTER COLUMN "language" SET NOT NULL`);
+    }
+
+}

crowdin.yml

@@ -0,0 +1,11 @@
+"project_id" : "512896"
+"base_path" : "."
+"base_url" : "https://api.crowdin.com"
+"preserve_hierarchy": true
+
+files: [
+ {
+  "source" : "/frontend/src/translations/locales/en-GB.json",
+  "translation" : "/frontend/src/translations/locales/%locale%.json"
+ }
+]

dependencies.md

@@ -9,4 +9,3 @@
 - @types/node-fetch: 2.5.12
 - redis 3.1.2 (new version incompatible with express-redis)
 - passport 0.5.0 (new version, including 0.5.2 breaks set user when using Docker, but not locally)
-

docs/docs/self-hosting/optionals.md

@@ -107,7 +107,7 @@ services:
       GA_ID: '' # Optional, Google Analytics ID (UA-1234456-7)
       SENTRY_URL: '' # Optional, Sentry URL (https://[email protected]/1234567)
       GIPHY_API_KEY: '' # Optional, can be obtained here: https://developers.giphy.com/
-      DEFAULT_LANGUAGE: 'en' # Set the default language for new users
+      DEFAULT_LANGUAGE: 'en-GB' # Set the default language for new users
 
       # -- Do Not Change --
       BACKEND_HOST: backend # This should be the name of the backend service

frontend/Dockerfile

@@ -25,7 +25,10 @@ COPY --chown=node:node ./.env ./.env
 
 RUN yarn build
 
-FROM nginx:alpine
+FROM nginx:latest
+
+RUN apt update
+RUN apt -y remove libfreetype6
 
 COPY --from=Node /home/node/app/build /usr/share/nginx/html
 COPY ./docker/nginx.conf.template /etc/nginx/conf.d/default.conf.template

frontend/docker/frontend-entrypoint.sh

@@ -10,6 +10,6 @@ sed -i "s#NO_GA#${GA_ID:-}#g" /usr/share/nginx/html/index.html
 sed -i "s#NO_SENTRY#${SENTRY_URL:-}#g" /usr/share/nginx/html/index.html
 sed -i "s#NO_GIPHY#${GIPHY_API_KEY:-}#g" /usr/share/nginx/html/index.html
 sed -i "s#NO_STRIPE#${STRIPE_KEY:-}#g" /usr/share/nginx/html/index.html
-sed -i "s#NO_DEFAULT_LANGUAGE#${DEFAULT_LANGUAGE:-en}#g" /usr/share/nginx/html/index.html
+sed -i "s#NO_DEFAULT_LANGUAGE#${DEFAULT_LANGUAGE:-en-GB}#g" /usr/share/nginx/html/index.html
 
 exec "$@"

frontend/package.json

@@ -40,6 +40,9 @@
     "emoji-mart": "3.0.1",
     "flag-icons": "6.2.0",
     "http-proxy-middleware": "2.0.6",
+    "i18next": "^21.6.16",
+    "i18next-browser-languagedetector": "^6.1.4",
+    "i18next-resources-to-backend": "^1.0.0",
     "isemail": "3.2.0",
     "lexorank": "1.0.4",
     "lodash": "4.17.21",
@@ -57,6 +60,7 @@
     "react-ga": "3.3.0",
     "react-giphy-searchbox": "1.5.4",
     "react-helmet": "6.1.0",
+    "react-i18next": "^11.16.7",
     "react-markdown": "8.0.3",
     "react-password-strength-bar": "0.4.0",
     "react-router-dom": "6.3.0",