Adding support for SMTP-based emails (#383)

* Introducing smtp sender

* Remove old sendgrid config

* alpha

* Email doc

* copy templates

* Readme and versions

* lock versions

* remove stuff

* Customise quick start

* Moving some options to advanced settings

* Improve docker compose editor

.env.example

@@ -32,10 +32,13 @@ OKTA_SECRET=
 BASE_URL=http://localhost:3000
 SECURE_COOKIES=false
 SENDGRID_API_KEY=
-SENDGRID_SENDER=
-SENDGRID_VERIFICATION_EMAIL_TID=
-SENDGRID_RESET_PASSWORD_TID=
-SENDGRID_SELF_HOST_EMAIL_TID=
+SENDGRID_SENDER=[email protected]
+MAIL_SMTP_HOST=
+MAIL_PORT=465
+MAIL_SECURE=true
+MAIL_USER=
+MAIL_PASSWORD=
+MAIL_SENDER=[email protected]
 STRIPE_SECRET=
 STRIPE_KEY=
 STRIPE_WEBHOOK_SECRET=

.github/workflows/alpha.yml

@@ -2,7 +2,7 @@ name: 'Alpha Build'
 
 on:
   push:
-    branches: [v4140/integration]
+    branches: [v4140/x]
 
 jobs:
   build:

README.md

@@ -89,6 +89,8 @@ This will run a demo version, which you can turn into a fully licenced version b
 
 - Upgrade to React 18
 - Replace icons by emoji for columns headers (fully customisable)
+- Adding SMTP support for self-hosting, in addition to SendGrid
+- Simplification of SendGrid setup, by removing the need of creating email templates. They are now hardcoded.
 
 ### Version 4.13.0
 

backend/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@retrospected/backend",
-  "version": "4.13.0",
+  "version": "4.14.0",
   "license": "GNU GPLv3",
   "private": true,
   "scripts": {
-    "build": "rimraf dist && tsc --build",
+    "build": "rimraf dist && tsc --build && yarn copy-templates",
     "start": "nodemon --exec 'yarn fix & ts-node' --files ./src/index.ts",
     "create-migration": "ts-node ./node_modules/typeorm/cli.js --config src/db/orm-config.ts migration:generate -n ",
     "create-empty-migration": "ts-node ./node_modules/typeorm/cli.js --config src/db/orm-config.ts migration:create -n ",
@@ -15,7 +15,8 @@
     "ci-test": "CI=true yarn test",
     "fix": "eslint 'src/**/*.ts' --fix",
     "backend-production": "yarn migrate-production && cd ./dist/src && node index.js",
-    "migrate-production": "node ./node_modules/typeorm/cli.js --config dist/src/db/orm-config.js migration:run"
+    "migrate-production": "node ./node_modules/typeorm/cli.js --config dist/src/db/orm-config.js migration:run",
+    "copy-templates": "copyfiles -u 0 src/**/*.html dist/"
   },
   "dependencies": {
     "@sendgrid/mail": "7.6.2",
@@ -32,6 +33,7 @@
     "@types/lodash": "4.14.181",
     "@types/node": "17.0.23",
     "@types/node-fetch": "2.5.12",
+    "@types/nodemailer": "6.4.4",
     "@types/passport": "1.0.7",
     "@types/passport-github2": "1.2.5",
     "@types/passport-google-oauth20": "2.0.11",
@@ -46,6 +48,7 @@
     "bcryptjs": "2.4.3",
     "chalk": "4.1.2",
     "connect-redis": "6.1.3",
+    "copyfiles": "2.4.1",
     "crypto-js": "4.1.1",
     "csurf": "1.11.0",
     "date-fns": "2.28.0",
@@ -58,11 +61,13 @@
     "express-rate-limit": "6.3.0",
     "express-session": "1.17.2",
     "freemail": "1.7.0",
+    "handlebars": "4.7.7",
     "jest": "27.5.1",
     "lexorank": "1.0.4",
     "lodash": "4.17.21",
     "moment": "2.29.1",
     "node-fetch": "2.6.7",
+    "nodemailer": "6.7.3",
     "nodemon": "2.0.15",
     "passport": "0.5.0",
     "passport-github2": "0.1.12",
@@ -84,7 +89,7 @@
     "ts-jest": "27.1.4",
     "ts-node": "10.7.0",
     "typeorm": "0.2.45",
-    "typeorm-naming-strategies": "^3.0.0",
+    "typeorm-naming-strategies": "3.0.0",
     "typescript": "4.6.3",
     "uuid": "8.3.2"
   },

backend/src/admin/router.ts

@@ -9,6 +9,7 @@ import { isLicenced } from '../security/is-licenced';
 import { AdminChangePasswordPayload, BackendCapabilities } from '../common';
 import { getIdentityFromRequest, hashPassword } from '../utils';
 import csurf from 'csurf';
+import { canSendEmails } from '../email/utils';
 
 const router = express.Router();
 const csrfProtection = csurf();
@@ -19,11 +20,7 @@ router.get('/self-hosting', async (_, res) => {
     adminEmail: config.SELF_HOSTED_ADMIN,
     selfHosted: config.SELF_HOSTED,
     licenced: !!licence,
-    sendGridAvailable:
-      !!config.SENDGRID_API_KEY &&
-      !!config.SENDGRID_RESET_PASSWORD_TID &&
-      !!config.SENDGRID_VERIFICATION_EMAIL_TID &&
-      !!config.SENDGRID_SENDER,
+    emailAvailable: canSendEmails(),
     disableAnonymous: config.DISABLE_ANONYMOUS_LOGIN,
     disablePasswords: config.DISABLE_PASSWORD_LOGIN,
     disablePasswordRegistration: config.DISABLE_PASSWORD_REGISTRATION,

backend/src/auth/register/register-user.ts

@@ -3,7 +3,7 @@ import { v4 } from 'uuid';
 import { hashPassword } from '../../utils';
 import { UserIdentityEntity } from '../../db/entities';
 import { getIdentityByUsername, registerUser } from '../../db/actions/users';
-import config from '../../config';
+import { canSendEmails } from '../../email/utils';
 
 export default async function registerPasswordUser(
   details: RegisterPayload
@@ -24,10 +24,7 @@ export default async function registerPasswordUser(
     type: 'password',
     username: details.username,
     password: hashedPassword,
-    emailVerification:
-      config.SENDGRID_API_KEY && config.SENDGRID_VERIFICATION_EMAIL_TID
-        ? v4()
-        : undefined,
+    emailVerification: canSendEmails() ? v4() : undefined,
     language: details.language,
   });
 

backend/src/common/payloads.ts

@@ -53,7 +53,7 @@ export interface SelfHostedCheckPayload {
 
 export interface BackendCapabilities {
   selfHosted: boolean;
-  sendGridAvailable: boolean;
+  emailAvailable: boolean;
   adminEmail: string;
   licenced: boolean;
   oAuth: OAuthAvailabilities;

backend/src/config.ts

@@ -85,12 +85,6 @@ const config: BackendConfig = {
   OKTA_SECRET: defaults('OKTA_SECRET', ''),
   SENDGRID_API_KEY: defaults('SENDGRID_API_KEY', ''),
   SENDGRID_SENDER: defaults('SENDGRID_SENDER', ''),
-  SENDGRID_VERIFICATION_EMAIL_TID: defaults(
-    'SENDGRID_VERIFICATION_EMAIL_TID',
-    ''
-  ),
-  SENDGRID_RESET_PASSWORD_TID: defaults('SENDGRID_RESET_PASSWORD_TID', ''),
-  SENDGRID_SELF_HOST_EMAIL_TID: defaults('SENDGRID_SELF_HOST_EMAIL_TID', ''),
   STRIPE_SECRET: defaults('STRIPE_SECRET', ''),
   STRIPE_WEBHOOK_SECRET: defaults('STRIPE_WEBHOOK_SECRET', ''),
   STRIPE_TEAM_PRODUCT: defaults('STRIPE_TEAM_PRODUCT', ''),
@@ -106,6 +100,12 @@ const config: BackendConfig = {
   RATE_LIMIT_WS_POINTS: defaultsNumber('RATE_LIMIT_WS_POINTS', 600),
   RATE_LIMIT_WS_DURATION: defaultsNumber('RATE_LIMIT_WS_DURATION', 60),
   WS_MAX_BUFFER_SIZE: defaultsNumber('WS_MAX_BUFFER_SIZE', 10_000),
+  MAIL_SMTP_HOST: defaults('MAIL_SMTP_HOST', ''),
+  MAIL_PORT: defaultsNumber('MAIL_PORT', 465),
+  MAIL_SECURE: defaultsBool('MAIL_SECURE', true),
+  MAIL_SENDER: defaults('MAIL_SENDER', ''),
+  MAIL_USER: defaults('MAIL_USER', ''),
+  MAIL_PASSWORD: defaults('MAIL_PASSWORD', ''),
 };
 
 export default config;

backend/src/email/emailSender.ts

@@ -1,66 +1,75 @@
-import sendGrid, { MailDataRequired } from '@sendgrid/mail';
+import chalk from 'chalk';
 import config from '../config';
 import randomWords from './random-words';
+import { sendGridSender } from './sendgrid-sender';
+import { smtpSender } from './smtp-sender';
+import {
+  getPasswordResetTemplate,
+  getRegisterTemplate,
+  getSelfHostedWelcomeEmailTemplate,
+} from './template-loader';
+import { EmailSender } from './types';
 
-if (config.SENDGRID_API_KEY) {
-  sendGrid.setApiKey(config.SENDGRID_API_KEY);
+function getSender(): EmailSender | null {
+  if (config.SENDGRID_API_KEY) {
+    console.log(
+      chalk`📨  {red SendGrid} is going to be used for {yellow emails}`
+    );
+    return sendGridSender;
+  } else if (config.MAIL_SMTP_HOST) {
+    console.log(chalk`📨  {red SMTP} is going to be used for {yellow emails}`);
+    return smtpSender;
+  }
+
+  console.log(
+    chalk`📨  {red NO EMAIL PROVIDER CONFIGURED}. It is recommended to setup a SendGrid or SMTP email provider for account management.`
+  );
+  return null;
 }
 
+const send = getSender();
+
 export async function sendVerificationEmail(
   email: string,
   name: string,
   code: string
 ) {
-  if (!config.SENDGRID_API_KEY) {
-    throw Error('Sendgrid is not activated.');
+  if (!send) {
+    return;
   }
-  const msg: MailDataRequired = {
-    to: email,
-    from: config.SENDGRID_SENDER,
-    templateId: config.SENDGRID_VERIFICATION_EMAIL_TID,
-    dynamicTemplateData: {
-      name,
-      code,
-      domain: config.BASE_URL,
-      email: encodeURIComponent(email),
-    },
-  };
-  try {
-    await sendGrid.send(msg);
-  } catch (e) {
-    console.error('Send grid error: ', e);
+  const template = await getRegisterTemplate(
+    encodeURIComponent(email),
+    name,
+    code,
+    config.BASE_URL
+  );
+  const result = send(
+    email,
+    'Welcome to Retrospected - Verify your email',
+    template
+  );
+  if (!result) {
+    console.error('Sending email did not work');
   }
 }
 
-type SendGridError = {
-  response: {
-    body: unknown;
-  };
-};
-
 export async function sendResetPassword(
   email: string,
   name: string,
   code: string
 ) {
-  if (!config.SENDGRID_API_KEY) {
-    throw Error('Sendgrid is not activated.');
+  if (!send) {
+    return;
   }
-  const msg: MailDataRequired = {
-    to: email,
-    from: config.SENDGRID_SENDER,
-    templateId: config.SENDGRID_RESET_PASSWORD_TID,
-    dynamicTemplateData: {
-      name,
-      code,
-      domain: config.BASE_URL,
-      email: encodeURIComponent(email),
-    },
-  };
-  try {
-    await sendGrid.send(msg);
-  } catch (e: unknown) {
-    console.error('Send grid error: ', e, (e as SendGridError).response.body);
+  const template = await getPasswordResetTemplate(
+    encodeURIComponent(email),
+    name,
+    code,
+    config.BASE_URL
+  );
+  const result = send(email, 'Retrospected - Reset your password', template);
+  if (!result) {
+    console.error('Sending email did not work');
   }
 }
 
@@ -73,30 +82,23 @@ export async function sendSelfHostWelcome(
   name: string,
   key: string
 ) {
-  if (!config.SENDGRID_API_KEY) {
-    throw Error('Sendgrid is not activated.');
+  if (!send) {
+    return;
   }
-
   const dbPassword = generatePassword();
   const pgAdminPassword = generatePassword();
   const sessionSecret = generatePassword();
 
-  const msg: MailDataRequired = {
-    to: email,
-    from: config.SENDGRID_SENDER,
-    templateId: config.SENDGRID_SELF_HOST_EMAIL_TID,
-    dynamicTemplateData: {
-      name,
-      key,
-      dbPassword,
-      pgAdminPassword,
-      sessionSecret,
-      email,
-    },
-  };
-  try {
-    await sendGrid.send(msg);
-  } catch (e: unknown) {
-    console.error('Send grid error: ', e, (e as SendGridError).response.body);
+  const template = await getSelfHostedWelcomeEmailTemplate(
+    name,
+    key,
+    dbPassword,
+    pgAdminPassword,
+    sessionSecret,
+    email
+  );
+  const result = send(email, 'Welcome to Retrospected Self-Hosted!', template);
+  if (!result) {
+    console.error('Sending email did not work');
   }
 }

backend/src/email/register.template.html

@@ -0,0 +1,5 @@
+<div>Hello {{name}}!</div>
+<div><br></div>
+<div>Please click on the email below to validate your email and start using Retrospected right away.</div>
+<div><br></div>
+<div><a href="{{domain}}/validate?email={{email}}&amp;code={{code}}">{{domain}}/validate?email={{email}}&amp;code={{code}}</a></div>

backend/src/email/reset-password.template.html

@@ -0,0 +1,5 @@
+<div style="font-family: inherit; text-align: inherit">Hi {{name}},</div>
+<div style="font-family: inherit; text-align: inherit"><br></div>
+<div style="font-family: inherit; text-align: inherit">To reset your password, follow the link:</div>
+<div style="font-family: inherit; text-align: inherit"><br></div>
+<div style="font-family: inherit; text-align: inherit"><a href="{{domain}}/reset?email={{email}}&amp;code={{code}}">{{domain}}/reset?email={{email}}&amp;code={{code}}</a></div>