From 1f79708ea3d2d64ae94ea0661e00687edd10b345 Mon Sep 17 00:00:00 2001 From: Matiss Janis Aboltins Date: Tue, 7 Nov 2023 18:10:16 +0000 Subject: [PATCH] :sparkles: ability to add migrations (#267) --- .gitignore | 2 + app.js | 11 ++- jest.config.json | 3 +- jest.global-setup.js | 10 +++ jest.global-teardown.js | 5 ++ jest.setup.js | 17 ----- migrations/1694360000000-create-folders.js | 24 ++++++ migrations/1694360479680-create-account-db.js | 30 ++++++++ .../1694362247011-create-secret-table.js | 16 ++++ package.json | 1 + src/account-db.js | 23 +----- src/app-account.js | 4 - .../services/gocardless-service.js | 45 ++++++----- src/app-sync.js | 3 - src/app.js | 11 --- src/db.js | 2 +- src/migrations.js | 30 ++++++++ src/services/secrets-service.js | 36 --------- src/sql/account.sql | 17 ----- src/sql/secrets.sql | 4 - upcoming-release-notes/267.md | 6 ++ yarn.lock | 74 ++++++++++++++++++- 22 files changed, 230 insertions(+), 144 deletions(-) create mode 100644 jest.global-setup.js create mode 100644 jest.global-teardown.js delete mode 100644 jest.setup.js create mode 100644 migrations/1694360000000-create-folders.js create mode 100644 migrations/1694360479680-create-account-db.js create mode 100644 migrations/1694362247011-create-secret-table.js create mode 100644 src/migrations.js delete mode 100644 src/sql/account.sql delete mode 100644 src/sql/secrets.sql create mode 100644 upcoming-release-notes/267.md diff --git a/.gitignore b/.gitignore index e114b191b..ad1e69fb0 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,8 @@ build/ *.pem *.key artifacts.json +.migrate +.migrate-test # Yarn .pnp.* diff --git a/app.js b/app.js index b76fc06eb..4e254b5da 100644 --- a/app.js +++ b/app.js @@ -1,6 +1,9 @@ import run from './src/app.js'; +import runMigrations from './src/migrations.js'; -run().catch((err) => { - console.log('Error starting app:', err); - process.exit(1); -}); +runMigrations() + .then(run) + .catch((err) => { + console.log('Error starting app:', err); + process.exit(1); + }); diff --git a/jest.config.json b/jest.config.json index 8785cee1a..95fb27235 100644 --- a/jest.config.json +++ b/jest.config.json @@ -1,5 +1,6 @@ { - "setupFiles": ["./jest.setup.js"], + "globalSetup": "./jest.global-setup.js", + "globalTeardown": "./jest.global-teardown.js", "testPathIgnorePatterns": ["dist", "/node_modules/", "/build/"], "roots": [""], "moduleFileExtensions": ["ts", "js", "json"], diff --git a/jest.global-setup.js b/jest.global-setup.js new file mode 100644 index 000000000..36f53cf1c --- /dev/null +++ b/jest.global-setup.js @@ -0,0 +1,10 @@ +import getAccountDb from './src/account-db.js'; +import runMigrations from './src/migrations.js'; + +export default async function setup() { + await runMigrations(); + + // Insert a fake "valid-token" fixture that can be reused + const db = getAccountDb(); + await db.mutate('INSERT INTO sessions (token) VALUES (?)', ['valid-token']); +} diff --git a/jest.global-teardown.js b/jest.global-teardown.js new file mode 100644 index 000000000..4e19fc385 --- /dev/null +++ b/jest.global-teardown.js @@ -0,0 +1,5 @@ +import runMigrations from './src/migrations.js'; + +export default async function teardown() { + await runMigrations('down'); +} diff --git a/jest.setup.js b/jest.setup.js deleted file mode 100644 index 18208ceb8..000000000 --- a/jest.setup.js +++ /dev/null @@ -1,17 +0,0 @@ -import fs from 'node:fs'; -import path from 'node:path'; -import getAccountDb from './src/account-db.js'; -import config from './src/load-config.js'; - -// Delete previous test database (force creation of a new one) -const dbPath = path.join(config.serverFiles, 'account.sqlite'); -if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath); - -// Create path for test user files and delete previous files there -if (fs.existsSync(config.userFiles)) - fs.rmSync(config.userFiles, { recursive: true }); -fs.mkdirSync(config.userFiles); - -// Insert a fake "valid-token" fixture that can be reused -const db = getAccountDb(); -db.mutate('INSERT INTO sessions (token) VALUES (?)', ['valid-token']); diff --git a/migrations/1694360000000-create-folders.js b/migrations/1694360000000-create-folders.js new file mode 100644 index 000000000..2ba62b8df --- /dev/null +++ b/migrations/1694360000000-create-folders.js @@ -0,0 +1,24 @@ +import fs from 'node:fs/promises'; +import config from '../src/load-config.js'; + +async function ensureExists(path) { + try { + await fs.mkdir(path); + } catch (err) { + if (err.code == 'EEXIST') { + return null; + } + + throw err; + } +} + +export const up = async function () { + await ensureExists(config.serverFiles); + await ensureExists(config.userFiles); +}; + +export const down = async function () { + await fs.rm(config.serverFiles, { recursive: true, force: true }); + await fs.rm(config.userFiles, { recursive: true, force: true }); +}; diff --git a/migrations/1694360479680-create-account-db.js b/migrations/1694360479680-create-account-db.js new file mode 100644 index 000000000..fd57b79cc --- /dev/null +++ b/migrations/1694360479680-create-account-db.js @@ -0,0 +1,30 @@ +import getAccountDb from '../src/account-db.js'; + +export const up = async function () { + await getAccountDb().exec(` + CREATE TABLE IF NOT EXISTS auth + (password TEXT PRIMARY KEY); + + CREATE TABLE IF NOT EXISTS sessions + (token TEXT PRIMARY KEY); + + CREATE TABLE IF NOT EXISTS files + (id TEXT PRIMARY KEY, + group_id TEXT, + sync_version SMALLINT, + encrypt_meta TEXT, + encrypt_keyid TEXT, + encrypt_salt TEXT, + encrypt_test TEXT, + deleted BOOLEAN DEFAULT FALSE, + name TEXT); + `); +}; + +export const down = async function () { + await getAccountDb().exec(` + DROP TABLE auth; + DROP TABLE sessions; + DROP TABLE files; + `); +}; diff --git a/migrations/1694362247011-create-secret-table.js b/migrations/1694362247011-create-secret-table.js new file mode 100644 index 000000000..2f60a0815 --- /dev/null +++ b/migrations/1694362247011-create-secret-table.js @@ -0,0 +1,16 @@ +import getAccountDb from '../src/account-db.js'; + +export const up = async function () { + await getAccountDb().exec(` + CREATE TABLE IF NOT EXISTS secrets ( + name TEXT PRIMARY KEY, + value BLOB + ); + `); +}; + +export const down = async function () { + await getAccountDb().exec(` + DROP TABLE secrets; + `); +}; diff --git a/package.json b/package.json index edd1868e7..71a378ba5 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "express-rate-limit": "^6.7.0", "express-response-size": "^0.0.3", "jws": "^4.0.0", + "migrate": "^2.0.0", "nordigen-node": "^1.2.6", "uuid": "^9.0.0" }, diff --git a/src/account-db.js b/src/account-db.js index 3c1e5845c..b33fee867 100644 --- a/src/account-db.js +++ b/src/account-db.js @@ -1,34 +1,15 @@ -import fs from 'node:fs'; import { join } from 'node:path'; import openDatabase from './db.js'; -import config, { sqlDir } from './load-config.js'; -import createDebug from 'debug'; +import config from './load-config.js'; import * as uuid from 'uuid'; import * as bcrypt from 'bcrypt'; -const debug = createDebug('actual:account-db'); - let _accountDb = null; export default function getAccountDb() { if (_accountDb == null) { - if (!fs.existsSync(config.serverFiles)) { - debug(`creating server files directory: '${config.serverFiles}'`); - fs.mkdirSync(config.serverFiles); - } - - let dbPath = join(config.serverFiles, 'account.sqlite'); - let needsInit = !fs.existsSync(dbPath); - + const dbPath = join(config.serverFiles, 'account.sqlite'); _accountDb = openDatabase(dbPath); - - if (needsInit) { - debug(`initializing account database: '${dbPath}'`); - let initSql = fs.readFileSync(join(sqlDir, 'account.sql'), 'utf8'); - _accountDb.exec(initSql); - } else { - debug(`opening account database: '${dbPath}'`); - } } return _accountDb; diff --git a/src/app-account.js b/src/app-account.js index a5cd0b8a2..f992a3dd6 100644 --- a/src/app-account.js +++ b/src/app-account.js @@ -13,10 +13,6 @@ app.use(errorMiddleware); export { app as handlers }; -export function init() { - // eslint-disable-previous-line @typescript-eslint/no-empty-function -} - // Non-authenticated endpoints: // // /needs-bootstrap diff --git a/src/app-gocardless/services/gocardless-service.js b/src/app-gocardless/services/gocardless-service.js index 66f464270..7e67330ce 100644 --- a/src/app-gocardless/services/gocardless-service.js +++ b/src/app-gocardless/services/gocardless-service.js @@ -17,17 +17,12 @@ import jwt from 'jws'; import { SecretName, secretsService } from '../../services/secrets-service.js'; const GoCardlessClient = nordigenNode.default; -const goCardlessClient = new GoCardlessClient({ - secretId: secretsService.get(SecretName.nordigen_secretId), - secretKey: secretsService.get(SecretName.nordigen_secretKey), -}); - -secretsService.onUpdate(SecretName.nordigen_secretId, (newSecret) => { - goCardlessClient.secretId = newSecret; -}); -secretsService.onUpdate(SecretName.nordigen_secretKey, (newSecret) => { - goCardlessClient.secretKey = newSecret; -}); + +const getGocardlessClient = () => + new GoCardlessClient({ + secretId: secretsService.get(SecretName.nordigen_secretId), + secretKey: secretsService.get(SecretName.nordigen_secretKey), + }); export const handleGoCardlessError = (response) => { switch (response.status_code) { @@ -58,7 +53,9 @@ export const goCardlessService = { * @returns {boolean} */ isConfigured: () => { - return !!(goCardlessClient.secretId && goCardlessClient.secretKey); + return !!( + getGocardlessClient().secretId && getGocardlessClient().secretKey + ); }, /** @@ -76,7 +73,7 @@ export const goCardlessService = { return clockTimestamp >= payload.exp; }; - if (isExpiredJwtToken(goCardlessClient.token)) { + if (isExpiredJwtToken(getGocardlessClient().token)) { // Generate new access token. Token is valid for 24 hours // Note: access_token is automatically injected to other requests after you successfully obtain it const tokenData = await client.generateToken(); @@ -479,25 +476,25 @@ export const goCardlessService = { */ export const client = { getBalances: async (accountId) => - await goCardlessClient.account(accountId).getBalances(), + await getGocardlessClient().account(accountId).getBalances(), getTransactions: async ({ accountId, dateFrom, dateTo }) => - await goCardlessClient.account(accountId).getTransactions({ + await getGocardlessClient().account(accountId).getTransactions({ dateFrom, dateTo, country: undefined, }), getInstitutions: async (country) => - await goCardlessClient.institution.getInstitutions({ country }), + await getGocardlessClient().institution.getInstitutions({ country }), getInstitutionById: async (institutionId) => - await goCardlessClient.institution.getInstitutionById(institutionId), + await getGocardlessClient().institution.getInstitutionById(institutionId), getDetails: async (accountId) => - await goCardlessClient.account(accountId).getDetails(), + await getGocardlessClient().account(accountId).getDetails(), getMetadata: async (accountId) => - await goCardlessClient.account(accountId).getMetadata(), + await getGocardlessClient().account(accountId).getMetadata(), getRequisitionById: async (requisitionId) => - await goCardlessClient.requisition.getRequisitionById(requisitionId), + await getGocardlessClient().requisition.getRequisitionById(requisitionId), deleteRequisition: async (requisitionId) => - await goCardlessClient.requisition.deleteRequisition(requisitionId), + await getGocardlessClient().requisition.deleteRequisition(requisitionId), initSession: async ({ redirectUrl, institutionId, @@ -509,7 +506,7 @@ export const client = { redirectImmediate, accountSelection, }) => - await goCardlessClient.initSession({ + await getGocardlessClient().initSession({ redirectUrl, institutionId, referenceId, @@ -520,7 +517,7 @@ export const client = { redirectImmediate, accountSelection, }), - generateToken: async () => await goCardlessClient.generateToken(), + generateToken: async () => await getGocardlessClient().generateToken(), exchangeToken: async ({ refreshToken }) => - await goCardlessClient.exchangeToken({ refreshToken }), + await getGocardlessClient().exchangeToken({ refreshToken }), }; diff --git a/src/app-sync.js b/src/app-sync.js index b95e8262f..5d036c3ab 100644 --- a/src/app-sync.js +++ b/src/app-sync.js @@ -15,9 +15,6 @@ const app = express(); app.use(errorMiddleware); export { app as handlers }; -// eslint-disable-next-line -export async function init() {} - // This is a version representing the internal format of sync // messages. When this changes, all sync files need to be reset. We // will check this version when syncing and notify the user if they diff --git a/src/app.js b/src/app.js index 6d588d238..4c20da83d 100644 --- a/src/app.js +++ b/src/app.js @@ -71,17 +71,6 @@ function parseHTTPSConfig(value) { } export default async function run() { - if (!fs.existsSync(config.serverFiles)) { - fs.mkdirSync(config.serverFiles); - } - - if (!fs.existsSync(config.userFiles)) { - fs.mkdirSync(config.userFiles); - } - - await accountApp.init(); - await syncApp.init(); - if (config.https) { const https = await import('node:https'); const httpsOptions = { diff --git a/src/db.js b/src/db.js index 80696fa00..a4d57a6a9 100644 --- a/src/db.js +++ b/src/db.js @@ -27,7 +27,7 @@ class WrappedDatabase { * @param {string} sql */ exec(sql) { - this.db.exec(sql); + return this.db.exec(sql); } /** diff --git a/src/migrations.js b/src/migrations.js new file mode 100644 index 000000000..66f59d4a6 --- /dev/null +++ b/src/migrations.js @@ -0,0 +1,30 @@ +import migrate from 'migrate'; +import config from './load-config.js'; + +export default function run(direction = 'up') { + console.log( + `Checking if there are any migrations to run for direction "${direction}"...`, + ); + + return new Promise((resolve) => + migrate.load( + { + stateStore: `.migrate${config.mode === 'test' ? '-test' : ''}`, + }, + (err, set) => { + if (err) { + throw err; + } + + set[direction]((err) => { + if (err) { + throw err; + } + + console.log('Migrations: DONE'); + resolve(); + }); + }, + ), + ); +} diff --git a/src/services/secrets-service.js b/src/services/secrets-service.js index 9e229588d..ee64fc5c4 100644 --- a/src/services/secrets-service.js +++ b/src/services/secrets-service.js @@ -1,7 +1,4 @@ import createDebug from 'debug'; -import fs from 'node:fs'; -import { sqlDir } from '../load-config.js'; -import { join } from 'node:path'; import getAccountDb from '../account-db.js'; /** @@ -18,18 +15,6 @@ class SecretsDb { constructor() { this.debug = createDebug('actual:secrets-db'); this.db = null; - this.initialize(); - } - - initialize() { - if (!this.db) { - this.db = this.open(); - } - - this.debug(`initializing secrets table'`); - //Create secret table if it doesn't exist - const initSql = fs.readFileSync(join(sqlDir, 'secrets.sql'), 'utf8'); - this.db.exec(initSql); } open() { @@ -64,7 +49,6 @@ class SecretsDb { const secretsDb = new SecretsDb(); const _cachedSecrets = new Map(); -const _observers = new Map(); /** * A service for managing secrets stored in `secretsDb`. */ @@ -78,25 +62,6 @@ export const secretsService = { return _cachedSecrets.get(name) ?? secretsDb.get(name)?.value ?? null; }, - /** - * Callbacks new secret value when a secret changes. - * @param {SecretName} name - The name of the secret to retrieve. - * @param {function(string): void} callback - The new secret value callback. - * @returns {void} - */ - onUpdate: (name, callback) => { - const observers = _observers.get(name) ?? []; - observers.push(callback); - _observers.set(name, observers); - }, - - _notifyObservers: (name, value) => { - const observers = _observers.get(name) ?? []; - for (const observer of observers) { - observer(value); - } - }, - /** * Sets the value of a secret by name. * @param {SecretName} name - The name of the secret to set. @@ -108,7 +73,6 @@ export const secretsService = { if (result.changes === 1) { _cachedSecrets.set(name, value); - secretsService._notifyObservers(name, value); } return result; }, diff --git a/src/sql/account.sql b/src/sql/account.sql deleted file mode 100644 index 8fe266200..000000000 --- a/src/sql/account.sql +++ /dev/null @@ -1,17 +0,0 @@ - -CREATE TABLE auth - (password TEXT PRIMARY KEY); - -CREATE TABLE sessions - (token TEXT PRIMARY KEY); - -CREATE TABLE files - (id TEXT PRIMARY KEY, - group_id TEXT, - sync_version SMALLINT, - encrypt_meta TEXT, - encrypt_keyid TEXT, - encrypt_salt TEXT, - encrypt_test TEXT, - deleted BOOLEAN DEFAULT FALSE, - name TEXT); diff --git a/src/sql/secrets.sql b/src/sql/secrets.sql deleted file mode 100644 index 0cf078614..000000000 --- a/src/sql/secrets.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE TABLE IF NOT EXISTS secrets ( - name TEXT PRIMARY KEY, - value BLOB -); diff --git a/upcoming-release-notes/267.md b/upcoming-release-notes/267.md new file mode 100644 index 000000000..24c96b581 --- /dev/null +++ b/upcoming-release-notes/267.md @@ -0,0 +1,6 @@ +--- +category: Enhancements +authors: [MatissJanis] +--- + +Ability to add and run database/fs migrations diff --git a/yarn.lock b/yarn.lock index c90494235..a9d0b629a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1638,6 +1638,7 @@ __metadata: express-response-size: ^0.0.3 jest: ^29.3.1 jws: ^4.0.0 + migrate: ^2.0.0 nordigen-node: ^1.2.6 prettier: ^2.8.3 supertest: ^6.3.1 @@ -2129,7 +2130,7 @@ __metadata: languageName: node linkType: hard -"chalk@npm:^4.0.0": +"chalk@npm:^4.0.0, chalk@npm:^4.1.2": version: 4.1.2 resolution: "chalk@npm:4.1.2" dependencies: @@ -2256,6 +2257,13 @@ __metadata: languageName: node linkType: hard +"commander@npm:^2.20.3": + version: 2.20.3 + resolution: "commander@npm:2.20.3" + checksum: ab8c07884e42c3a8dbc5dd9592c606176c7eb5c1ca5ff274bcf907039b2c41de3626f684ea75ccf4d361ba004bbaff1f577d5384c155f3871e456bdf27becf9e + languageName: node + linkType: hard + "component-emitter@npm:^1.3.0": version: 1.3.0 resolution: "component-emitter@npm:1.3.0" @@ -2358,6 +2366,13 @@ __metadata: languageName: node linkType: hard +"dateformat@npm:^4.6.3": + version: 4.6.3 + resolution: "dateformat@npm:4.6.3" + checksum: c3aa0617c0a5b30595122bc8d1bee6276a9221e4d392087b41cbbdf175d9662ae0e50d0d6dcdf45caeac5153c4b5b0844265f8cd2b2245451e3da19e39e3b65d + languageName: node + linkType: hard + "dayjs@npm:^1.11.3": version: 1.11.7 resolution: "dayjs@npm:1.11.7" @@ -2526,6 +2541,13 @@ __metadata: languageName: node linkType: hard +"dotenv@npm:^16.0.0": + version: 16.3.1 + resolution: "dotenv@npm:16.3.1" + checksum: 15d75e7279018f4bafd0ee9706593dd14455ddb71b3bcba9c52574460b7ccaf67d5cf8b2c08a5af1a9da6db36c956a04a1192b101ee102a3e0cf8817bbcf3dfd + languageName: node + linkType: hard + "ecdsa-sig-formatter@npm:1.0.11": version: 1.0.11 resolution: "ecdsa-sig-formatter@npm:1.0.11" @@ -4447,6 +4469,29 @@ __metadata: languageName: node linkType: hard +"migrate@npm:^2.0.0": + version: 2.0.0 + resolution: "migrate@npm:2.0.0" + dependencies: + chalk: ^4.1.2 + commander: ^2.20.3 + dateformat: ^4.6.3 + dotenv: ^16.0.0 + inherits: ^2.0.3 + minimatch: ^9.0.1 + mkdirp: ^3.0.1 + slug: ^8.2.2 + bin: + migrate: bin/migrate + migrate-create: bin/migrate-create + migrate-down: bin/migrate-down + migrate-init: bin/migrate-init + migrate-list: bin/migrate-list + migrate-up: bin/migrate-up + checksum: d7e5f476d32c638e7c6ee15e36e0a049f69ad3cb011011623658defa656cba2c75c0128d005ed0a06ea6d6426200297d809d1338db63cd580c03e8d42001a7bd + languageName: node + linkType: hard + "mime-db@npm:1.52.0": version: 1.52.0 resolution: "mime-db@npm:1.52.0" @@ -4513,6 +4558,15 @@ __metadata: languageName: node linkType: hard +"minimatch@npm:^9.0.1": + version: 9.0.3 + resolution: "minimatch@npm:9.0.3" + dependencies: + brace-expansion: ^2.0.1 + checksum: 253487976bf485b612f16bf57463520a14f512662e592e95c571afdab1442a6a6864b6c88f248ce6fc4ff0b6de04ac7aa6c8bb51e868e99d1d65eb0658a708b5 + languageName: node + linkType: hard + "minimist@npm:^1.2.0, minimist@npm:^1.2.3": version: 1.2.6 resolution: "minimist@npm:1.2.6" @@ -4624,6 +4678,15 @@ __metadata: languageName: node linkType: hard +"mkdirp@npm:^3.0.1": + version: 3.0.1 + resolution: "mkdirp@npm:3.0.1" + bin: + mkdirp: dist/cjs/src/bin.js + checksum: 972deb188e8fb55547f1e58d66bd6b4a3623bf0c7137802582602d73e6480c1c2268dcbafbfb1be466e00cc7e56ac514d7fd9334b7cf33e3e2ab547c16f83a8d + languageName: node + linkType: hard + "ms@npm:2.0.0": version: 2.0.0 resolution: "ms@npm:2.0.0" @@ -5503,6 +5566,15 @@ __metadata: languageName: node linkType: hard +"slug@npm:^8.2.2": + version: 8.2.3 + resolution: "slug@npm:8.2.3" + bin: + slug: cli.js + checksum: eb2fbf8d13df0a94f09ffd7c20e02d5e88c1fdd51e178fe8e670937747dec5a97efd172956b914efd5eb3fadd65a306a68f4eed0327a8c5c9a8af30f2c95a46b + languageName: node + linkType: hard + "smart-buffer@npm:^4.2.0": version: 4.2.0 resolution: "smart-buffer@npm:4.2.0"