Introduce separate parser bundles

I don't think that the way astexplorer currently works (building
everything as a single web app) is very maintainable. Bundling so many
packages seems to be destined to fail at some point.

With this commit I'm trying a new approach: Building separate bundles
for parsers. The main reasoning behind the current approach is:
- Instead of having to dictate with which tool a parser bundle needs to
  be generated, every parser has its own build process. This would allow
  us to choose the simplest approach/tool for a specific parser. My hope
  is that this won't turn into a maintenance nightmare because the
  configuration for every parser can stay rather simple, but we'll see.
- The main parser build script can be run periodically on the server and
  automatically update bundles to the latest version. Because they are
  built separately, it doesn't require too many resources on the server
  either.

I plan to roll this out to more parsers over time. Things will probably
have to change to accommodate every parser and maybe even more stuff to
accommodate transforms.

Also I will need to find a way to automatically deploy updates to the
`parsers/` directory.

parsers/.gitignore

@@ -0,0 +1,2 @@
+.updated
+node_modules

parsers/_configs/rollup.config.js

@@ -0,0 +1,29 @@
+import resolve from '@rollup/plugin-node-resolve';
+import commonjs from '@rollup/plugin-commonjs';
+import json from '@rollup/plugin-json';
+import {terser} from 'rollup-plugin-terser';
+import globals from 'rollup-plugin-node-globals';
+
+export const output = {
+  // Create a UMD build so that we can require it in node as well to extract
+  // the final version number.
+  format: 'umd',
+  name: 'parser',
+  amd: {
+    id: 'parser',
+  },
+  plugins: [terser()],
+};
+
+export default {
+  input: 'index.js',
+  output,
+  plugins: [
+    resolve({
+      browser: true,
+    }),
+    commonjs(),
+    json(),
+    globals(),
+  ],
+}

parsers/_scripts/link

@@ -0,0 +1,57 @@
+#!/usr/bin/env node
+
+const semver = require('semver');
+const path = require('path');
+const fs = require('fs');
+
+function fatal(msg) {
+  process.stderr.write(msg + '\n')
+  process.exit(1)
+}
+
+function forceSymlink(target, path) {
+  if (fs.existsSync(path)) {
+    fs.unlinkSync(path)
+  }
+  fs.symlinkSync(target, path);
+  console.log(`Linked ${path} -> ${target}`)
+}
+
+const bundlePath = process.argv[2]
+if (!bundlePath) {
+  fatal('No bundle path passed.')
+}
+
+try {
+  const bundle = require(bundlePath);
+  const version = bundle.version;
+  if (!version) {
+    fatal("Unable to determine version.")
+  }
+
+  // Copy bundle to full version
+  const bundleDir = path.dirname(bundlePath)
+  const name = path.basename(bundlePath, '.js')
+  const fullPath = path.join(bundleDir, `${name}@${version}.js`)
+  try {
+    fs.copyFileSync(bundlePath, fullPath)
+    console.log(`Copied ${bundlePath} -> ${fullPath}.`)
+  } catch(e) {
+    fatal('Unable to copy bundle.')
+  }
+
+  if (semver.valid(version)) {
+    try {
+      // Link major version ([email protected] -> [email protected])
+      forceSymlink(fullPath, path.join(bundleDir, `${name}@${semver.major(version)}.js`));
+      // Link major,minor version ([email protected] -> [email protected])
+      forceSymlink(fullPath, path.join(bundleDir, `${name}@${semver.major(version)}.${semver.minor(version)}.js`));
+    } catch (e) {
+      fatal('Unable to link bunlde: ' + e.message)
+    }
+  } else {
+    process.stderr.write('Version is not valid semver, not linking bundle.')
+  }
+} catch(e) {
+  fatal('Unable to load bundle: ' + e.message)
+}

parsers/_scripts/update.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+if [ -e '.ignore' ]; then
+  echo "ignored"
+  exit 0
+fi
+
+if [ -n "$FORCE_UPDATE" -o ! -e .updated ]; then
+  # always update
+  echo "force update"
+  touch .updated
+else
+  # npm outdated --parseable returns data in the format
+  # path:have:wanted:latest
+  # An error doesn't mean that we need to update. We also need to compare 
+  # "have" and "wanted"
+  if ! output=$(npm outdated --parseable); then
+    needs_update=
+    for line in $output; do
+      want=$(echo "$line" | cut -d ':' -f 2)
+      have=$(echo "$line" | cut -d ':' -f 3)
+      if [ "$have" != "$want" ]; then
+        needs_update="${needs_update}$have -> $want\n"
+      fi
+    done
+    if [ -n "$needs_update" ]; then
+      echo "Need update"
+      printf "$needs_update"
+      # Without '--force' npm will refuse to install packages because the
+      # package name is often the same as the parser that is a dependency
+      npm up --force --no-save
+      touch .updated
+      exit 0
+    fi
+  fi
+  echo "packages up-to-date"
+fi

parsers/acorn/Makefile

@@ -0,0 +1,3 @@
+$(OUT_DIR)/%: $(CONFIGS_DIR)/rollup.config.js index.js .updated package.json Makefile
+	$(BIN_DIR)/rollup --config "$<" --file "$@"
+	$(SCRIPTS_DIR)/link "$@"

parsers/acorn/index.js

@@ -0,0 +1,11 @@
+import {Parser as acorn} from 'acorn'
+import {parse as loose} from 'acorn-loose'
+import acornjsx from 'acorn-jsx'
+import {version} from 'acorn/package.json'
+
+export default {
+  acorn,
+  loose,
+  acornjsx,
+  version,
+}

parsers/acorn/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "acorn",
+  "dependencies": {
+    "acorn": "^7.0.0",
+    "acorn-jsx": "^5.2.0",
+    "acorn-loose": "^7.0.0"
+  }
+}

parsers/build.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Helper script build a specific or all parsers
+
+printf "Last run: %s\n\n" "$(date +'%Y-%m-%d %H:%M %z')"
+
+if [ ! -f ./build.sh ]; then
+  echo "You have to set the current working directory to the location of this script" >&2
+  exit 1
+fi
+
+export ROOT_DIR=$(realpath .)
+export OUT_DIR=$(realpath "${OUT_DIR:-../out/parser}")
+export BIN_DIR=$(realpath ./node_modules/.bin)
+export SCRIPTS_DIR=$(realpath ./_scripts)
+export CONFIGS_DIR=$(realpath ./_configs)
+
+for i in "$@"; do
+  case "$i" in
+    --force|-f)
+      export FORCE_UPDATE=1
+      ;;
+    *)
+      dir="$i"
+      ;;
+  esac
+done
+
+## Find all parsers
+for mkfile in $(find . -name 'node_modules' -prune -o -name 'Makefile' -print); do
+  pushd $(dirname $mkfile) > /dev/null
+  printf "#\n# ${PWD##$ROOT_DIR/}\n#\n"
+
+  if ! $SCRIPTS_DIR/update.sh; then
+    echo "Unable to update parser." >&2
+    continue
+  fi
+
+  bundle_name=$(jq -r '.name // ""' package.json)
+  if [ -z "$bundle_name" ]; then
+    echo "Unable to determine bundle name." >&2
+    continue
+  fi
+
+  bundle_path="$OUT_DIR/${bundle_name}.js"
+
+  if ! make "$bundle_path"; then
+    echo "Unable to build bundle." >&2
+    continue
+  fi
+
+  echo
+  popd > /dev/null
+done

parsers/flow/Makefile

@@ -0,0 +1,3 @@
+$(OUT_DIR)/%: index.js .updated package.json Makefile
+	$(BIN_DIR)/browserify -p tinyify --ignore fs --ignore constants "$<" --standalone parser > "$@"
+	$(SCRIPTS_DIR)/link "$@"

parsers/flow/index.js

@@ -0,0 +1,2 @@
+exports.flowParser = require('flow-parser');
+exports.version = require('flow-parser/package.json').version;

parsers/flow/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "flow-parser",
+  "dependencies": {
+    "flow-parser": "^0.*"
+  }
+}

parsers/package.json

@@ -0,0 +1,15 @@
+{
+  "dependencies": {
+    "@rollup/plugin-commonjs": "^11.1.0",
+    "@rollup/plugin-json": "^4.0.2",
+    "@rollup/plugin-node-resolve": "^7.1.3",
+    "browserify": "^16.5.1",
+    "common-shakeify": "^0.6.2",
+    "rollup": "^2.6.1",
+    "rollup-plugin-node-globals": "^1.4.0",
+    "rollup-plugin-replace": "^2.2.0",
+    "rollup-plugin-terser": "^5.3.0",
+    "semver": "^7.3.2",
+    "tinyify": "^2.5.2"
+  }
+}

parsers/typescript/Makefile

@@ -0,0 +1,3 @@
+$(OUT_DIR)/%: $(CONFIGS_DIR)/rollup.config.js index.js .updated package.json Makefile
+	$(BIN_DIR)/rollup --config "$<" --file "$@"
+	$(SCRIPTS_DIR)/link "$@"

parsers/typescript/index.js

@@ -0,0 +1,17 @@
+import typescript from 'typescript'
+import {version} from 'typescript/package.json'
+
+const syntaxKind = {};
+
+for (const name of Object.keys(typescript.SyntaxKind).filter(x => isNaN(parseInt(x)))) {
+    const value = typescript.SyntaxKind[name];
+    if (!syntaxKind[value]) {
+        syntaxKind[value] = name;
+    }
+}
+
+export default {
+  typescript,
+  syntaxKind,
+  version
+}

parsers/typescript/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "typescript",
+  "dependencies": {
+    "typescript": "^3.8.3"
+  }
+}

website/src/parser-loader.js

@@ -0,0 +1,53 @@
+// This is an amd like loader function to load parser bundles that are built
+// separately. See <REPO ROOT>/parsers for more information.
+
+const loading = new Map();
+
+global.define = function(parserID, deps, factory) {
+  const url = document.currentScript.getAttribute('src');
+  const entry = loading.get(url);
+  if (!entry) {
+    console.error(`Tried to load parser '${url}' with lost request.`);
+    return;
+  }
+
+  try {
+    // Should we call resolve outside try...catch ?
+    if (typeof deps === 'function') {
+      factory = deps;
+    }
+    entry.resolve(factory());
+  } catch(error) {
+    entry.reject(error);
+  }
+}
+global.define.amd = true;
+
+export function loadParser(parserID) {
+  const url = `parser/${parserID}.js`;
+  if (loading.has(url)) {
+    return loading.get(url).promise;
+  }
+
+  const entry = {};
+  loading.set(url, entry);
+
+  return entry.promise = new Promise((resolve, reject) => {
+    entry.resolve = resolve;
+    entry.reject  = reject;
+    const script = document.createElement('script');
+    script.onload = function() {
+      document.head.removeChild(this);
+      // Promise will be resolved (or rejected) in the 'define' function called
+      // by the loaded script.
+    };
+    script.onerror = function() {
+      document.head.removeChild(this);
+      // It's OK to call reject here because 'define' won't be called
+      // anyways.
+      reject(new Error(`Unable to load parser "${parserID}" (from ${this.src}). See network tab/developer tools for more information.`));
+    };
+    document.head.appendChild(script);
+    script.src = url;
+  });
+}

website/src/parsers/js/acorn.js

@@ -1,6 +1,6 @@
 import React from 'react';
 import defaultParserInterface from './utils/defaultESTreeParserInterface';
-import pkg from 'acorn/package.json';
+import {loadParser} from '../../parser-loader.js';
 
 const ID = 'acorn';
 
@@ -9,29 +9,22 @@ export default {
 
   id: ID,
   displayName: ID,
-  version: `${pkg.version}`,
-  homepage: pkg.homepage,
+  homepage: 'https://github.com/acornjs/acorn',
   locationProps: new Set(['range', 'loc', 'start', 'end']),
 
-  loadParser(callback) {
-    require(['acorn', 'acorn-loose', 'acorn-jsx'], (acorn, acornLoose, acornJsx) => {
-      callback({
-        acorn,
-        acornLoose,
-        acornJsx,
-      });
-    });
+  loadParser() {
+    return loadParser('acorn@7')
   },
 
   parse(parsers, code, options={}) {
     let parser;
     if (options['plugins.jsx'] && !options.loose) {
-      const cls = parsers.acorn.Parser.extend(parsers.acornJsx());
+      const cls = parsers.acorn.extend(parsers.acornjsx());
       parser = cls.parse.bind(cls);
     } else {
       parser = options.loose ?
-        parsers.acornLoose.parse:
-        parsers.acorn.parse;
+        parsers.loose :
+        parsers.acorn.parse.bind(parsers.acorn);
     }
 
     return parser(code, options);

website/src/parsers/js/flow.js

@@ -1,5 +1,5 @@
 import defaultParserInterface from './utils/defaultESTreeParserInterface';
-import pkg from 'flow-parser/package.json';
+import {loadParser} from '../../parser-loader.js';
 
 const ID = 'flow';
 export const defaultOptions = {
@@ -32,15 +32,15 @@ export default {
 
   id: ID,
   displayName: ID,
-  version: pkg.version,
-  homepage: pkg.homepage || 'https://flowtype.org/',
+  homepage: 'https://flowtype.org/',
   locationProps: new Set(['range', 'loc']),
 
-  loadParser(callback) {
-    require(['flow-parser'], callback);
+  loadParser() {
+    return loadParser('flow-parser@0')
   },
 
-  parse(flowParser, code, options) {
+  parse({flowParser}, code, options) {
+    console.log(flowParser);
     return flowParser.parse(code, options);
   },