Isolated parser builds

parsers/.gitignore

@@ -0,0 +1,2 @@
+.updated
+node_modules

parsers/_configs/rollup.config.js

@@ -0,0 +1,29 @@
+import resolve from '@rollup/plugin-node-resolve';
+import commonjs from '@rollup/plugin-commonjs';
+import json from '@rollup/plugin-json';
+import {terser} from 'rollup-plugin-terser';
+import globals from 'rollup-plugin-node-globals';
+
+export const output = {
+  // Create a UMD build so that we can require it in node as well to extract
+  // the final version number.
+  format: 'umd',
+  name: 'parser',
+  amd: {
+    id: 'parser',
+  },
+  plugins: [terser()],
+};
+
+export default {
+  input: 'index.js',
+  output,
+  plugins: [
+    resolve({
+      browser: true,
+    }),
+    commonjs(),
+    json(),
+    globals(),
+  ],
+}

parsers/_scripts/link

@@ -0,0 +1,57 @@
+#!/usr/bin/env node
+
+const semver = require('semver');
+const path = require('path');
+const fs = require('fs');
+
+function fatal(msg) {
+  process.stderr.write(msg + '\n')
+  process.exit(1)
+}
+
+function forceSymlink(target, path) {
+  if (fs.existsSync(path)) {
+    fs.unlinkSync(path)
+  }
+  fs.symlinkSync(target, path);
+  console.log(`Linked ${path} -> ${target}`)
+}
+
+const bundlePath = process.argv[2]
+if (!bundlePath) {
+  fatal('No bundle path passed.')
+}
+
+try {
+  const bundle = require(bundlePath);
+  const version = bundle.version;
+  if (!version) {
+    fatal("Unable to determine version.")
+  }
+
+  // Copy bundle to full version
+  const bundleDir = path.dirname(bundlePath)
+  const name = path.basename(bundlePath, '.js')
+  const fullPath = path.join(bundleDir, `${name}@${version}.js`)
+  try {
+    fs.copyFileSync(bundlePath, fullPath)
+    console.log(`Copied ${bundlePath} -> ${fullPath}.`)
+  } catch(e) {
+    fatal('Unable to copy bundle.')
+  }
+
+  if (semver.valid(version)) {
+    try {
+      // Link major version ([email protected] -> [email protected])
+      forceSymlink(fullPath, path.join(bundleDir, `${name}@${semver.major(version)}.js`));
+      // Link major,minor version ([email protected] -> [email protected])
+      forceSymlink(fullPath, path.join(bundleDir, `${name}@${semver.major(version)}.${semver.minor(version)}.js`));
+    } catch (e) {
+      fatal('Unable to link bunlde: ' + e.message)
+    }
+  } else {
+    process.stderr.write('Version is not valid semver, not linking bundle.')
+  }
+} catch(e) {
+  fatal('Unable to load bundle: ' + e.message)
+}

parsers/_scripts/update.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+if [ -e '.ignore' ]; then
+  echo "ignored"
+  exit 0
+fi
+
+if [ -n "$FORCE_UPDATE" -o ! -e .updated ]; then
+  # always update
+  echo "force update"
+  touch .updated
+else
+  # npm outdated --parseable returns data in the format
+  # path:have:wanted:latest
+  # An error doesn't mean that we need to update. We also need to compare 
+  # "have" and "wanted"
+  if ! output=$(npm outdated --parseable); then
+    needs_update=
+    for line in $output; do
+      want=$(echo "$line" | cut -d ':' -f 2)
+      have=$(echo "$line" | cut -d ':' -f 3)
+      if [ "$have" != "$want" ]; then
+        needs_update="${needs_update}$have -> $want\n"
+      fi
+    done
+    if [ -n "$needs_update" ]; then
+      echo "Need update"
+      printf "$needs_update"
+      # Without '--force' npm will refuse to install packages because the
+      # package name is often the same as the parser that is a dependency
+      npm up --force --no-save
+      touch .updated
+      exit 0
+    fi
+  fi
+  echo "packages up-to-date"
+fi

parsers/acorn/Makefile

@@ -0,0 +1,3 @@
+$(OUT_DIR)/%: $(CONFIGS_DIR)/rollup.config.js index.js .updated package.json Makefile
+	$(BIN_DIR)/rollup --config "$<" --file "$@"
+	$(SCRIPTS_DIR)/link "$@"

parsers/acorn/index.js

@@ -0,0 +1,11 @@
+import {Parser as acorn} from 'acorn'
+import {parse as loose} from 'acorn-loose'
+import acornjsx from 'acorn-jsx'
+import {version} from 'acorn/package.json'
+
+export default {
+  acorn,
+  loose,
+  acornjsx,
+  version,
+}

parsers/acorn/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "acorn",
+  "dependencies": {
+    "acorn": "^7.0.0",
+    "acorn-jsx": "^5.2.0",
+    "acorn-loose": "^7.0.0"
+  }
+}

parsers/build.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Helper script build a specific or all parsers
+
+printf "Last run: %s\n\n" "$(date +'%Y-%m-%d %H:%M %z')"
+
+if [ ! -f ./build.sh ]; then
+  echo "You have to set the current working directory to the location of this script" >&2
+  exit 1
+fi
+
+export ROOT_DIR=$(realpath .)
+export OUT_DIR=$(realpath "${OUT_DIR:-../out/parser}")
+export BIN_DIR=$(realpath ./node_modules/.bin)
+export SCRIPTS_DIR=$(realpath ./_scripts)
+export CONFIGS_DIR=$(realpath ./_configs)
+
+for i in "$@"; do
+  case "$i" in
+    --force|-f)
+      export FORCE_UPDATE=1
+      ;;
+    *)
+      dir="$i"
+      ;;
+  esac
+done
+
+## Find all parsers
+for mkfile in $(find . -name 'node_modules' -prune -o -name 'Makefile' -print); do
+  pushd $(dirname $mkfile) > /dev/null
+  printf "#\n# ${PWD##$ROOT_DIR/}\n#\n"
+
+  if ! $SCRIPTS_DIR/update.sh; then
+    echo "Unable to update parser." >&2
+    continue
+  fi
+
+  bundle_name=$(jq -r '.name // ""' package.json)
+  if [ -z "$bundle_name" ]; then
+    echo "Unable to determine bundle name." >&2
+    continue
+  fi
+
+  bundle_path="$OUT_DIR/${bundle_name}.js"
+
+  if ! make "$bundle_path"; then
+    echo "Unable to build bundle." >&2
+    continue
+  fi
+
+  echo
+  popd > /dev/null
+done

parsers/flow/Makefile

@@ -0,0 +1,3 @@
+$(OUT_DIR)/%: index.js .updated package.json Makefile
+	$(BIN_DIR)/browserify -p tinyify --ignore fs --ignore constants "$<" --standalone parser > "$@"
+	$(SCRIPTS_DIR)/link "$@"

parsers/flow/index.js

@@ -0,0 +1,2 @@
+exports.flowParser = require('flow-parser');
+exports.version = require('flow-parser/package.json').version;

parsers/flow/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "flow-parser",
+  "dependencies": {
+    "flow-parser": "^0.*"
+  }
+}

parsers/package.json

@@ -0,0 +1,15 @@
+{
+  "dependencies": {
+    "@rollup/plugin-commonjs": "^11.1.0",
+    "@rollup/plugin-json": "^4.0.2",
+    "@rollup/plugin-node-resolve": "^7.1.3",
+    "browserify": "^16.5.1",
+    "common-shakeify": "^0.6.2",
+    "rollup": "^2.6.1",
+    "rollup-plugin-node-globals": "^1.4.0",
+    "rollup-plugin-replace": "^2.2.0",
+    "rollup-plugin-terser": "^5.3.0",
+    "semver": "^7.3.2",
+    "tinyify": "^2.5.2"
+  }
+}

parsers/typescript/Makefile

@@ -0,0 +1,3 @@
+$(OUT_DIR)/%: $(CONFIGS_DIR)/rollup.config.js index.js .updated package.json Makefile
+	$(BIN_DIR)/rollup --config "$<" --file "$@"
+	$(SCRIPTS_DIR)/link "$@"

parsers/typescript/index.js

@@ -0,0 +1,17 @@
+import typescript from 'typescript'
+import {version} from 'typescript/package.json'
+
+const syntaxKind = {};
+
+for (const name of Object.keys(typescript.SyntaxKind).filter(x => isNaN(parseInt(x)))) {
+    const value = typescript.SyntaxKind[name];
+    if (!syntaxKind[value]) {
+        syntaxKind[value] = name;
+    }
+}
+
+export default {
+  typescript,
+  syntaxKind,
+  version
+}

parsers/typescript/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "typescript",
+  "dependencies": {
+    "typescript": "^3.8.3"
+  }
+}

website/src/components/Toolbar.js

@@ -7,12 +7,12 @@ import TransformButton from './buttons/TransformButton';
 import KeyMapButton from './buttons/KeyMapButton';
 
 export default function Toolbar(props) {
-  let {parser, transformer, showTransformer} = props;
+  let {parser, transformer, showTransformer, parseResult, transformResult} = props;
   let parserInfo = parser.displayName;
   let transformerInfo = '';
   if (parser) {
-    if (parser.version) {
-      parserInfo += '-' + parser.version;
+    if (parseResult && parseResult.version) {
+      parserInfo += '-' + parseResult.version;
     }
     if (parser.homepage) {
       parserInfo =
@@ -21,8 +21,8 @@ export default function Toolbar(props) {
   }
   if (showTransformer) {
     transformerInfo = transformer.displayName;
-    if (transformer.version) {
-      transformerInfo += '-' + transformer.version;
+    if (transformResult && transformResult.version) {
+      transformerInfo += '-' + transformResult.version;
     }
     if (transformer.homepage) {
       transformerInfo =
@@ -69,4 +69,6 @@ Toolbar.propTypes = {
   showTransformer: PropTypes.bool,
   canSave: PropTypes.bool,
   canFork: PropTypes.bool,
+  parseResult: PropTypes.object,
+  transformResult: PropTypes.object,
 };

website/src/containers/ToolbarContainer.js

@@ -28,6 +28,8 @@ function mapStateToProps(state) {
     keyMap: selectors.getKeyMap(state),
     showTransformer: selectors.showTransformer(state),
     snippet: selectors.getRevision(state),
+    parseResult: selectors.getParseResult(state),
+    transformResult: selectors.getTransformResult(state),
   };
 }
 

website/src/parser-loader.js

@@ -0,0 +1,53 @@
+// This is an amd like loader function to load parser bundles that are built
+// separately. See <REPO ROOT>/parsers for more information.
+
+const loading = new Map();
+
+global.define = function(parserID, deps, factory) {
+  const url = document.currentScript.getAttribute('src');
+  const entry = loading.get(url);
+  if (!entry) {
+    console.error(`Tried to load parser '${url}' with lost request.`);
+    return;
+  }
+
+  try {
+    // Should we call resolve outside try...catch ?
+    if (typeof deps === 'function') {
+      factory = deps;
+    }
+    entry.resolve(factory());
+  } catch(error) {
+    entry.reject(error);
+  }
+}
+global.define.amd = true;
+
+export function loadParser(parserID) {
+  const url = `parser/${parserID}.js`;
+  if (loading.has(url)) {
+    return loading.get(url).promise;
+  }
+
+  const entry = {};
+  loading.set(url, entry);
+
+  return entry.promise = new Promise((resolve, reject) => {
+    entry.resolve = resolve;
+    entry.reject  = reject;
+    const script = document.createElement('script');
+    script.onload = function() {
+      document.head.removeChild(this);
+      // Promise will be resolved (or rejected) in the 'define' function called
+      // by the loaded script.
+    };
+    script.onerror = function() {
+      document.head.removeChild(this);
+      // It's OK to call reject here because 'define' won't be called
+      // anyways.
+      reject(new Error(`Unable to load parser "${parserID}" (from ${this.src}). See network tab/developer tools for more information.`));
+    };
+    document.head.appendChild(script);
+    script.src = url;
+  });
+}