CommonMark syntax highlighting _should_ work but I think the remark s…

…erializer is removing all inline HTML

docs/src/foundations/markup_languages/commonmark.common.md

@@ -3,5 +3,12 @@ order: -2
 ---
 # Using CommonMark
 
-{#normal-italic-bold}
-normal _italic_ **bold**
+<aside>some html</aside>
+
+normal _italic_ **bold**
+
+`inline code`
+
+```js
+console.log("hi")
+```

src/input/parseFile.ts

@@ -18,8 +18,9 @@ import { getPandocAST } from "../pandoc.js";
 import { getInputFormatForFileName } from "./fileExtensions.js";
 import { LogCollector } from "../utils/logUtils.js";
 import { fsbase, fsext, fsname, fssplit, refjoin } from "../utils/pathUtils.js";
-import { Root } from "mdast";
 import { mdASTWithoutPositions } from "../utils/astUtils.js";
+import remarkGfm from "remark-gfm";
+import remarkDirective from "remark-directive";
 
 function removeExtensionFromPath(path_: string): string {
   return path_.slice(0, path_.length - path.parse(path_).ext.length);
@@ -54,7 +55,10 @@ export async function parseFile(
 
   let polyglotDoc: PolyglotDoc | undefined;
 
-  const remarkProcessor = unified().use(remarkParse); //.use(remarkGfm);
+  const remarkProcessor = unified()
+    .use(remarkParse)
+    .use(remarkGfm)
+    .use(remarkDirective);
 
   switch (getInputFormatForFileName(fsbase(fsPath), config, frontMatter)) {
     case "djot":

src/plugins/autoTitlePlugin.ts

@@ -1,6 +1,5 @@
 import { Heading } from "@djot/djot";
 import { visit, EXIT } from "unist-util-visit";
-import unist from "unist";
 
 import { applyFilter } from "../engine/djotFiltersPlus.js";
 import { DjockeyDoc, DjockeyPlugin } from "../types.js";

src/plugins/syntaxHighlightingPlugin.ts

@@ -1,10 +1,6 @@
-import {
-  DjockeyConfigResolved,
-  DjockeyDoc,
-  DjockeyPlugin,
-  DjockeyRenderer,
-} from "../types.js";
-import { applyFilter } from "../engine/djotFiltersPlus.js";
+import { Html, Parent } from "mdast";
+import { Node } from "unist";
+import { visit, CONTINUE, SKIP } from "unist-util-visit";
 import {
   AstNode,
   CodeBlock,
@@ -22,10 +18,25 @@ import {
   HighlighterGeneric,
   LanguageRegistration,
 } from "shiki/index.mjs";
+
+import {
+  DjockeyConfigResolved,
+  DjockeyDoc,
+  DjockeyPlugin,
+  DjockeyRenderer,
+  UnistVisitable,
+} from "../types.js";
+import { applyFilter } from "../engine/djotFiltersPlus.js";
 import djotTextmateGrammar from "../djotTextmateGrammar.js";
 import { showPromiseListAsProgressBar } from "../utils/asyncUtils.js";
 import { LogCollector } from "../utils/logUtils.js";
 
+declare module "unist" {
+  interface Data {
+    hlRequestID?: string;
+  }
+}
+
 let nextID = 0;
 
 export class SyntaxHighlightingPlugin implements DjockeyPlugin {
@@ -172,6 +183,55 @@ export class SyntaxHighlightingPlugin implements DjockeyPlugin {
     }));
   }
 
+  readRemarkDoc(
+    doc: UnistVisitable & Parent,
+    djockeyDoc: DjockeyDoc,
+    getIsNodeReservedByAnotherPlugin: (node: AstNode) => boolean
+  ) {
+    // TODO: node reservations
+    visit(doc, "code", (node) => {
+      // if (getIsNodeReservedByAnotherPlugin(node)) return;
+      if (node.data?.hlRequestID) return CONTINUE; // Already scheduled
+
+      const lang = this.getNodeLang(
+        djockeyDoc,
+        node.lang,
+        "default_code_block_language"
+      );
+      if (!lang) return CONTINUE;
+
+      const hlRequestID = `${nextID++}`;
+      this.highlightRequests[hlRequestID] = {
+        text: node.value,
+        lang,
+      };
+
+      node.data = { ...node.data, hlRequestID: hlRequestID };
+
+      console.log("SCHEDULE HL FOR", node);
+      return SKIP;
+    });
+
+    visit(doc, "inlineCode", (node) => {
+      // if (getIsNodeReservedByAnotherPlugin(node)) return;
+      if (node.data?.hlRequestID) return; // Already scheduled
+
+      // In remark we always highlight as plaintext because there's no syntax
+      // for specifying anything else
+
+      const lang = "text";
+
+      const hlRequestID = `${nextID++}`;
+      this.highlightRequests[hlRequestID] = {
+        text: node.value,
+        lang,
+      };
+
+      node.data = { ...node.data, hlRequestID: hlRequestID };
+      return SKIP;
+    });
+  }
+
   onPass_read(args: {
     doc: DjockeyDoc;
     logCollector: LogCollector;
@@ -188,9 +248,12 @@ export class SyntaxHighlightingPlugin implements DjockeyPlugin {
           );
           break;
         case "mdast":
-          args.logCollector.warning(
-            `Syntax highlighting ignoring ${doc.refPath}`
+          this.readRemarkDoc(
+            pDoc.value,
+            doc,
+            args.getIsNodeReservedByAnotherPlugin
           );
+          break;
       }
     }
   }
@@ -226,9 +289,7 @@ export class SyntaxHighlightingPlugin implements DjockeyPlugin {
           this.highlightDjotDoc(pDoc.value);
           break;
         case "mdast":
-          args.logCollector.warning(
-            `Syntax highlighter not rendering ${doc.refPath}`
-          );
+          this.highlightRemarkDoc(pDoc.value);
           break;
       }
     }
@@ -276,4 +337,41 @@ export class SyntaxHighlightingPlugin implements DjockeyPlugin {
       },
     }));
   }
+
+  highlightRemarkDoc(remarkDoc: UnistVisitable & Parent) {
+    visit(remarkDoc, "code", (node: Node) => {
+      console.log("Replacing", node);
+      const hlRequestID = node.data?.hlRequestID;
+      if (!hlRequestID) return CONTINUE;
+
+      const newText = this.highlightResults[hlRequestID];
+      if (!newText) return CONTINUE;
+
+      node.type = "html";
+      (node as Html).value = newText;
+
+      console.log("Result:", node);
+      return SKIP;
+    });
+
+    visit(remarkDoc, "inlineCode", (node: Node) => {
+      const hlRequestID = node.data?.hlRequestID;
+      if (!hlRequestID) return CONTINUE;
+
+      let newText = this.highlightResults[hlRequestID];
+      if (!newText) return CONTINUE;
+
+      // Shiki insists on rendering <pre> tags, so just switch them to <span>
+      const OLD_PREFIX = "<pre ";
+      const OLD_SUFFIX = "</pre>";
+      newText =
+        "<span " +
+        newText.slice(OLD_PREFIX.length, newText.length - OLD_SUFFIX.length) +
+        "</span>";
+
+      node.type = "html";
+      (node as Html).value = newText;
+      return SKIP;
+    });
+  }
 }

src/renderers/htmlRenderer.ts

@@ -3,6 +3,7 @@ import path from "path";
 import fastGlob from "fast-glob";
 import micromatch from "micromatch";
 import { parseFragment, serialize } from "parse5";
+import yaml from "js-yaml";
 
 import { renderHTML } from "@djot/djot";
 import { Environment } from "nunjucks";
@@ -27,11 +28,11 @@ import {
   writeFile,
 } from "../utils/pathUtils.js";
 import { LogCollector } from "../utils/logUtils.js";
-import { mystToHtml } from "myst-to-html";
 import { unified } from "unified";
 import remarkRehype from "remark-rehype";
 import rehypeFormat from "rehype-format";
 import rehypeStringify from "rehype-stringify";
+import { mdASTWithoutPositions } from "../utils/astUtils.js";
 
 export class HTMLRenderer implements DjockeyRenderer {
   identifier: DjockeyOutputFormat = "html";
@@ -185,6 +186,12 @@ export class HTMLRenderer implements DjockeyRenderer {
           break;
         case "mdast":
           const tree = structuredClone(doc.docs[k].value);
+
+          console.log(
+            "Rendering tree\n",
+            yaml.dump(mdASTWithoutPositions(tree))
+          );
+
           const processor = await unified()
             .use(remarkRehype)
             .use(rehypeFormat)

src/types.ts

@@ -66,10 +66,11 @@ export interface DjockeyConfigResolved extends DjockeyConfig {
   link_mappings: LinkMappingConfig[];
 }
 
+export type UnistVisitable = Parameters<typeof visit>[0];
 export type PolyglotDoc_Djot = { kind: "djot"; value: Doc };
 export type PolyglotDoc_MDAST = {
   kind: "mdast";
-  value: Parameters<typeof visit>[0] & Parent;
+  value: UnistVisitable & Parent;
 };
 
 export type PolyglotDoc = PolyglotDoc_Djot | PolyglotDoc_MDAST;

src/utils/astUtils.ts

@@ -1,7 +1,7 @@
 import { Block, Heading, Inline } from "@djot/djot";
 import { PhrasingContent } from "mdast";
 import unist from "unist";
-import { visit } from "unist-util-visit";
+import { visit, EXIT } from "unist-util-visit";
 import { toString } from "mdast-util-to-string";
 
 import { applyFilter } from "../engine/djotFiltersPlus.js";
@@ -38,9 +38,12 @@ export function getFirstHeadingIsAlreadyDocumentTitle(
     case "mdast":
       visit(
         (polyglotDoc as PolyglotDoc_MDAST).value,
-        "Heading",
+        "heading",
         function (node, index, parent) {
-          console.log(node);
+          didFindNode = true;
+          const text = toString(node);
+          returnValue = text === doc.title;
+          return EXIT;
         }
       );
   }