build: generate proper anchor ids for headings in markdown

Currently we compute the anchor ids for headings in guides and
overviews manually. Our logic here does not match with the ids generated
by Github, so one needs to either fix up all references to the anchor
for the docs, but that will break the links in most markdown editors.

We fix this by using the integrated `slugify` method provided by the
`marked` package. This one seems to match the usual id generation in
markdown files (as in Github and most markdown editors).

This fixes links in the cdk-testing harness overview as observed
originally in #19366.

Author: devversion

package.json

@@ -91,7 +91,7 @@
     "@types/gulp": "3.8.32",
     "@types/inquirer": "^0.0.43",
     "@types/jasmine": "^3.5.4",
-    "@types/marked": "^0.4.2",
+    "@types/marked": "^0.7.4",
     "@types/merge2": "^0.3.30",
     "@types/minimist": "^1.2.0",
     "@types/node": "^12.11.1",
@@ -132,7 +132,7 @@
     "karma-sauce-launcher": "^2.0.2",
     "karma-sourcemap-loader": "^0.3.7",
     "madge": "^3.4.4",
-    "marked": "^0.6.2",
+    "marked": "^1.0.0",
     "merge2": "^1.2.3",
     "minimatch": "^3.0.4",
     "minimist": "^1.2.0",

tools/markdown-to-html/docs-marked-renderer.ts

@@ -1,9 +1,6 @@
-import {Renderer} from 'marked';
+import {Renderer, Slugger} from 'marked';
 import {basename, extname} from 'path';
 
-/** Regular expression that matches whitespace. */
-const whitespaceRegex = /\W+/g;
-
 /** Regular expression that matches example comments. */
 const exampleCommentRegex = /<!--\s*example\(([^)]+)\)\s*-->/g;
 
@@ -13,15 +10,23 @@ const exampleCommentRegex = /<!--\s*example\(([^)]+)\)\s*-->/g;
  */
 export class DocsMarkdownRenderer extends Renderer {
 
+  /** Set of fragment links discovered in the currently rendered file. */
+  private _referencedFragments = new Set<string>();
+
+  /**
+   * Slugger provided by the `marked` package. Can be used to create unique
+   * ids  for headings.
+   */
+  private _slugger = new Slugger();
+
   /**
    * Transforms a markdown heading into the corresponding HTML output. In our case, we
    * want to create a header-link for each H3 and H4 heading. This allows users to jump to
    * specific parts of the docs.
    */
-  heading(label: string, level: number, _raw: string) {
+  heading(label: string, level: number, raw: string) {
     if (level === 3 || level === 4) {
-      const headingId = label.toLowerCase().replace(whitespaceRegex, '-');
-
+      const headingId = this._slugger.slug(raw);
       return `
         <h${level} id="${headingId}" class="docs-header-link">
           <span header-link="${headingId}"></span>
@@ -42,6 +47,11 @@ export class DocsMarkdownRenderer extends Renderer {
       return super.link(`guide/${basename(href, extname(href))}`, title, text);
     }
 
+    // Keep track of all fragments discovered in a file.
+    if (href.startsWith('#')) {
+      this._referencedFragments.add(href.substr(1));
+    }
+
     return super.link(href, title, text);
   }
 
@@ -90,7 +100,26 @@ export class DocsMarkdownRenderer extends Renderer {
    * Method that will be called after a markdown file has been transformed to HTML. This method
    * can be used to finalize the content (e.g. by adding an additional wrapper HTML element)
    */
-  finalizeOutput(output: string): string {
+  finalizeOutput(output: string, fileName: string): string {
+    const failures: string[] = [];
+
+    // Collect any fragment links that do not resolve to existing fragments in the
+    // rendered file. We want to error for broken fragment links.
+    this._referencedFragments.forEach(id => {
+      if (this._slugger.seen[id] === undefined) {
+        failures.push(`Found link to "${id}". This heading does not exist.`);
+      }
+    });
+
+    if (failures.length) {
+      console.error(`Could not process file: ${fileName}. Please fix the following errors:`);
+      failures.forEach(message => console.error(`  -  ${message}`));
+      process.exit(1);
+    }
+
+    this._slugger.seen = {};
+    this._referencedFragments.clear();
+
     return `<div class="docs-markdown">${output}</div>`;
   }
 }

tools/markdown-to-html/transform-markdown.ts

@@ -27,7 +27,8 @@ if (require.main === module) {
   // Bazel bin directory.
   inputFiles.forEach(inputPath => {
     const outputPath = join(bazelBinPath, inputPath.replace(markdownExtension, '.html'));
-    const htmlOutput = markdownRenderer.finalizeOutput(marked(readFileSync(inputPath, 'utf8')));
+    const htmlOutput = markdownRenderer.finalizeOutput(
+        marked(readFileSync(inputPath, 'utf8')), inputPath);
 
     writeFileSync(outputPath, htmlOutput);
   });

yarn.lock

@@ -1324,10 +1324,10 @@
   resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.0.tgz#719551d2352d301ac8b81db732acb6bdc28dbdef"
   integrity sha512-1w52Nyx4Gq47uuu0EVcsHBxZFJgurQ+rTKS3qMHxR1GY2T8c2AJYd6vZoZ9q1rupaDjU0yT+Jc2XTyXkjeMA+Q==
 
-"@types/marked@^0.4.2":
-  version "0.4.2"
-  resolved "https://registry.yarnpkg.com/@types/marked/-/marked-0.4.2.tgz#64a89e53ea37f61cc0f3ee1732c555c2dbf6452f"
-  integrity sha512-cDB930/7MbzaGF6U3IwSQp6XBru8xWajF5PV2YZZeV8DyiliTuld11afVztGI9+yJZ29il5E+NpGA6ooV/Cjkg==
+"@types/marked@^0.7.4":
+  version "0.7.4"
+  resolved "https://registry.yarnpkg.com/@types/marked/-/marked-0.7.4.tgz#607685669bb1bbde2300bc58ba43486cbbee1f0a"
+  integrity sha512-fdg0NO4qpuHWtZk6dASgsrBggY+8N4dWthl1bAQG9ceKUNKFjqpHaDKCAhRUI6y8vavG7hLSJ4YBwJtZyZEXqw==
 
 "@types/merge2@^0.3.30":
   version "0.3.30"
@@ -7776,16 +7776,16 @@ marked@^0.3.2:
   resolved "https://registry.yarnpkg.com/marked/-/marked-0.3.19.tgz#5d47f709c4c9fc3c216b6d46127280f40b39d790"
   integrity sha512-ea2eGWOqNxPcXv8dyERdSr/6FmzvWwzjMxpfGB/sbMccXoct+xY+YukPD+QTUZwyvK7BZwcr4m21WBOW41pAkg==
 
-marked@^0.6.2:
-  version "0.6.2"
-  resolved "https://registry.yarnpkg.com/marked/-/marked-0.6.2.tgz#c574be8b545a8b48641456ca1dbe0e37b6dccc1a"
-  integrity sha512-LqxwVH3P/rqKX4EKGz7+c2G9r98WeM/SW34ybhgNGhUQNKtf1GmmSkJ6cDGJ/t6tiyae49qRkpyTw2B9HOrgUA==
-
 marked@^0.7.0:
   version "0.7.0"
   resolved "https://registry.yarnpkg.com/marked/-/marked-0.7.0.tgz#b64201f051d271b1edc10a04d1ae9b74bb8e5c0e"
   integrity sha512-c+yYdCZJQrsRjTPhUx7VKkApw9bwDkNbHUKo1ovgcfDjb2kc8rLuRbIFyXL5WOEUwzSSKo3IXpph2K6DqB/KZg==
 
+marked@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/marked/-/marked-1.0.0.tgz#d35784245a04871e5988a491e28867362e941693"
+  integrity sha512-Wo+L1pWTVibfrSr+TTtMuiMfNzmZWiOPeO7rZsQUY5bgsxpHesBEcIWJloWVTFnrMXnf/TL30eTFSGJddmQAng==
+
 matchdep@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/matchdep/-/matchdep-2.0.0.tgz#c6f34834a0d8dbc3b37c27ee8bbcb27c7775582e"