refactor(components-example): extract regions from component example source files (#19376)

* refactor: functionality to extract regions from component example source files

* add README for region-parser

* lint issues

* add more descriptive comments for changes in bazel

* calculate file extension from base path instead of passing it in

* remove expanded field

Co-authored-by: Annie Wang <[email protected]>

Author: annieyw

src/components-examples/BUILD.bazel

@@ -72,7 +72,7 @@ filegroup(
 )
 
 highlight_files(
-    name = "highlighted-source-files",
+    name = "examples-highlighted",
     srcs = [":example-source-files"],
     tags = ["docs-package"],
 )
@@ -97,16 +97,20 @@ package_docs_content(
 
         # For the live-examples in our docs, we want to package the highlighted files
         # into the docs content. These will be used to show the source code for examples.
-        ":highlighted-source-files": "examples-highlighted",
+        # Note: `examples-highlighted` is a tree artifact that we want to store as is
+        # in the docs-content. Hence there is no target section name.
+        ":examples-highlighted": "",
     },
     tags = ["docs-package"],
 )
 
 ng_package(
     name = "npm_package",
     srcs = ["package.json"],
-    data = [":docs-content"],
     entry_point = ":public-api.ts",
+    # this is a workaround to store a tree artifact in the ng_package.
+    # ng_package does not properly handle tree artifacts currently so we escalate to nested_packages
+    nested_packages = [":docs-content"],
     tags = ["docs-package"],
     deps = [":components-examples"] + ALL_EXAMPLES,
 )

tools/highlight-files/BUILD.bazel

@@ -8,6 +8,8 @@ ts_library(
     srcs = glob(["**/*.ts"]),
     tsconfig = ":tsconfig.json",
     deps = [
+        "//tools/region-parser",
+        "@npm//@types/fs-extra",
         "@npm//@types/node",
     ],
 )
@@ -16,6 +18,7 @@ nodejs_binary(
     name = "highlight-files",
     data = [
         ":sources",
+        "@npm//fs-extra",
         "@npm//highlight.js",
     ],
     entry_point = ":highlight-files.ts",

tools/highlight-files/highlight-files.ts

@@ -3,9 +3,10 @@
  * multiple input files using highlight.js. The output will be HTML files.
  */
 
-import {readFileSync, writeFileSync} from 'fs';
-import {extname, join} from 'path';
+import {readFileSync, writeFileSync, ensureDirSync} from 'fs-extra';
+import {dirname, extname, join, relative} from 'path';
 import {highlightCodeBlock} from './highlight-code-block';
+import {regionParser} from '../region-parser/region-parser';
 
 /**
  * Determines the command line arguments for the current Bazel action. Since this action can
@@ -25,21 +26,48 @@ function getBazelActionArguments() {
   return args;
 }
 
+function detectAndHighlightRegionBlocks(parsed:
+                                          { contents: string, regions: { [p: string]: string } },
+                                        basePath: string,
+                                        outDir: string) {
+  const fileExtension = extname(basePath).substring(1);
+  for (const [regionName, regionSnippet] of Object.entries(parsed.regions)) {
+    // Create files for each found region
+    if (!regionName) {
+      continue;
+    }
+    const highlightedRegion = highlightCodeBlock(regionSnippet, fileExtension);
+    // Convert "my-component-example.ts" into "my-component-example_region-ts.html"
+    const regionBaseOutputPath = basePath.replace(`.${fileExtension}`,
+      `_${regionName}-${fileExtension}.html`);
+    const regionOutputPath = join(outDir, regionBaseOutputPath);
+    ensureDirSync(dirname(regionOutputPath));
+    writeFileSync(regionOutputPath, highlightedRegion);
+  }
+}
+
 if (require.main === module) {
-  // The script expects the bazel-bin path as first argument. All remaining arguments will be
-  // considered as markdown input files that need to be transformed.
-  const [bazelBinPath, ...inputFiles] = getBazelActionArguments();
-
-  // Walk through each input file and write transformed markdown output to the specified
-  // Bazel bin directory.
-  inputFiles.forEach(inputPath => {
-    const fileExtension = extname(inputPath).substring(1);
-    // Convert "my-component-example.ts" into "my-component-example-ts.html"
-    const baseOutputPath = inputPath.replace(`.${fileExtension}`, `-${fileExtension}.html`);
-    const outputPath = join(bazelBinPath, baseOutputPath);
-    const htmlOutput = highlightCodeBlock(readFileSync(inputPath, 'utf8'), fileExtension);
-
-    writeFileSync(outputPath, htmlOutput);
-  });
+    // The script expects the output directory as first argument. Second is the name of the
+    // package where this the highlight target is declared. All remaining arguments will be
+    // considered as markdown input files that need to be transformed.
+    const [outDir, packageName, ...inputFiles] = getBazelActionArguments();
+
+    // Walk through each input file and write transformed markdown output
+    // to the specified output directory.
+    for (const execPath of inputFiles) {
+        // Compute a relative path from the package to the actual input file.
+        // e.g `src/components-examples/cdk/<..>/example.ts` becomes `cdk/<..>/example.ts`.
+        const basePath = relative(packageName, execPath);
+        const fileExtension = extname(basePath).substring(1);
+        const parsed = regionParser(readFileSync(execPath, 'utf8'), fileExtension);
+        detectAndHighlightRegionBlocks(parsed, basePath, outDir);
+        // Convert "my-component-example.ts" into "my-component-example-ts.html"
+        const baseOutputPath = basePath.replace(`.${fileExtension}`, `-${fileExtension}.html`);
+        const outputPath = join(outDir, baseOutputPath);
+        const htmlOutput = highlightCodeBlock(parsed.contents, fileExtension);
+
+        ensureDirSync(dirname(outputPath));
+        writeFileSync(outputPath, htmlOutput);
+    }
 
 }

tools/highlight-files/index.bzl

@@ -1,13 +1,3 @@
-"""
-  Gets a path relative to the specified label. This is achieved by just removing the label
-  package path from the specified path. e.g. the path is "guides/test/my-text.md" and the
-  label package is "guides/". The expected path would be "test/my-text.md".
-"""
-
-def _relative_to_label(label, short_path):
-    # TODO(devversion): extract into generic utility under tools/
-    return short_path[len(label.package) + 1:]
-
 """
   Implementation of the "highlight_files" rule. The implementation runs the
   highlight-files executable in order to highlight the specified source files.
@@ -16,7 +6,7 @@ def _relative_to_label(label, short_path):
 def _highlight_files(ctx):
     input_files = ctx.files.srcs
     args = ctx.actions.args()
-    expected_outputs = []
+    output_dir = ctx.actions.declare_directory(ctx.label.name)
 
     # Do nothing if there are no input files. Bazel will throw if we schedule an action
     # that returns no outputs.
@@ -29,41 +19,27 @@ def _highlight_files(ctx):
     # Read more here: https://docs.bazel.build/versions/master/skylark/lib/Args.html#use_param_file
     args.use_param_file(param_file_arg = "--param-file=%s")
 
-    # Add the bazel bin directory to the command arguments. The script needs to know about
-    # the output directory because the input files are not in the same location as the bazel
-    # bin directory.
-    args.add(ctx.bin_dir.path)
-
-    for input_file in input_files:
-        # Extension of the input file (e.g. "ts" or "css")
-        file_extension = input_file.extension
-
-        # Determine the input file path relatively to the current package path. This is necessary
-        # because we want to preserve directories for the input files and `declare_file` expects a
-        # path that is relative to the current package. We remove the file extension including the dot
-        # because we will constructo an output file using a different extension.
-        relative_basepath = _relative_to_label(ctx.label, input_file.short_path)[:-len(file_extension) - 1]
+    # Add the output directory path to the command arguments
+    args.add(output_dir.path)
 
-        # Construct the output path from the relative basepath and file extension. For example:
-        # "autocomplete.ts" should result in "autocomplete-ts.html".
-        expected_outputs += [
-            ctx.actions.declare_file("%s-%s.html" % (relative_basepath, file_extension)),
-        ]
+    # Add the name of the label package. This will be used in the
+    # action to compute package-relative paths.
+    args.add(ctx.label.package)
 
-        # Add the path for the input file to the command line arguments, so that the executable
-        # can process it.
-        args.add(input_file.path)
+    # Add the input files to the command arguments. These files will
+    # be processed by the highlight binary.
+    args.add_all(input_files)
 
     # Run the highlight-files executable that highlights the specified source files.
     ctx.actions.run(
         inputs = input_files,
         executable = ctx.executable._highlight_files,
-        outputs = expected_outputs,
+        outputs = [output_dir],
         arguments = [args],
         progress_message = "HighlightFiles",
     )
 
-    return DefaultInfo(files = depset(expected_outputs))
+    return DefaultInfo(files = depset([output_dir]))
 
 """
   Rule definition for the "highlight_files" rule that can accept arbritary source files

tools/markdown-to-html/docs-marked-renderer.ts

@@ -64,15 +64,13 @@ export class DocsMarkdownRenderer extends Renderer {
    *   {
    *    "example": "exampleName",
    *    "file": "example-html.html",
-   *    "lines": [5, 10],
-   *    "expanded": true
+   *    "region": "some-region",
    *   }
    *  ) -->`
    *  turns into
    *  `<div material-docs-example="exampleName"
    *        file="example-html.html"
-   *        lines="[5, 10]"
-   *        expanded="true"></div>`
+   *        region="some-region"></div>`
    *
    *  (old API)
    *  `<!-- example(name) -->`
@@ -82,11 +80,10 @@ export class DocsMarkdownRenderer extends Renderer {
   html(html: string) {
         html = html.replace(exampleCommentRegex, (_match: string, content: string) => {
               if (content.startsWith('{')) {
-                  const {example, file, lines, expanded} = JSON.parse(content);
+                  const {example, file, region} = JSON.parse(content);
                   return `<div material-docs-example="${example}"
                                file="${file}"
-                               lines="${JSON.stringify(lines)}"
-                               expanded="${!!expanded}"></div>`;
+                               region="${region}"></div>`;
               } else {
                   return `<div material-docs-example="${content}"></div>`;
               }

tools/package-docs-content/BUILD.bazel

@@ -7,6 +7,7 @@ nodejs_binary(
     name = "package-docs-content",
     data = [
         ":sources",
+        "@npm//fs-extra",
     ],
     entry_point = ":package-docs-content.ts",
 )
@@ -15,5 +16,8 @@ ts_library(
     name = "sources",
     srcs = glob(["**/*.ts"]),
     tsconfig = ":tsconfig.json",
-    deps = ["@npm//@types/node"],
+    deps = [
+        "@npm//@types/fs-extra",
+        "@npm//@types/node",
+    ],
 )

tools/package-docs-content/index.bzl

@@ -4,22 +4,25 @@
 """
 
 def _package_docs_content(ctx):
-    # Directory that will contain all grouped input files. This directory will be created
-    # relatively to the current target package. (e.g. "bin/src/components-examples/docs-content")
-    output_dir = ctx.attr.name
-
     # Arguments that will be passed to the packager executable.
     args = ctx.actions.args()
 
-    # List of outputs that should be generated by the packager action. Bazel will automatically
-    # throw an error if any output has not been generated properly.
-    expected_outputs = []
+    # Directory that will contain all grouped input files. This directory will be
+    # created relatively to the current target package. For example:
+    # "bin/src/components-examples/docs-content/docs-content". The reason we need to
+    # repeat `docs-content` is that the ng_package rule does not properly handle tree
+    # artifacts in data. Instead, we create a tree artifact that can be put into nested_packages.
+    # Nested packages do not preserve the tree artifact name (i.e. the directory name),
+    # so all contents of the docs-content would be put directly into the @angular/components-examples package.
+    # To avoid that, we create another folder like docs-content in the tree artifact that
+    # is preserved as content of the tree artifact.
+    output_dir = ctx.actions.declare_directory("%s/%s" % (ctx.attr.name, ctx.attr.name))
 
     # Support passing arguments through a parameter file. This is necessary because on Windows
     # there is an argument limit and we need to handle a large amount of input files. Bazel
     # switches between parameter file and normal argument passing based on the operating system.
     # Read more here: https://docs.bazel.build/versions/master/skylark/lib/Args.html#use_param_file
-    args.use_param_file(param_file_arg = "--param-file=%s")
+    args.use_param_file(param_file_arg = "--param-file=%s", use_always = True)
 
     # Walk through each defined input target and the associated section and compute the
     # output file which will be added to the executable arguments.
@@ -29,25 +32,25 @@ def _package_docs_content(ctx):
 
         for input_file in section_files:
             # Creates a relative path from the input file. We don't want to include the full
-            # path in docs content. e.g. `docs-content/overviews/cdk/src/cdk/a11y/a11y.html`.
-            # Instead, we want the path to be: `docs-content/overviews/cdk/a11y/a11y.html`.
-            section_relative_file_name = input_file.short_path[len(base_dir):]
-
-            # For each input file, we want to create a copy that is stored in the output directory
-            # within its specified section. e.g. "pkg_bin/docs-content/guides/getting-started.html"
-            output_file = ctx.actions.declare_file(
-                "%s/%s/%s" % (output_dir, section_name, section_relative_file_name),
-            )
+            # path in docs content. e.g. `/docs-content/overviews/cdk/src/cdk/a11y/a11y.html`. Instead,
+            # we want the path to be: `/docs-content/overviews/cdk/a11y/a11y.html`.
+            section_relative_file_name = input_file.short_path[len(base_dir) + 1:]
 
-            # Add the output file to the expected outputs so that Bazel throws an error if the file
-            # hasn't been generated properly.
-            expected_outputs += [output_file]
+            # The section name can be empty. This is reasonable when tree artifacts are copied
+            # over to the resulting package so that only their contents are transferred.
+            # TODO(devversion): Revisit if this can be improved so that the section name
+            # is respected. `args.add_all` can unfold tree artifact contents.
+            # https://cs.opensource.google/bazel/bazel/+/master:src/main/java/com/google/devtools/build/lib/analysis/skylark/Args.java;l=381-382;drc=9a6997d595fbf3447e911346034edfbde7d8b57e?q=addAll&ss=bazel
+            if section_name:
+                expected_out_path = "%s/%s/%s" % (output_dir.path, section_name, section_relative_file_name)
+            else:
+                expected_out_path = "%s/%s" % (output_dir.path, section_relative_file_name)
 
             # Pass the input file path and the output file path to the packager executable. We need
             # to do this for each file because we cannot determine the general path to the output
             # directory in a reliable way because Bazel targets cannot just "declare" a directory.
             # See: https://docs.bazel.build/versions/master/skylark/lib/actions.html
-            args.add("%s,%s" % (input_file.path, output_file.path))
+            args.add("%s,%s" % (input_file.path, expected_out_path))
 
     # Do nothing if there are no input files. Bazel will throw if we schedule an action
     # that returns no outputs.
@@ -59,12 +62,12 @@ def _package_docs_content(ctx):
     ctx.actions.run(
         inputs = ctx.files.srcs,
         executable = ctx.executable._packager,
-        outputs = expected_outputs,
+        outputs = [output_dir],
         arguments = [args],
         progress_message = "PackageDocsContent",
     )
 
-    return DefaultInfo(files = depset(expected_outputs))
+    return DefaultInfo(files = depset([output_dir]))
 
 """
   Rule definition for the "package_docs_content" rule that can accept arbritary source files

tools/package-docs-content/package-docs-content.ts

@@ -5,7 +5,8 @@
  * multiple times.
  */
 
-import {readFileSync, writeFileSync} from 'fs';
+import {readFileSync, writeFileSync, ensureDirSync, statSync, copySync} from 'fs-extra';
+import {dirname} from 'path';
 
 /**
  * Determines the command line arguments for the current Bazel action. Since this action can
@@ -29,10 +30,19 @@ if (require.main === module) {
   // Process all file pairs that have been passed to this executable. Each argument will
   // consist of the input file path and the desired output location.
   getBazelActionArguments().forEach(argument => {
-    // Each argument that has been passed consists of an input file path and the expected
-    // output path. e.g. {path_to_input_file},{expected_output_path}
-    const [inputFilePath, outputPath] = argument.split(',', 2);
+      // Each argument that has been passed consists of an input file path and the expected
+      // output path. e.g. {path_to_input_file},{expected_output_path}
+      const [execFilePath, expectedOutput] = argument.split(',', 2);
+
+      // Ensure the directory exists. Bazel does not create the tree
+      // artifact by default.
+       ensureDirSync(dirname(expectedOutput));
+
+      if (statSync(execFilePath).isDirectory()) {
+        copySync(execFilePath, expectedOutput);
+      } else {
+        writeFileSync(expectedOutput, readFileSync(execFilePath, 'utf8'));
+      }
 
-    writeFileSync(outputPath, readFileSync(inputFilePath, 'utf8'));
   });
 }

tools/region-parser/BUILD.bazel

@@ -0,0 +1,12 @@
+load("//tools:defaults.bzl", "ts_library")
+
+package(default_visibility = ["//visibility:public"])
+
+ts_library(
+    name = "region-parser",
+    srcs = glob(["**/*.ts"]),
+    tsconfig = ":tsconfig.json",
+    deps = [
+        "@npm//@types/node",
+    ],
+)

tools/region-parser/README.md

@@ -0,0 +1,3 @@
+The implementation of `region-parser` is copied from the `angular/angular` repo. 
+It has been adapted from JavaScript to Typescript along with a few minor changes.
+This is part of an ongoing effort to share more components between [angular.io] and [material.angular.io].

tools/region-parser/region-matchers/block-c.ts

@@ -0,0 +1,5 @@
+// These kind of comments are used CSS and other languages that do not support inline comments
+export const blockC = {
+  regionStartMatcher: /^\s*\/\*\s*#docregion\s*(.*)\s*\*\/\s*$/,
+  regionEndMatcher: /^\s*\/\*\s*#enddocregion\s*(.*)\s*\*\/\s*$/,
+};