feat(@angular-devkit/build-angular): add experimental Rollup concatenation option

Author: filipesilva

packages/angular_devkit/build_angular/package.json

@@ -41,6 +41,7 @@
     "postcss-loader": "3.0.0",
     "raw-loader": "3.1.0",
     "regenerator-runtime": "0.13.3",
+    "rollup": "1.21.4",
     "rxjs": "6.5.3",
     "sass": "1.22.12",
     "sass-loader": "8.0.0",

packages/angular_devkit/build_angular/src/angular-cli-files/models/build-options.ts

@@ -81,6 +81,8 @@ export interface BuildOptions {
 
   /* When specified it will be used instead of the script target in the tsconfig.json. */
   scriptTargetOverride?: ScriptTarget;
+
+  experimentalRollupConcatenation?: boolean;
 }
 
 export interface WebpackTestOptions extends BuildOptions {

packages/angular_devkit/build_angular/src/angular-cli-files/models/webpack-configs/common.ts

@@ -12,12 +12,14 @@ import {
 import { tags } from '@angular-devkit/core';
 import * as CopyWebpackPlugin from 'copy-webpack-plugin';
 import * as path from 'path';
+import { RollupOptions } from 'rollup';
 import { ScriptTarget } from 'typescript';
 import {
   Compiler,
   Configuration,
   ContextReplacementPlugin,
   HashedModuleIdsPlugin,
+  Rule,
   compilation,
   debug,
 } from 'webpack';
@@ -29,6 +31,7 @@ import { BundleBudgetPlugin } from '../../plugins/bundle-budget';
 import { CleanCssWebpackPlugin } from '../../plugins/cleancss-webpack-plugin';
 import { NamedLazyChunksPlugin } from '../../plugins/named-chunks-plugin';
 import { ScriptsWebpackPlugin } from '../../plugins/scripts-webpack-plugin';
+import { WebpackRollupLoader } from '../../plugins/webpack';
 import { findAllNodeModules, findUp } from '../../utilities/find-up';
 import { WebpackConfigOptions } from '../build-options';
 import { getEsVersionForFileName, getOutputHashFormat, normalizeExtraEntryPoints } from './utils';
@@ -57,6 +60,7 @@ export function getCommonConfig(wco: WebpackConfigOptions): Configuration {
 
   // tslint:disable-next-line:no-any
   const extraPlugins: any[] = [];
+  const extraRules: Rule[] = [];
   const entryPoints: { [key: string]: string[] } = {};
 
   const targetInFileName = getEsVersionForFileName(
@@ -65,7 +69,51 @@ export function getCommonConfig(wco: WebpackConfigOptions): Configuration {
   );
 
   if (buildOptions.main) {
-    entryPoints['main'] = [path.resolve(root, buildOptions.main)];
+    const mainPath = path.resolve(root, buildOptions.main);
+    entryPoints['main'] = [mainPath];
+
+    if (buildOptions.experimentalRollupConcatenation) {
+      // NOTE: the following are known problems with experimentalRollupConcatenation
+      // - vendorChunk, commonChunk, namedChunks: these won't work, because by the time webpack
+      // sees the chunks, the context of where they came from is lost.
+      // - webWorkerTsConfig: workers must be imported via a root relative path (e.g.
+      // `app/search/search.worker`) instead of a relative path (`/search.worker`) because
+      // of the same reason as above.
+      // - loadChildren string syntax: doesn't work because rollup cannot follow the imports.
+
+      // Rollup options, except entry module, which is automatically inferred.
+      const rollupOptions: RollupOptions = {};
+
+      // Add rollup plugins/rules.
+      extraRules.push({
+        test: mainPath,
+        // Ensure rollup loader executes after other loaders.
+        enforce: 'post',
+        use: [{
+          loader: WebpackRollupLoader,
+          options: rollupOptions,
+        }],
+      });
+
+      // Rollup bundles will include the dynamic System.import that was inside Angular and webpack
+      // will emit warnings because it can't resolve it. We just ignore it.
+      // TODO: maybe use https://webpack.js.org/configuration/stats/#statswarningsfilter instead.
+
+      // Ignore all "Critical dependency: the request of a dependency is an expression" warnings.
+      extraPlugins.push(new ContextReplacementPlugin(/./));
+      // Ignore "System.import() is deprecated" warnings for the main file and js files.
+      // Might still get them if @angular/core gets split into a lazy module.
+      extraRules.push({
+        test: mainPath,
+        enforce: 'post',
+        parser: { system: true },
+      });
+      extraRules.push({
+        test: /\.js$/,
+        enforce: 'post',
+        parser: { system: true },
+      });
+    }
   }
 
   let differentialLoadingNeeded = false;
@@ -476,6 +524,7 @@ export function getCommonConfig(wco: WebpackConfigOptions): Configuration {
           enforce: 'pre',
           ...sourceMapUseRule,
         },
+        ...extraRules,
       ],
     },
     optimization: {

packages/angular_devkit/build_angular/src/angular-cli-files/plugins/webpack-rollup-loader.ts

@@ -0,0 +1,142 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+// Adapted from https://github.com/erikdesjardins/webpack-rollup-loader/blob/master/index.js
+
+import { VirtualFileSystemDecorator } from '@ngtools/webpack/src/virtual_file_system_decorator';
+import { dirname, join } from 'path';
+import { OutputAsset, OutputChunk, rollup } from 'rollup';
+import { RawSourceMap } from 'source-map';
+import webpack = require('webpack');
+
+function splitRequest(request: string) {
+  const inx = request.lastIndexOf('!');
+  if (inx === -1) {
+    return {
+      loaders: '',
+      resource: request,
+    };
+  } else {
+    return {
+      loaders: request.slice(0, inx + 1),
+      resource: request.slice(inx + 1),
+    };
+  }
+}
+
+// Load resolve paths using Webpack.
+function webpackResolutionPlugin(
+  loaderContext: webpack.loader.LoaderContext,
+  entryId: string,
+  entryIdCodeAndMap: { code: string, map: RawSourceMap },
+) {
+  return {
+    name: 'webpack-resolution-plugin',
+    resolveId: (id: string, importerId: string) => {
+      if (id === entryId) {
+        return entryId;
+      } else {
+        return new Promise((resolve, reject) => {
+          // split apart resource paths because Webpack's this.resolve() can't handle `loader!`
+          // prefixes
+          const parts = splitRequest(id);
+          const importerParts = splitRequest(importerId);
+
+          // resolve the full path of the imported file with Webpack's module loader
+          // this will figure out node_modules imports, Webpack aliases, etc.
+          loaderContext.resolve(
+            dirname(importerParts.resource),
+            parts.resource,
+            (err, fullPath) => err ? reject(err) : resolve(parts.loaders + fullPath),
+          );
+        });
+      }
+    },
+    load: (id: string) => {
+      if (id === entryId) {
+        return entryIdCodeAndMap;
+      }
+
+      return new Promise((resolve, reject) => {
+        // load the module with Webpack
+        // this will apply all relevant loaders, etc.
+        loaderContext.loadModule(
+          id,
+          (err, source, map) => err ? reject(err) : resolve({ code: source, map: map }),
+        );
+      });
+    },
+  };
+}
+
+export default function webpackRollupLoader(
+  this: webpack.loader.LoaderContext,
+  source: string,
+  sourceMap: RawSourceMap,
+) {
+  // Note: this loader isn't cacheable because it will add the lazy chunks to the
+  // virtual file system on completion.
+  const callback = this.async();
+  if (!callback) {
+    throw new Error('Async loader support is required.');
+  }
+  const options = this.query || {};
+  const entryId = this.resourcePath;
+  const sourcemap = this.sourceMap;
+
+  // Get the VirtualFileSystemDecorator that AngularCompilerPlugin added so we can write to it.
+  // Since we use webpackRollupLoader as a post loader, this should be there.
+  // TODO: we should be able to do this in a more elegant way by again decorating webpacks
+  // input file system inside a custom WebpackRollupPlugin, modelled after AngularCompilerPlugin.
+  const vfs = this._compiler.inputFileSystem as VirtualFileSystemDecorator;
+  const virtualWrite = (path: string, data: string) =>
+    vfs.getWebpackCompilerHost().writeFile(path, data, false);
+
+  // Bundle with Rollup
+  const rollupOptions = {
+    ...options,
+    input: entryId,
+    plugins: [
+      ...(options.plugins || []),
+      webpackResolutionPlugin(this, entryId, { code: source, map: sourceMap }),
+    ],
+  };
+
+  rollup(rollupOptions)
+    .then(build => build.generate({ format: 'es', sourcemap }))
+    .then(
+      (result) => {
+        const [mainChunk, ...otherChunksOrAssets] = result.output;
+
+        // Write other chunks and assets to the virtual file system so that webpack can load them.
+        const resultDir = dirname(entryId);
+        otherChunksOrAssets.forEach(chunkOrAsset => {
+          const { fileName, type } = chunkOrAsset;
+          if (type == 'chunk') {
+            const { code, map } = chunkOrAsset as OutputChunk;
+            virtualWrite(join(resultDir, fileName), code);
+            if (map) {
+              // Also write the map if there's one.
+              // Probably need scriptsSourceMap set on CLI to load it.
+              virtualWrite(join(resultDir, `${fileName}.map`), map.toString());
+            }
+          } else if (type == 'asset') {
+            const { source } = chunkOrAsset as OutputAsset;
+            // Source might be a Buffer. Just assuming it's a string for now.
+            virtualWrite(join(resultDir, fileName), source as string);
+          }
+        });
+
+        // Always return the main chunk from webpackRollupLoader.
+        // Cast to any here is needed because of a typings incompatibility between source-map versions.
+        // tslint:disable-next-line:no-any
+        callback(null, mainChunk.code, (mainChunk as any).map);
+      },
+      (err) => callback(err),
+    );
+}

packages/angular_devkit/build_angular/src/angular-cli-files/plugins/webpack.ts

@@ -20,3 +20,4 @@ export {
 
 import { join } from 'path';
 export const RawCssLoader = require.resolve(join(__dirname, 'raw-css-loader'));
+export const WebpackRollupLoader = require.resolve(join(__dirname, 'webpack-rollup-loader'));

packages/angular_devkit/build_angular/src/browser/schema.json

@@ -353,6 +353,11 @@
         "anonymous",
         "use-credentials"
       ]
+    },
+    "experimentalRollupConcatenation": {
+      "type": "boolean",
+      "description": "Concatenate modules with Rollup before bundling them with Webpack.",
+      "default": false
     }
   },
   "additionalProperties": false,

packages/angular_devkit/build_angular/src/utils/webpack-browser-config.ts

@@ -48,6 +48,18 @@ export async function generateWebpackConfig(
     throw new Error(`The 'buildOptimizer' option cannot be used without 'aot'.`);
   }
 
+  // Ensure Rollup Concatenation is only used with compatible options.
+  if (options.experimentalRollupConcatenation) {
+    if (!options.aot) {
+      throw new Error(`The 'experimentalRollupConcatenation' option cannot be used without 'aot'.`);
+    }
+
+    if (options.vendorChunk || options.commonChunk || options.namedChunks) {
+      throw new Error(`The 'experimentalRollupConcatenation' option cannot be used with the`
+        + `'vendorChunk', 'commonChunk', 'namedChunks' options set to true.`);
+    }
+  }
+
   const tsConfigPath = path.resolve(workspaceRoot, options.tsConfig);
   const tsConfig = readTsconfig(tsConfigPath);
 

packages/angular_devkit/build_angular/test/browser/rollup_spec_large.ts

@@ -0,0 +1,102 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import { Architect } from '@angular-devkit/architect';
+import {
+  BrowserBuildOutput,
+  browserBuild,
+  createArchitect,
+  host,
+  lazyModuleFiles,
+  lazyModuleFnImport,
+} from '../utils';
+
+
+describe('Browser Builder Rollup Concatenation test', () => {
+  const target = { project: 'app', target: 'build' };
+  const overrides = {
+    experimentalRollupConcatenation: true,
+    // JIT Rollup bundles will include require calls to .css and .html file, that have lost their
+    // path context. AOT code already inlines resources so that's not a problem.
+    aot: true,
+    // Webpack can't separate rolled-up modules into chunks.
+    vendorChunk: false,
+    commonChunk: false,
+    namedChunks: false,
+  };
+  const prodOverrides = {
+    // Usual prod options.
+    fileReplacements: [{
+      replace: 'src/environments/environment.ts',
+      with: 'src/environments/environment.prod.ts',
+    }],
+    optimization: true,
+    sourceMap: false,
+    extractCss: true,
+    namedChunks: false,
+    aot: true,
+    extractLicenses: true,
+    vendorChunk: false,
+    buildOptimizer: true,
+    // Extra prod options we need for experimentalRollupConcatenation.
+    commonChunk: false,
+    // Just for convenience.
+    outputHashing: 'none',
+  };
+  const rollupProdOverrides = {
+    ...prodOverrides,
+    experimentalRollupConcatenation: true,
+  };
+  let architect: Architect;
+
+  const getOutputSize = async (output: BrowserBuildOutput) =>
+    (await Promise.all(
+      Object.keys(output.files)
+        .filter(name => name.endsWith('.js') &&
+          // These aren't concatenated by Rollup so no point comparing.
+          !['runtime.js', 'polyfills.js'].includes(name))
+        .map(name => output.files[name]),
+    ))
+      .map(content => content.length)
+      .reduce((acc, curr) => acc + curr, 0);
+
+  beforeEach(async () => {
+    await host.initialize().toPromise();
+    architect = (await createArchitect(host.root())).architect;
+  });
+
+  afterEach(async () => host.restore().toPromise());
+
+  it('works', async () => {
+    await browserBuild(architect, host, target, overrides);
+  });
+
+  it('works with lazy modules', async () => {
+    host.writeMultipleFiles(lazyModuleFiles);
+    host.writeMultipleFiles(lazyModuleFnImport);
+    await browserBuild(architect, host, target, overrides);
+  });
+
+  it('creates smaller or same size bundles for app without lazy bundles', async () => {
+    const prodOutput = await browserBuild(architect, host, target, prodOverrides);
+    const prodSize = await getOutputSize(prodOutput);
+    const rollupProdOutput = await browserBuild(architect, host, target, rollupProdOverrides);
+    const rollupProd = await getOutputSize(rollupProdOutput);
+    expect(prodSize).toBeGreaterThan(rollupProd);
+  });
+
+  it('creates smaller bundles for apps with lazy bundles', async () => {
+    host.writeMultipleFiles(lazyModuleFiles);
+    host.writeMultipleFiles(lazyModuleFnImport);
+    const prodOutput = await browserBuild(architect, host, target, prodOverrides);
+    const prodSize = await getOutputSize(prodOutput);
+    const rollupProdOutput = await browserBuild(architect, host, target, rollupProdOverrides);
+    const rollupProd = await getOutputSize(rollupProdOutput);
+    expect(prodSize).toBeGreaterThan(rollupProd);
+  });
+});