fix(@angular-devkit/build-angular): dedupe duplicate modules

Webpack relies on package managers to do module hoisting and doesn't have any deduping logic since version 4.

However relaying on package manager has a number of short comings, such as when having the same library with the same version laid out in different parts of the node_modules tree.

Example:
```
/node_modules/[email protected]
/node_modules/library-1/node_modules/[email protected]
/node_modules/library-2/node_modules/[email protected]
```

In the above case, in the final bundle we'll end up with 3 versions of tslib instead of 2, even though 2 of the modules are identical.

Webpack has an open issue for this webpack/webpack#5593 (Duplicate modules - NOT solvable by `npm dedupe`)

With this change we add a custom resolve plugin that dedupes modules with the same name and versions that are laid out in different parts of the node_modules tree.

Author: alan-agius4

packages/angular_devkit/build_angular/src/angular-cli-files/models/webpack-configs/common.ts

@@ -24,7 +24,6 @@ import {
   Plugin,
   Rule,
   RuleSetLoader,
-  compilation,
   debug,
 } from 'webpack';
 import { RawSource } from 'webpack-sources';
@@ -40,6 +39,7 @@ import {
 } from '../../../utils/environment-options';
 import {
   BundleBudgetPlugin,
+  DedupeModuleResolvePlugin,
   NamedLazyChunksPlugin,
   OptimizeCssWebpackPlugin,
   ScriptsWebpackPlugin,
@@ -475,7 +475,10 @@ export function getCommonConfig(wco: WebpackConfigOptions): Configuration {
       extensions: ['.ts', '.tsx', '.mjs', '.js'],
       symlinks: !buildOptions.preserveSymlinks,
       modules: [wco.tsConfig.options.baseUrl || projectRoot, 'node_modules'],
-      plugins: [PnpWebpackPlugin],
+      plugins: [
+        PnpWebpackPlugin,
+        new DedupeModuleResolvePlugin({ verbose: buildOptions.verbose }),
+      ],
     },
     resolveLoader: {
       symlinks: !buildOptions.preserveSymlinks,

packages/angular_devkit/build_angular/src/angular-cli-files/plugins/dedupe-module-resolve-plugin.ts

@@ -0,0 +1,81 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+interface NormalModuleFactoryRequest {
+  request: string;
+  context: {
+    issuer: string;
+  };
+  relativePath: string;
+  path: string;
+  descriptionFileData: {
+    name: string;
+    version: string;
+  };
+  descriptionFileRoot: string;
+  descriptionFilePath: string;
+}
+
+export interface DedupeModuleResolvePluginOptions {
+  verbose?: boolean;
+}
+
+/**
+ * DedupeModuleResolvePlugin is a webpack resolver plugin which dedupes modules with the same name and versions
+ * that are laid out in different parts of the node_modules tree.
+ *
+ * This is needed because Webpack relies on package managers to hoist modules and doesn't have any deduping logic.
+ */
+export class DedupeModuleResolvePlugin {
+  modules = new Map<string, NormalModuleFactoryRequest>();
+
+  constructor(private options?: DedupeModuleResolvePluginOptions) { }
+
+  // tslint:disable-next-line: no-any
+  apply(resolver: any) {
+    resolver
+      .getHook('before-described-relative')
+      .tapPromise('DedupeModuleResolvePlugin', async (request: NormalModuleFactoryRequest) => {
+        if (request.relativePath !== '.') {
+          return;
+        }
+
+        const moduleId = request.descriptionFileData.name + '@' + request.descriptionFileData.version;
+        const prevResolvedModule = this.modules.get(moduleId);
+
+        if (!prevResolvedModule) {
+          // This is the first time we visit this module.
+          this.modules.set(moduleId, request);
+
+          return;
+        }
+
+        const {
+          path,
+          descriptionFilePath,
+          descriptionFileRoot,
+        } = prevResolvedModule;
+
+        if (request.path === path) {
+          // No deduping needed.
+          // Current path and previously resolved path are the same.
+          return;
+        }
+
+        if (this.options?.verbose) {
+          // tslint:disable-next-line: no-console
+          console.warn(`[DedupeModuleResolvePlugin]: ${request.path} -> ${path}`);
+        }
+
+        // Alter current request with previously resolved module.
+        request.path = path;
+        request.descriptionFileRoot = descriptionFileRoot;
+        request.descriptionFilePath = descriptionFilePath;
+      });
+  }
+}

packages/angular_devkit/build_angular/src/angular-cli-files/plugins/webpack.ts

@@ -14,6 +14,7 @@ export { ScriptsWebpackPlugin, ScriptsWebpackPluginOptions } from './scripts-web
 export { SuppressExtractedTextChunksWebpackPlugin } from './suppress-entry-chunks-webpack-plugin';
 export { RemoveHashPlugin, RemoveHashPluginOptions } from './remove-hash-plugin';
 export { NamedLazyChunksPlugin } from './named-chunks-plugin';
+export { DedupeModuleResolvePlugin } from './dedupe-module-resolve-plugin';
 export { CommonJsUsageWarnPlugin } from './common-js-usage-warn-plugin';
 export {
   default as PostcssCliResources,

tests/legacy-cli/e2e/tests/misc/debupe-duplicate-modules.ts

@@ -0,0 +1,41 @@
+import { expectFileToMatch, writeFile } from '../../utils/fs';
+import { ng, silentNpm } from '../../utils/process';
+import { updateJsonFile } from '../../utils/project';
+import { expectToFail } from '../../utils/utils';
+
+export default async function () {
+    // Force duplicate modules
+    await updateJsonFile('package.json', json => {
+        json.dependencies = {
+            ...json.dependencies,
+            'tslib': '2.0.0',
+            'tslib-1': 'npm:[email protected]',
+            'tslib-1-copy': 'npm:[email protected]',
+        };
+    });
+
+    await silentNpm('install');
+
+    await writeFile('./src/main.ts',
+        `
+        import { __assign as __assign_0 } from 'tslib';
+        import { __assign as __assign_1 } from 'tslib-1';
+        import { __assign as __assign_2 } from 'tslib-1-copy';
+
+        console.log({
+            __assign_0,
+            __assign_1,
+            __assign_2,
+        })
+    `);
+
+    const { stderr } = await ng('build', '--verbose', '--no-vendor-chunk');
+    if (!/\[DedupeModuleResolvePlugin\]:.+\/node_modules\/tslib-1-copy -> .+\/node_modules\/tslib-1/.test(stderr)) {
+        throw new Error('Expected stderr to contain [DedupeModuleResolvePlugin] log for tslib.');
+    }
+
+    const outFile = 'dist/test-project/main.js';
+    await expectFileToMatch(outFile, './node_modules/tslib/tslib.es6.js');
+    await expectFileToMatch(outFile, './node_modules/tslib-1/tslib.es6.js');
+    await expectToFail(() => expectFileToMatch(outFile, './node_modules/tslib-1-copy/tslib.es6.js'));
+}