Skip to content

Commit 8efeeb2

Browse files
clydinmgechev
clydin
authored and
mgechev
committed
Backport additional differential loading improvements (#15777)
* build: add @babel/core typings * refactor(@angular-devkit/build-angular): process bundle code quality improvements * refactor(@angular-devkit/build-angular): reorganize bundle processing for browser builder * fix(@angular-devkit/build-angular): workaround high memory usage for differential loading Large files (10MB+) currently cause an excessive amount of memory usage during AST processing. This is currently being remedied upstream. However for the current time period, this change allows for successful builds without increasing the Node.js memory limit. * build: update terser to version 4.3.8 (#15696)
1 parent 19ad809 commit 8efeeb2

File tree

7 files changed

+642
-431
lines changed

7 files changed

+642
-431
lines changed

‎package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@
8787
"@bazel/jasmine": "0.35.0",
8888
"@bazel/karma": "0.35.0",
8989
"@bazel/typescript": "0.35.0",
90+
"@types/babel__core": "7.1.3",
9091
"@types/browserslist": "^4.4.0",
9192
"@types/caniuse-lite": "^1.0.0",
9293
"@types/clean-css": "^4.2.1",

‎packages/angular_devkit/build_angular/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@
5454
"stylus": "0.54.5",
5555
"stylus-loader": "3.0.2",
5656
"tree-kill": "1.2.1",
57-
"terser": "4.1.4",
57+
"terser": "4.3.8",
5858
"terser-webpack-plugin": "1.4.1",
5959
"webpack": "4.39.2",
6060
"webpack-dev-middleware": "3.7.0",

‎packages/angular_devkit/build_angular/src/browser/action-cache.ts

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
/**
2+
* @license
3+
* Copyright Google Inc. All Rights Reserved.
4+
*
5+
* Use of this source code is governed by an MIT-style license that can be
6+
* found in the LICENSE file at https://angular.io/license
7+
*/
8+
import { createHash } from 'crypto';
9+
import * as findCacheDirectory from 'find-cache-dir';
10+
import * as fs from 'fs';
11+
import { manglingDisabled } from '../utils/mangle-options';
12+
import { CacheKey, ProcessBundleOptions, ProcessBundleResult } from '../utils/process-bundle';
13+
14+
const cacache = require('cacache');
15+
const cacheDownlevelPath = findCacheDirectory({ name: 'angular-build-dl' });
16+
const packageVersion = require('../../package.json').version;
17+
18+
// Workaround Node.js issue prior to 10.16 with copyFile on macOS
19+
// https://github.com/angular/angular-cli/issues/15544 & https://github.com/nodejs/node/pull/27241
20+
let copyFileWorkaround = false;
21+
if (process.platform === 'darwin') {
22+
const version = process.versions.node.split('.').map(part => Number(part));
23+
if (version[0] < 10 || version[0] === 11 || (version[0] === 10 && version[1] < 16)) {
24+
copyFileWorkaround = true;
25+
}
26+
}
27+
28+
export interface CacheEntry {
29+
path: string;
30+
size: number;
31+
integrity?: string;
32+
}
33+
34+
export class BundleActionCache {
35+
constructor(private readonly integrityAlgorithm?: string) {}
36+
37+
static copyEntryContent(entry: CacheEntry | string, dest: fs.PathLike): void {
38+
if (copyFileWorkaround) {
39+
try {
40+
fs.unlinkSync(dest);
41+
} catch {}
42+
}
43+
44+
fs.copyFileSync(
45+
typeof entry === 'string' ? entry : entry.path,
46+
dest,
47+
fs.constants.COPYFILE_FICLONE,
48+
);
49+
if (process.platform !== 'win32') {
50+
// The cache writes entries as readonly and when using copyFile the permissions will also be copied.
51+
// See: https://github.com/npm/cacache/blob/073fbe1a9f789ba42d9a41de7b8429c93cf61579/lib/util/move-file.js#L36
52+
fs.chmodSync(dest, 0o644);
53+
}
54+
}
55+
56+
generateBaseCacheKey(content: string): string {
57+
// Create base cache key with elements:
58+
// * package version - different build-angular versions cause different final outputs
59+
// * code length/hash - ensure cached version matches the same input code
60+
const algorithm = this.integrityAlgorithm || 'sha1';
61+
const codeHash = createHash(algorithm)
62+
.update(content)
63+
.digest('base64');
64+
let baseCacheKey = `${packageVersion}|${content.length}|${algorithm}-${codeHash}`;
65+
if (manglingDisabled) {
66+
baseCacheKey += '|MD';
67+
}
68+
69+
return baseCacheKey;
70+
}
71+
72+
generateCacheKeys(action: ProcessBundleOptions): string[] {
73+
const baseCacheKey = this.generateBaseCacheKey(action.code);
74+
75+
// Postfix added to sourcemap cache keys when vendor sourcemaps are present
76+
// Allows non-destructive caching of both variants
77+
const SourceMapVendorPostfix = !!action.sourceMaps && action.vendorSourceMaps ? '|vendor' : '';
78+
79+
// Determine cache entries required based on build settings
80+
const cacheKeys = [];
81+
82+
// If optimizing and the original is not ignored, add original as required
83+
if ((action.optimize || action.optimizeOnly) && !action.ignoreOriginal) {
84+
cacheKeys[CacheKey.OriginalCode] = baseCacheKey + '|orig';
85+
86+
// If sourcemaps are enabled, add original sourcemap as required
87+
if (action.sourceMaps) {
88+
cacheKeys[CacheKey.OriginalMap] = baseCacheKey + SourceMapVendorPostfix + '|orig-map';
89+
}
90+
}
91+
// If not only optimizing, add downlevel as required
92+
if (!action.optimizeOnly) {
93+
cacheKeys[CacheKey.DownlevelCode] = baseCacheKey + '|dl';
94+
95+
// If sourcemaps are enabled, add downlevel sourcemap as required
96+
if (action.sourceMaps) {
97+
cacheKeys[CacheKey.DownlevelMap] = baseCacheKey + SourceMapVendorPostfix + '|dl-map';
98+
}
99+
}
100+
101+
return cacheKeys;
102+
}
103+
104+
async getCacheEntries(cacheKeys: (string | null)[]): Promise<(CacheEntry | null)[] | false> {
105+
// Attempt to get required cache entries
106+
const cacheEntries = [];
107+
for (const key of cacheKeys) {
108+
if (key) {
109+
const entry = await cacache.get.info(cacheDownlevelPath, key);
110+
if (!entry) {
111+
return false;
112+
}
113+
cacheEntries.push({
114+
path: entry.path,
115+
size: entry.size,
116+
integrity: entry.metadata && entry.metadata.integrity,
117+
});
118+
} else {
119+
cacheEntries.push(null);
120+
}
121+
}
122+
123+
return cacheEntries;
124+
}
125+
126+
async getCachedBundleResult(action: ProcessBundleOptions): Promise<ProcessBundleResult | null> {
127+
const entries = action.cacheKeys && await this.getCacheEntries(action.cacheKeys);
128+
if (!entries) {
129+
return null;
130+
}
131+
132+
const result: ProcessBundleResult = { name: action.name };
133+
134+
let cacheEntry = entries[CacheKey.OriginalCode];
135+
if (cacheEntry) {
136+
result.original = {
137+
filename: action.filename,
138+
size: cacheEntry.size,
139+
integrity: cacheEntry.integrity,
140+
};
141+
142+
BundleActionCache.copyEntryContent(cacheEntry, result.original.filename);
143+
144+
cacheEntry = entries[CacheKey.OriginalMap];
145+
if (cacheEntry) {
146+
result.original.map = {
147+
filename: action.filename + '.map',
148+
size: cacheEntry.size,
149+
};
150+
151+
BundleActionCache.copyEntryContent(cacheEntry, result.original.filename + '.map');
152+
}
153+
} else if (!action.ignoreOriginal) {
154+
// If the original wasn't processed (and therefore not cached), add info
155+
result.original = {
156+
filename: action.filename,
157+
size: Buffer.byteLength(action.code, 'utf8'),
158+
map:
159+
action.map === undefined
160+
? undefined
161+
: {
162+
filename: action.filename + '.map',
163+
size: Buffer.byteLength(action.map, 'utf8'),
164+
},
165+
};
166+
}
167+
168+
cacheEntry = entries[CacheKey.DownlevelCode];
169+
if (cacheEntry) {
170+
result.downlevel = {
171+
filename: action.filename.replace('es2015', 'es5'),
172+
size: cacheEntry.size,
173+
integrity: cacheEntry.integrity,
174+
};
175+
176+
BundleActionCache.copyEntryContent(cacheEntry, result.downlevel.filename);
177+
178+
cacheEntry = entries[CacheKey.DownlevelMap];
179+
if (cacheEntry) {
180+
result.downlevel.map = {
181+
filename: action.filename.replace('es2015', 'es5') + '.map',
182+
size: cacheEntry.size,
183+
};
184+
185+
BundleActionCache.copyEntryContent(cacheEntry, result.downlevel.filename + '.map');
186+
}
187+
}
188+
189+
return result;
190+
}
191+
}

‎packages/angular_devkit/build_angular/src/browser/action-executor.ts

Lines changed: 86 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -7,47 +7,110 @@
77
*/
88
import JestWorker from 'jest-worker';
99
import * as os from 'os';
10+
import * as path from 'path';
11+
import { ProcessBundleOptions, ProcessBundleResult } from '../utils/process-bundle';
12+
import { BundleActionCache } from './action-cache';
1013

11-
export class ActionExecutor<Input extends { size: number }, Output> {
12-
private largeWorker: JestWorker;
13-
private smallWorker: JestWorker;
14+
let workerFile = require.resolve('../utils/process-bundle');
15+
workerFile =
16+
path.extname(workerFile) === '.ts'
17+
? require.resolve('../utils/process-bundle-bootstrap')
18+
: workerFile;
1419

15-
private smallThreshold = 32 * 1024;
20+
export class BundleActionExecutor {
21+
private largeWorker?: JestWorker;
22+
private smallWorker?: JestWorker;
23+
private cache: BundleActionCache;
24+
25+
constructor(
26+
private workerOptions: unknown,
27+
integrityAlgorithm?: string,
28+
private readonly sizeThreshold = 32 * 1024,
29+
) {
30+
this.cache = new BundleActionCache(integrityAlgorithm);
31+
}
32+
33+
private static executeMethod<O>(worker: JestWorker, method: string, input: unknown): Promise<O> {
34+
return ((worker as unknown) as Record<string, (i: unknown) => Promise<O>>)[method](input);
35+
}
36+
37+
private ensureLarge(): JestWorker {
38+
if (this.largeWorker) {
39+
return this.largeWorker;
40+
}
1641

17-
constructor(actionFile: string, private readonly actionName: string) {
1842
// larger files are processed in a separate process to limit memory usage in the main process
19-
this.largeWorker = new JestWorker(actionFile, {
20-
exposedMethods: [actionName],
21-
});
43+
return (this.largeWorker = new JestWorker(workerFile, {
44+
exposedMethods: ['process'],
45+
setupArgs: [this.workerOptions],
46+
}));
47+
}
48+
49+
private ensureSmall(): JestWorker {
50+
if (this.smallWorker) {
51+
return this.smallWorker;
52+
}
2253

2354
// small files are processed in a limited number of threads to improve speed
2455
// The limited number also prevents a large increase in memory usage for an otherwise short operation
25-
this.smallWorker = new JestWorker(actionFile, {
26-
exposedMethods: [actionName],
56+
return (this.smallWorker = new JestWorker(workerFile, {
57+
exposedMethods: ['process'],
58+
setupArgs: [this.workerOptions],
2759
numWorkers: os.cpus().length < 2 ? 1 : 2,
2860
// Will automatically fallback to processes if not supported
2961
enableWorkerThreads: true,
30-
});
62+
}));
3163
}
3264

33-
execute(options: Input): Promise<Output> {
34-
if (options.size > this.smallThreshold) {
35-
return ((this.largeWorker as unknown) as Record<string, (options: Input) => Promise<Output>>)[
36-
this.actionName
37-
](options);
65+
private executeAction<O>(method: string, action: { code: string }): Promise<O> {
66+
// code.length is not an exact byte count but close enough for this
67+
if (action.code.length > this.sizeThreshold) {
68+
return BundleActionExecutor.executeMethod<O>(this.ensureLarge(), method, action);
3869
} else {
39-
return ((this.smallWorker as unknown) as Record<string, (options: Input) => Promise<Output>>)[
40-
this.actionName
41-
](options);
70+
return BundleActionExecutor.executeMethod<O>(this.ensureSmall(), method, action);
4271
}
4372
}
4473

45-
executeAll(options: Input[]): Promise<Output[]> {
46-
return Promise.all(options.map(o => this.execute(o)));
74+
async process(action: ProcessBundleOptions) {
75+
const cacheKeys = this.cache.generateCacheKeys(action);
76+
action.cacheKeys = cacheKeys;
77+
78+
// Try to get cached data, if it fails fallback to processing
79+
try {
80+
const cachedResult = await this.cache.getCachedBundleResult(action);
81+
if (cachedResult) {
82+
return cachedResult;
83+
}
84+
} catch {}
85+
86+
return this.executeAction<ProcessBundleResult>('process', action);
87+
}
88+
89+
async *processAll(actions: Iterable<ProcessBundleOptions>) {
90+
const executions = new Map<Promise<ProcessBundleResult>, Promise<ProcessBundleResult>>();
91+
for (const action of actions) {
92+
const execution = this.process(action);
93+
executions.set(
94+
execution,
95+
execution.then(result => {
96+
executions.delete(execution);
97+
98+
return result;
99+
}),
100+
);
101+
}
102+
103+
while (executions.size > 0) {
104+
yield Promise.race(executions.values());
105+
}
47106
}
48107

49108
stop() {
50-
this.largeWorker.end();
51-
this.smallWorker.end();
109+
if (this.largeWorker) {
110+
this.largeWorker.end();
111+
}
112+
if (this.smallWorker) {
113+
this.smallWorker.end();
114+
}
52115
}
53116
}

0 commit comments

Comments
 (0)