Skip to content

build: improve postinstall patches with better messaging and prompt #19615

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 12, 2020
Merged

build: improve postinstall patches with better messaging and prompt #19615

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
357 changes: 213 additions & 144 deletions tools/postinstall/apply-patches.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
const shelljs = require('shelljs');
const path = require('path');
const fs = require('fs');
const inquirer = require('inquirer');
const chalk = require('chalk');

/**
* Version of the post install patch. Needs to be incremented when
Expand All @@ -23,154 +25,172 @@ const projectDir = path.join(__dirname, '../..');
*/
const PATCHES_PER_FILE = {};

shelljs.set('-e');
shelljs.cd(projectDir);

// Workaround for https://github.com/angular/angular/issues/18810.
shelljs.exec('ngc -p angular-tsconfig.json');

// Workaround for: https://github.com/angular/angular/issues/32651. We just do not
// generate re-exports for secondary entry-points. Similar to what "ng-packagr" does.
searchAndReplace(
/(?!function\s+)createMetadataReexportFile\([^)]+\);/, '',
'node_modules/@angular/bazel/src/ng_package/packager.js');
searchAndReplace(
/(?!function\s+)createTypingsReexportFile\([^)]+\);/, '',
'node_modules/@angular/bazel/src/ng_package/packager.js');

// Workaround for: https://github.com/angular/angular/pull/32650
searchAndReplace(
'var indexFile;', `
var indexFile = files.find(f => f.endsWith('/public-api.ts'));
`,
'node_modules/@angular/compiler-cli/src/metadata/bundle_index_host.js');
searchAndReplace(
'var resolvedEntryPoint = null;', `
var resolvedEntryPoint = tsFiles.find(f => f.endsWith('/public-api.ts')) || null;
`,
'node_modules/@angular/compiler-cli/src/ngtsc/entry_point/src/logic.js');

// Workaround for: https://hackmd.io/MlqFp-yrSx-0mw4rD7dnQQ?both. We only want to discard
// the metadata of files in the bazel managed node modules. That way we keep the default
// behavior of ngc-wrapped except for dependencies between sources of the library. This makes
// the "generateCodeForLibraries" flag more accurate in the Bazel environment where previous
// compilations should not be treated as external libraries. Read more about this in the document.
searchAndReplace(
/if \((this\.options\.generateCodeForLibraries === false)/, `
const fs = require('fs');
const hasFlatModuleBundle = fs.existsSync(filePath.replace('.d.ts', '.metadata.json'));
if ((filePath.includes('node_modules/') || !hasFlatModuleBundle) && $1`,
'node_modules/@angular/compiler-cli/src/transformers/compiler_host.js');
applyPatch(path.join(__dirname, './flat_module_factory_resolution.patch'));
// The three replacements below ensure that metadata files can be read by NGC and
// that metadata files are collected as Bazel action inputs.
searchAndReplace(
/(const NGC_ASSETS = \/[^(]+\()([^)]*)(\).*\/;)/, '$1$2|metadata.json$3',
'node_modules/@angular/bazel/src/ngc-wrapped/index.js');
searchAndReplace(
/^((\s*)results = depset\(dep.angular.summaries, transitive = \[results]\))$/m,
`$1#\n$2results = depset(dep.angular.metadata, transitive = [results])`,
'node_modules/@angular/bazel/src/ng_module.bzl');
searchAndReplace(
/^((\s*)results = depset\(target.angular\.summaries if _has_target_angular_summaries\(target\) else \[]\))$/m,
`$1#\n$2results = depset(target.angular.metadata if _has_target_angular_summaries(target) else [], transitive = [results])`,
'node_modules/@angular/bazel/src/ng_module.bzl');
// Ensure that "metadata" of transitive dependencies can be collected.
searchAndReplace(
/providers\["angular"]\["metadata"] = outs\.metadata/,
`$& + [m for dep in ctx.attr.deps if (hasattr(dep, "angular") and hasattr(dep.angular, "metadata")) for m in dep.angular.metadata]`,
'node_modules/@angular/bazel/src/ng_module.bzl');

// Workaround for: https://github.com/bazelbuild/rules_nodejs/issues/1208.
applyPatch(path.join(__dirname, './manifest_externs_hermeticity.patch'));

try {
// Temporary patch pre-req for https://github.com/angular/angular/pull/36333.
// Can be removed once @angular/bazel is updated here to include this patch.
// try/catch needed for this the material CI tests to work in angular/repo
applyPatch(path.join(__dirname, './@angular_bazel_ng_module.patch'));
} catch {}

try {
// Temporary patch pre-req for https://github.com/angular/angular/pull/36971.
// Can be removed once @angular/bazel is updated here to include this patch.
// try/catch needed for this as the framework repo has this patch already applied,
// and re-applying again causes an error.
applyPatch(path.join(__dirname, './@angular_bazel_ivy_flat_module.patch'));
} catch {}

// Workaround for https://github.com/angular/angular/issues/33452:
searchAndReplace(/angular_compiler_options = {/, `$&
"strictTemplates": True,`, 'node_modules/@angular/bazel/src/ng_module.bzl');

// More info in https://github.com/angular/angular/pull/33786
shelljs.rm('-rf', [
'node_modules/rxjs/add/',
'node_modules/rxjs/observable/',
'node_modules/rxjs/operator/',
// rxjs/operators is a public entry point that also contains files to support legacy deep import
// paths, so we need to preserve index.* and package.json files that are required for module
// resolution.
'node_modules/rxjs/operators/!(index.*|package.json)',
'node_modules/rxjs/scheduler/',
'node_modules/rxjs/symbol/',
'node_modules/rxjs/util/',
'node_modules/rxjs/internal/Rx.d.ts',
'node_modules/rxjs/AsyncSubject.*',
'node_modules/rxjs/BehaviorSubject.*',
'node_modules/rxjs/InnerSubscriber.*',
'node_modules/rxjs/interfaces.*',
'node_modules/rxjs/Notification.*',
'node_modules/rxjs/Observable.*',
'node_modules/rxjs/Observer.*',
'node_modules/rxjs/Operator.*',
'node_modules/rxjs/OuterSubscriber.*',
'node_modules/rxjs/ReplaySubject.*',
'node_modules/rxjs/Rx.*',
'node_modules/rxjs/Scheduler.*',
'node_modules/rxjs/Subject.*',
'node_modules/rxjs/SubjectSubscription.*',
'node_modules/rxjs/Subscriber.*',
'node_modules/rxjs/Subscription.*',
]);

// Apply all collected patches on a per-file basis. This is necessary because
// multiple edits might apply to the same file, and we only want to mark a given
// file as patched once all edits have been made.
Object.keys(PATCHES_PER_FILE).forEach(filePath => {
if (hasFileBeenPatched(filePath)) {
console.info('File ' + filePath + ' is already patched. Skipping..');
return;
const PATCH_MARKER_FILE_PATH = path.join(projectDir, 'node_modules/_ng-comp-patch-marker.json');

/** Registry of applied patches. */
let registry = null;

main();

async function main() {
shelljs.set('-e');
shelljs.cd(projectDir);

registry = await readAndValidatePatchMarker();

// Apply all patches synchronously.
applyPatches();

// Write the patch marker file so that we don't accidentally re-apply patches
// in subsequent Yarn installations.
fs.writeFileSync(PATCH_MARKER_FILE_PATH, JSON.stringify(registry, null, 2));
}

function applyPatches() {
// Workaround for https://github.com/angular/angular/issues/18810.
shelljs.exec('ngc -p angular-tsconfig.json');

// Workaround for: https://github.com/angular/angular/issues/32651. We just do not
// generate re-exports for secondary entry-points. Similar to what "ng-packagr" does.
searchAndReplace(
/(?!function\s+)createMetadataReexportFile\([^)]+\);/, '',
'node_modules/@angular/bazel/src/ng_package/packager.js');
searchAndReplace(
/(?!function\s+)createTypingsReexportFile\([^)]+\);/, '',
'node_modules/@angular/bazel/src/ng_package/packager.js');

// Workaround for: https://github.com/angular/angular/pull/32650
searchAndReplace(
'var indexFile;', `
var indexFile = files.find(f => f.endsWith('/public-api.ts'));
`,
'node_modules/@angular/compiler-cli/src/metadata/bundle_index_host.js');
searchAndReplace(
'var resolvedEntryPoint = null;', `
var resolvedEntryPoint = tsFiles.find(f => f.endsWith('/public-api.ts')) || null;
`,
'node_modules/@angular/compiler-cli/src/ngtsc/entry_point/src/logic.js');

// Workaround for: https://hackmd.io/MlqFp-yrSx-0mw4rD7dnQQ?both. We only want to discard
// the metadata of files in the bazel managed node modules. That way we keep the default
// behavior of ngc-wrapped except for dependencies between sources of the library. This makes
// the "generateCodeForLibraries" flag more accurate in the Bazel environment where previous
// compilations should not be treated as external libraries. Read more about this in the document.
searchAndReplace(
/if \((this\.options\.generateCodeForLibraries === false)/, `
const fs = require('fs');
const hasFlatModuleBundle = fs.existsSync(filePath.replace('.d.ts', '.metadata.json'));
if ((filePath.includes('node_modules/') || !hasFlatModuleBundle) && $1`,
'node_modules/@angular/compiler-cli/src/transformers/compiler_host.js');
applyPatch(path.join(__dirname, './flat_module_factory_resolution.patch'));
// The three replacements below ensure that metadata files can be read by NGC and
// that metadata files are collected as Bazel action inputs.
searchAndReplace(
/(const NGC_ASSETS = \/[^(]+\()([^)]*)(\).*\/;)/, '$1$2|metadata.json$3',
'node_modules/@angular/bazel/src/ngc-wrapped/index.js');
searchAndReplace(
/^((\s*)results = depset\(dep.angular.summaries, transitive = \[results]\))$/m,
`$1#\n$2results = depset(dep.angular.metadata, transitive = [results])`,
'node_modules/@angular/bazel/src/ng_module.bzl');
searchAndReplace(
/^((\s*)results = depset\(target.angular\.summaries if _has_target_angular_summaries\(target\) else \[]\))$/m,
`$1#\n$2results = depset(target.angular.metadata if _has_target_angular_summaries(target) else [], transitive = [results])`,
'node_modules/@angular/bazel/src/ng_module.bzl');
// Ensure that "metadata" of transitive dependencies can be collected.
searchAndReplace(
/providers\["angular"]\["metadata"] = outs\.metadata/,
`$& + [m for dep in ctx.attr.deps if (hasattr(dep, "angular") and hasattr(dep.angular, "metadata")) for m in dep.angular.metadata]`,
'node_modules/@angular/bazel/src/ng_module.bzl');

// Workaround for: https://github.com/bazelbuild/rules_nodejs/issues/1208.
applyPatch(path.join(__dirname, './manifest_externs_hermeticity.patch'));

try {
// Temporary patch pre-req for https://github.com/angular/angular/pull/36333.
// Can be removed once @angular/bazel is updated here to include this patch.
// try/catch needed for this the material CI tests to work in angular/repo
applyPatch(path.join(__dirname, './@angular_bazel_ng_module.patch'));
} catch {
}

let content = fs.readFileSync(filePath, 'utf8');
const patchFunctions = PATCHES_PER_FILE[filePath];
try {
// Temporary patch pre-req for https://github.com/angular/angular/pull/36971.
// Can be removed once @angular/bazel is updated here to include this patch.
// try/catch needed for this as the framework repo has this patch already applied,
// and re-applying again causes an error.
applyPatch(path.join(__dirname, './@angular_bazel_ivy_flat_module.patch'));
} catch {
}

// Workaround for https://github.com/angular/angular/issues/33452:
searchAndReplace(
/angular_compiler_options = {/, `$&
"strictTemplates": True,`,
'node_modules/@angular/bazel/src/ng_module.bzl');

console.info(`Patching file ${filePath} with ${patchFunctions.length} edits..`);
patchFunctions.forEach(patchFn => content = patchFn(content));
// More info in https://github.com/angular/angular/pull/33786
shelljs.rm('-rf', [
'node_modules/rxjs/add/',
'node_modules/rxjs/observable/',
'node_modules/rxjs/operator/',
// rxjs/operators is a public entry point that also contains files to support legacy deep import
// paths, so we need to preserve index.* and package.json files that are required for module
// resolution.
'node_modules/rxjs/operators/!(index.*|package.json)',
'node_modules/rxjs/scheduler/',
'node_modules/rxjs/symbol/',
'node_modules/rxjs/util/',
'node_modules/rxjs/internal/Rx.d.ts',
'node_modules/rxjs/AsyncSubject.*',
'node_modules/rxjs/BehaviorSubject.*',
'node_modules/rxjs/InnerSubscriber.*',
'node_modules/rxjs/interfaces.*',
'node_modules/rxjs/Notification.*',
'node_modules/rxjs/Observable.*',
'node_modules/rxjs/Observer.*',
'node_modules/rxjs/Operator.*',
'node_modules/rxjs/OuterSubscriber.*',
'node_modules/rxjs/ReplaySubject.*',
'node_modules/rxjs/Rx.*',
'node_modules/rxjs/Scheduler.*',
'node_modules/rxjs/Subject.*',
'node_modules/rxjs/SubjectSubscription.*',
'node_modules/rxjs/Subscriber.*',
'node_modules/rxjs/Subscription.*',
]);

// Apply all collected patches on a per-file basis. This is necessary because
// multiple edits might apply to the same file, and we only want to mark a given
// file as patched once all edits have been made.
Object.keys(PATCHES_PER_FILE).forEach(filePath => {
if (isFilePatched(filePath)) {
console.info('File ' + filePath + ' is already patched. Skipping..');
return;
}

fs.writeFileSync(filePath, content, 'utf8');
writePatchMarker(filePath);
});
let content = fs.readFileSync(filePath, 'utf8');
const patchFunctions = PATCHES_PER_FILE[filePath];

console.info(`Patching file ${filePath} with ${patchFunctions.length} edits..`);
patchFunctions.forEach(patchFn => content = patchFn(content));

fs.writeFileSync(filePath, content, 'utf8');
captureFileAsPatched(filePath);
});
}

/**
* Applies the given patch if not done already. Throws if the patch does
* not apply cleanly.
* Applies the given patch if not done already. Throws if the patch
* does not apply cleanly.
*/
function applyPatch(patchFile) {
// Note: We replace non-word characters from the patch marker file name.
// This is necessary because Yarn throws if cached node modules are restored
// which contain files with special characters. Below is an example error:
// ENOTDIR: not a directory, scandir '/<...>/node_modules/@angular_bazel_ng_module.<..>'".
const patchMarkerBasename = `${path.basename(patchFile).replace(/[^\w]/, '_')}`;
const patchMarkerPath = path.join(projectDir, 'node_modules/', patchMarkerBasename);

if (hasFileBeenPatched(patchMarkerPath)) {
if (isFilePatched(patchFile)) {
console.info('Patch: ' + patchFile + ' has been applied already. Skipping..');
return;
}

shelljs.cat(patchFile).exec('patch -p0');
writePatchMarker(patchMarkerPath);
captureFileAsPatched(patchFile);
}

/**
Expand All @@ -185,21 +205,70 @@ function searchAndReplace(search, replacement, relativeFilePath) {
fileEdits.push(originalContent => {
const newFileContent = originalContent.replace(search, replacement);
if (originalContent === newFileContent) {
throw Error(`Could not perform replacement in: ${filePath}.\n` +
throw Error(
`Could not perform replacement in: ${filePath}.\n` +
`Searched for pattern: ${search}`);
}
return newFileContent;
});
}

/** Gets a project unique id for a given file path. */
function getIdForFile(filePath) {
return path.relative(projectDir, filePath).replace(/\\/g, '/');
}

/** Marks the specified file as patched. */
function writePatchMarker(filePath) {
new shelljs.ShellString(PATCH_VERSION).to(`${filePath}.patch_marker`);
function captureFileAsPatched(filePath) {
registry.patched[getIdForFile(filePath)] = true;
}

/** Checks whether the given file is patched. */
function isFilePatched(filePath) {
return registry.patched[getIdForFile(filePath)] === true;
}

/** Checks if the given file has been patched. */
function hasFileBeenPatched(filePath) {
const markerFilePath = `${filePath}.patch_marker`;
return shelljs.test('-e', markerFilePath) &&
shelljs.cat(markerFilePath).toString().trim() === `${PATCH_VERSION}`;
/**
* Reads the patch marker from the node modules if present. Validates that applied
* patches are up-to-date. If not, an error will be reported with a prompt that
* allows convenient clean up of node modules in case those need to be cleaned up.
*/
async function readAndValidatePatchMarker() {
if (!shelljs.test('-e', PATCH_MARKER_FILE_PATH)) {
return {version: PATCH_VERSION, patched: {}};
}
const registry = JSON.parse(shelljs.cat(PATCH_MARKER_FILE_PATH));
// If the node modules are up-to-date, return the parsed patch registry.
if (registry.version === PATCH_VERSION) {
return registry;
}
// Print errors that explain the current situation where patches from another
// postinstall patch revision are applied in the current node modules.
if (registry.version < PATCH_VERSION) {
console.error(chalk.red('Your node modules have been patched by a previous Yarn install.'));
console.error(chalk.red('The postinstall patches have changed since then, and in order to'));
console.error(chalk.red('apply the most recent patches, your node modules need to be cleaned'));
console.error(chalk.red('up from past changes.'));
} else {
console.error(chalk.red('Your node modules already have patches applied from a more recent.'));
console.error(chalk.red('revision of the components repository. In order to be able to apply'));
console.error(chalk.red('patches for the current revision, your node modules need to be'));
console.error(chalk.red('cleaned up.'));
}

const {cleanupModules} = await inquirer.prompt({
name: 'cleanupModules',
type: 'confirm',
message: 'Clean up node modules automatically?',
default: false
});

if (cleanupModules) {
// This re-runs Yarn with `--check-files` mode. The postinstall will rerun afterwards,
// so we can exit with a zero exit-code here.
shelljs.exec('yarn --check-files --frozen-lockfile', {cwd: projectDir});
process.exit(0);
} else {
process.exit(1);
}
}