Skip to content

Commit 67073da

Browse files
kuhekuhe
committed
feat: inline cjs index
1 parent 9972041 commit 67073da

File tree

5 files changed

+568
-0
lines changed

5 files changed

+568
-0
lines changed

‎.changeset/rude-spiders-run.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
---
2+
---

‎package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
"@typescript-eslint/parser": "4.30.0",
4040
"chai": "^4.2.0",
4141
"chai-as-promised": "^7.1.1",
42+
"esbuild": "^0.19.9",
4243
"eslint": "7.32.0",
4344
"eslint-config-prettier": "8.3.0",
4445
"eslint-plugin-prettier": "3.4.1",

‎scripts/compilation/Inliner.js

Lines changed: 283 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,283 @@
1+
const fs = require("fs");
2+
const path = require("path");
3+
const { spawnProcess } = require("./../utils/spawn-process");
4+
const walk = require("./../utils/walk");
5+
const esbuild = require("esbuild");
6+
7+
const root = path.join(__dirname, "..", "..");
8+
9+
/**
10+
*
11+
* Inline a package as one dist file, preserves other files as re-export stubs,
12+
* preserves files with react-native variants as externals.
13+
*
14+
*/
15+
module.exports = class Inliner {
16+
constructor(pkg) {
17+
this.package = pkg;
18+
this.platform = "node";
19+
this.isPackage = fs.existsSync(path.join(root, "packages", pkg));
20+
this.isLib = fs.existsSync(path.join(root, "lib", pkg));
21+
this.isClient = !this.isPackage && !this.isLib;
22+
this.subfolder = this.isPackage ? "packages" : this.isLib ? "lib" : "clients";
23+
24+
this.packageDirectory = path.join(root, this.subfolder, pkg);
25+
26+
this.outfile = path.join(root, this.subfolder, pkg, "dist-cjs", "index.js");
27+
}
28+
29+
/**
30+
* step 0: delete the dist-cjs folder.
31+
*/
32+
async clean() {
33+
await spawnProcess("yarn", ["rimraf", "./dist-cjs", "tsconfig.cjs.tsbuildinfo"], { cwd: this.packageDirectory });
34+
console.log("Deleted ./dist-cjs in " + this.package);
35+
return this;
36+
}
37+
38+
/**
39+
* step 1: build the default tsc dist-cjs output with dispersed files.
40+
* we will need the files to be in place for stubbing.
41+
*/
42+
async tsc() {
43+
await spawnProcess("yarn", ["g:tsc", "-p", "tsconfig.cjs.json"], { cwd: this.packageDirectory });
44+
console.log("Finished recompiling ./dist-cjs in " + this.package);
45+
return this;
46+
}
47+
48+
/**
49+
* step 2: detect all variant files and their transitive local imports.
50+
* these files will not be inlined, in order to preserve the react-native dist-cjs file replacement behavior.
51+
*/
52+
async discoverVariants() {
53+
const pkgJson = require(path.join(root, this.subfolder, this.package, "package.json"));
54+
this.variantEntries = Object.entries(pkgJson["react-native"] ?? {});
55+
56+
for await (const file of walk(path.join(this.packageDirectory, "dist-cjs"))) {
57+
if (file.endsWith(".js") && fs.existsSync(file.replace(/\.js$/, ".native.js"))) {
58+
console.log("detected undeclared auto-variant", file);
59+
const canonical = file.replace(/(.*?)dist-cjs\//, "./dist-cjs/").replace(/\.js$/, "");
60+
const variant = canonical.replace(/(.*?)(\.js)?$/, "$1.native$2");
61+
62+
this.variantEntries.push([canonical, variant]);
63+
}
64+
if (fs.existsSync(file.replace(/\.js$/, ".browser.js"))) {
65+
// not applicable to CJS?
66+
}
67+
}
68+
69+
this.transitiveVariants = [];
70+
71+
for (const [k, v] of this.variantEntries) {
72+
for (const variantFile of [k, v]) {
73+
if (!variantFile.includes("dist-cjs/")) {
74+
continue;
75+
}
76+
const keyFile = path.join(
77+
this.packageDirectory,
78+
"dist-cjs",
79+
variantFile.replace(/(.*?)dist-cjs\//, "") + (variantFile.endsWith(".js") ? "" : ".js")
80+
);
81+
const keyFileContents = fs.readFileSync(keyFile, "utf-8");
82+
const requireStatements = keyFileContents.matchAll(/require\("(.*?)"\)/g);
83+
for (const requireStatement of requireStatements) {
84+
if (requireStatement[1]?.startsWith(".")) {
85+
// is relative import.
86+
const key = path
87+
.normalize(path.join(path.dirname(keyFile), requireStatement[1]))
88+
.replace(/(.*?)dist-cjs\//, "./dist-cjs/");
89+
console.log("Transitive variant file:", key);
90+
this.variantEntries.push([key, key]);
91+
this.transitiveVariants.push(key.replace(/(.*?)dist-cjs\//, "").replace(/(\.js)?$/, ""));
92+
}
93+
}
94+
}
95+
}
96+
97+
this.variantExternals = [];
98+
this.variantMap = {};
99+
100+
for (const [k, v] of this.variantEntries) {
101+
const prefix = "dist-cjs/";
102+
const keyPrefixIndex = k.indexOf(prefix);
103+
if (keyPrefixIndex === -1) {
104+
continue;
105+
}
106+
const valuePrefixIndex = v.indexOf(prefix);
107+
const keyRelativePath = k.slice(keyPrefixIndex + prefix.length);
108+
const valueRelativePath = v.slice(valuePrefixIndex + prefix.length);
109+
this.variantExternals.push(
110+
...[keyRelativePath, valueRelativePath].map((file) => (file.endsWith(".js") ? file : file + ".js"))
111+
);
112+
this.variantMap[keyRelativePath] = valueRelativePath;
113+
}
114+
115+
return this;
116+
}
117+
118+
/**
119+
* step 3: bundle the package index into dist-cjs/index.js except for node_modules
120+
* and also excluding any local files that have variants for react-native.
121+
*/
122+
async bundle() {
123+
this.variantExternalsForEsBuild = this.variantExternals.map(
124+
(variant) => "*/" + path.basename(variant).replace(/.js$/, "")
125+
);
126+
127+
await esbuild.build({
128+
platform: this.platform,
129+
bundle: true,
130+
format: "cjs",
131+
mainFields: ["main"],
132+
allowOverwrite: true,
133+
entryPoints: [path.join(root, this.subfolder, this.package, "src", "index.ts")],
134+
outfile: this.outfile,
135+
external: [
136+
"tslib",
137+
"@aws-crypto/*",
138+
"@smithy/*",
139+
"@aws-sdk/*",
140+
"typescript",
141+
"vscode-oniguruma",
142+
"pnpapi",
143+
"fast-xml-parser",
144+
"node_modules/*",
145+
...this.variantExternalsForEsBuild,
146+
],
147+
});
148+
return this;
149+
}
150+
151+
/**
152+
* step 4: rewrite all existing dist-cjs files except the index.js file.
153+
* These now become re-exports of the index to preserve deep-import behavior.
154+
*/
155+
async rewriteStubs() {
156+
for await (const file of walk(path.join(this.packageDirectory, "dist-cjs"))) {
157+
const relativePath = file.replace(path.join(this.packageDirectory, "dist-cjs"), "").slice(1);
158+
if (relativePath === "index.js") {
159+
console.log("Skipping index.js");
160+
continue;
161+
}
162+
163+
if (this.variantExternals.find((external) => relativePath.endsWith(external))) {
164+
console.log("Not rewriting.", relativePath, "is variant.");
165+
continue;
166+
}
167+
168+
console.log("Rewriting", relativePath, "as index re-export stub.");
169+
170+
const depth = relativePath.split("/").length - 1;
171+
const indexRelativePath =
172+
(depth === 0
173+
? "."
174+
: Array.from({ length: depth })
175+
.map(() => "..")
176+
.join("/")) + "/index.js";
177+
178+
fs.writeFileSync(file, `module.exports = require("${indexRelativePath}");`);
179+
}
180+
181+
return this;
182+
}
183+
184+
/**
185+
* step 5: rewrite variant external imports to correct path.
186+
* these externalized variants use relative imports for transitive variant files
187+
* which need to be rewritten when in the index.js file.
188+
*/
189+
async fixVariantImportPaths() {
190+
this.indexContents = fs.readFileSync(this.outfile, "utf-8");
191+
for (const variant of Object.keys(this.variantMap)) {
192+
const basename = path.basename(variant).replace(/.js$/, "");
193+
const dirname = path.dirname(variant);
194+
195+
const find = new RegExp(`require\\("./(.*?)/${basename}"\\)`);
196+
const replace = `require("./${dirname}/${basename}")`;
197+
198+
this.indexContents = this.indexContents.replace(find, replace);
199+
200+
console.log("replacing", find, "with", replace);
201+
}
202+
203+
fs.writeFileSync(this.outfile, this.indexContents, "utf-8");
204+
return this;
205+
}
206+
207+
/**
208+
* Step 5.5, dedupe imported externals.
209+
*/
210+
async dedupeExternals() {
211+
const redundantRequireStatements = this.indexContents.matchAll(
212+
/var import_([a-z_]+)(\d+) = require\("([@a-z\/-0-9]+)"\);/g
213+
);
214+
for (const requireStatement of redundantRequireStatements) {
215+
const variableSuffix = requireStatement[1];
216+
const packageName = requireStatement[3].replace("/", "\\/");
217+
218+
const original = this.indexContents.match(
219+
new RegExp(`var import_${variableSuffix} = require\\(\"${packageName}\"\\);`)
220+
);
221+
if (original) {
222+
let redundancyIndex = 0;
223+
let misses = 0;
224+
225+
// perform an incremental replacement instead of a global (\d+) replacement
226+
// to be safe.
227+
while (true) {
228+
const redundantRequire = `var import_${variableSuffix}${redundancyIndex} = require\\("${packageName}"\\);`;
229+
const redundantVariable = `import_${variableSuffix}${redundancyIndex}`;
230+
231+
if (this.indexContents.match(new RegExp(redundantRequire))) {
232+
console.log("replacing", redundantVariable);
233+
this.indexContents = this.indexContents
234+
.replace(new RegExp(redundantRequire, "g"), "")
235+
.replace(new RegExp(redundantVariable, "g"), `import_${variableSuffix}`);
236+
} else if (misses++ > 10) {
237+
break;
238+
}
239+
redundancyIndex++;
240+
}
241+
}
242+
}
243+
fs.writeFileSync(this.outfile, this.indexContents, "utf-8");
244+
return this;
245+
}
246+
247+
/**
248+
* step 6: we validate that the index.js file has a require statement
249+
* for any variant files, to ensure they are not in the inlined (bundled) index.
250+
*/
251+
async validate() {
252+
this.indexContents = fs.readFileSync(this.outfile, "utf-8");
253+
254+
const externalsToCheck = new Set(
255+
Object.keys(this.variantMap)
256+
.filter((variant) => !this.transitiveVariants.includes(variant))
257+
.map((variant) => path.basename(variant).replace(/.js$/, ""))
258+
);
259+
260+
for (const line of this.indexContents.split("\n")) {
261+
// we expect to see a line with require() and the variant external in it
262+
if (line.includes("require(")) {
263+
const checkOrder = [...externalsToCheck].sort().reverse();
264+
for (const external of checkOrder) {
265+
if (line.includes(external)) {
266+
console.log("Inline index confirmed require() for variant external:", external);
267+
externalsToCheck.delete(external);
268+
continue;
269+
}
270+
}
271+
}
272+
}
273+
274+
if (externalsToCheck.size) {
275+
throw new Error(
276+
"require() statements for the following variant externals: " +
277+
[...externalsToCheck].join(", ") +
278+
" were not found in the index."
279+
);
280+
}
281+
return this;
282+
}
283+
};

‎scripts/inline.js

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/**
2+
*
3+
* Inline a package as one dist file.
4+
*
5+
*/
6+
7+
const fs = require("fs");
8+
const path = require("path");
9+
const Inliner = require("./compilation/Inliner");
10+
11+
const root = path.join(__dirname, "..");
12+
13+
const package = process.argv[2];
14+
15+
if (!package) {
16+
/**
17+
* If no package is selected, this script sets all build:cjs scripts to
18+
* use this inliner script instead of only tsc.
19+
*/
20+
const packages = fs.readdirSync(path.join(root, "packages"));
21+
for (const pkg of packages) {
22+
const pkgJsonFilePath = path.join(root, "packages", pkg, "package.json");
23+
const pkgJson = require(pkgJsonFilePath);
24+
25+
pkgJson.scripts["build:cjs"] = `node ../../scripts/inline ${pkg}`;
26+
fs.writeFileSync(pkgJsonFilePath, JSON.stringify(pkgJson, null, 2));
27+
}
28+
} else {
29+
(async () => {
30+
const inliner = new Inliner(package);
31+
await inliner.clean();
32+
await inliner.tsc();
33+
await inliner.discoverVariants();
34+
await inliner.bundle();
35+
await inliner.rewriteStubs();
36+
await inliner.fixVariantImportPaths();
37+
await inliner.dedupeExternals();
38+
await inliner.validate();
39+
})();
40+
}

0 commit comments

Comments
 (0)