Skip to content

Commit ae000e6

Browse files
wbinnssmithForsakenHarmony
wbinnssmith
authored and
ForsakenHarmony
committed
Add pack/unpack scripts from nextpack (#68471)
This brings over the utility scripts `pnpm pack-next` and `pnpm unpack-next path/to/app` from Nextpack, along with the documentation, which has been added to `contributing/core/developing.md`.
1 parent 01b051b commit ae000e6

File tree

6 files changed

+347
-0
lines changed

6 files changed

+347
-0
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ dist
44
.next
55
target
66
packages/next/wasm/@next
7+
tarballs/
78

89
# dependencies
910
node_modules

‎contributing/core/developing.md

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,38 @@ To develop locally:
4343

4444
For instructions on how to build a project with your local version of the CLI,
4545
see **[Developing Using Your Local Version of Next.js](./developing-using-local-app.md)** as linking the package is not sufficient to develop locally.
46+
47+
## Testing a local Next.js version on an application
48+
49+
Since Turbopack doesn't support symlinks when pointing outside of the workspace directory, it can be difficult to develop against a local Next.js version. Neither `pnpm link` nor `file:` imports quite cut it. An alternative is to pack the Next.js version you want to test into a tarball and add it to the pnpm overrides of your test application. The following script will do it for you:
50+
51+
```bash
52+
pnpm pack-next --release && pnpm unpack-next path/to/project
53+
```
54+
55+
Or without running the build:
56+
57+
```bash
58+
pnpm pack-next --no-build --release && pnpm unpack-next path/to/project
59+
```
60+
61+
### Explanation of the scripts
62+
63+
```bash
64+
# Generate a tarball of the Next.js version you want to test
65+
$ pnpm pack-next
66+
67+
# If you need to build in release mode:
68+
$ pnpm pack-next --release
69+
# You can also pass any cargo argument to the script
70+
71+
# To skip the `pnpm i` and `pnpm build` steps in next.js (e. g. if you are running `pnpm dev`)
72+
$ pnpm pack-next --no-build
73+
```
74+
75+
Afterwards, you'll need to unpack the tarball into your test project. You can either manually edit the `package.json` to point to the new tarballs (see the stdout from `pack-next` script), or you can automatically unpack it with:
76+
77+
```bash
78+
# Unpack the tarballs generated with pack-next into project's node_modules
79+
$ pnpm unpack-next path/to/project
80+
```

‎package.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
"build": "turbo run build --remote-cache-timeout 60 --summarize true",
1313
"lerna": "lerna",
1414
"dev": "turbo run dev --parallel",
15+
"pack-next": "node scripts/pack-next.cjs",
1516
"test-types": "tsc",
1617
"test-unit": "jest test/unit/ packages/next/ packages/font",
1718
"test-dev": "cross-env NEXT_TEST_MODE=dev pnpm testheadless",
@@ -58,6 +59,7 @@
5859
"prepare": "husky",
5960
"sync-react": "node ./scripts/sync-react.js",
6061
"update-google-fonts": "node ./scripts/update-google-fonts.js",
62+
"unpack-next": "node scripts/unpack-next.cjs",
6163
"swc-build-native": "pnpm --filter=@next/swc build-native"
6264
},
6365
"devDependencies": {

‎scripts/pack-next.cjs

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
#!/usr/bin/env node
2+
3+
const {
4+
booleanArg,
5+
exec,
6+
execAsyncWithOutput,
7+
packageFiles,
8+
} = require('./pack-util.cjs')
9+
const fs = require('fs')
10+
const fsPromises = require('fs/promises')
11+
12+
const args = process.argv.slice(2)
13+
14+
const CWD = process.cwd()
15+
const TARBALLS = `${CWD}/tarballs`
16+
const NEXT_PACKAGES = `${CWD}/packages`
17+
const noBuild = booleanArg(args, '--no-build')
18+
19+
;(async () => {
20+
const { globby } = await import('globby')
21+
22+
// the debuginfo on macos is much smaller, so we don't typically need to strip
23+
const DEFAULT_PACK_NEXT_COMPRESS =
24+
process.platform === 'darwin' ? 'none' : 'strip'
25+
const PACK_NEXT_COMPRESS =
26+
process.env.PACK_NEXT_COMPRESS || DEFAULT_PACK_NEXT_COMPRESS
27+
28+
fs.mkdirSync(TARBALLS, { recursive: true })
29+
30+
if (!noBuild) {
31+
exec('Install Next.js build dependencies', 'pnpm i')
32+
exec('Build Next.js', 'pnpm run build')
33+
}
34+
35+
if (PACK_NEXT_COMPRESS !== 'strip') {
36+
// HACK: delete any pre-existing binaries to force napi-rs to rewrite it
37+
let binaries = await nextSwcBinaries()
38+
await Promise.all(binaries.map((bin) => fsPromises.rm(bin)))
39+
}
40+
41+
exec('Build native modules', 'pnpm run swc-build-native')
42+
43+
const NEXT_TARBALL = `${TARBALLS}/next.tar`
44+
const NEXT_SWC_TARBALL = `${TARBALLS}/next-swc.tar`
45+
const NEXT_MDX_TARBALL = `${TARBALLS}/next-mdx.tar`
46+
const NEXT_ENV_TARBALL = `${TARBALLS}/next-env.tar`
47+
const NEXT_BA_TARBALL = `${TARBALLS}/next-bundle-analyzer.tar`
48+
49+
async function nextSwcBinaries() {
50+
return await globby([`${NEXT_PACKAGES}/next-swc/native/*.node`])
51+
}
52+
53+
// We use neither:
54+
// * npm pack, as it doesn't include native modules in the tarball
55+
// * pnpm pack, as it tries to include target directories and compress them,
56+
// which takes forever.
57+
// Instead, we generate non-compressed tarballs.
58+
async function packWithTar(packagePath, tarballPath, extraArgs = []) {
59+
const paths = await packageFiles(packagePath)
60+
61+
const command = [
62+
'tar',
63+
'-c',
64+
// https://apple.stackexchange.com/a/444073
65+
...(process.platform === 'darwin' ? ['--no-mac-metadata'] : []),
66+
'-f',
67+
tarballPath,
68+
...extraArgs,
69+
'--',
70+
...paths.map((p) => `./${p}`),
71+
]
72+
73+
await execAsyncWithOutput(`Pack ${packagePath}`, command, {
74+
cwd: packagePath,
75+
})
76+
}
77+
78+
// Special-case logic for packing next-swc.
79+
//
80+
// pnpm emits `ERR_FS_FILE_TOO_LARGE` if the tarfile is >2GiB due to limits
81+
// in libuv (https://github.com/libuv/libuv/pull/1501). This is common with
82+
// next-swc due to the large amount of debugging symbols. We can fix this one
83+
// of two ways: strip or compression.
84+
//
85+
// We default to stripping (usually faster), but on Linux, we can compress
86+
// instead with objcopy, keeping debug symbols intact. This is controlled by
87+
// `PACK_NEXT_COMPRESS`.
88+
async function packNextSwc() {
89+
const packagePath = `${NEXT_PACKAGES}/next-swc`
90+
switch (PACK_NEXT_COMPRESS) {
91+
case 'strip':
92+
await execAsyncWithOutput('Stripping next-swc native binary', [
93+
'strip',
94+
...(process.platform === 'darwin' ? ['-x', '-'] : ['--']),
95+
await nextSwcBinaries(),
96+
])
97+
await packWithTar(packagePath, NEXT_SWC_TARBALL)
98+
break
99+
case 'objcopy-zstd':
100+
if (process.platform !== 'linux') {
101+
throw new Error('objcopy-zstd is only supported on Linux')
102+
}
103+
await Promise.all(
104+
(await nextSwcBinaries()).map((bin) =>
105+
execAsyncWithOutput(
106+
'Compressing debug symbols in next-swc native binary',
107+
['objcopy', '--compress-debug-sections=zstd', '--', bin]
108+
)
109+
)
110+
)
111+
await packWithTar(packagePath, NEXT_SWC_TARBALL)
112+
break
113+
case 'none':
114+
await packWithTar(packagePath, NEXT_SWC_TARBALL)
115+
break
116+
default:
117+
throw new Error(
118+
"PACK_NEXT_COMPRESS must be one of 'strip', 'objcopy-zstd', or 'none'"
119+
)
120+
}
121+
}
122+
123+
// build all tarfiles in parallel
124+
await Promise.all([
125+
packNextSwc(),
126+
...[
127+
[`${NEXT_PACKAGES}/next`, NEXT_TARBALL],
128+
[`${NEXT_PACKAGES}/next-mdx`, NEXT_MDX_TARBALL],
129+
[`${NEXT_PACKAGES}/next-env`, NEXT_ENV_TARBALL],
130+
[`${NEXT_PACKAGES}/next-bundle-analyzer`, NEXT_BA_TARBALL],
131+
].map(([packagePath, tarballPath]) =>
132+
packWithTar(packagePath, tarballPath)
133+
),
134+
])
135+
136+
console.log('Add the following overrides to your workspace package.json:')
137+
console.log(` "pnpm": {`)
138+
console.log(` "overrides": {`)
139+
console.log(` "next": ${JSON.stringify(`file:${NEXT_TARBALL}`)},`)
140+
console.log(
141+
` "@next/mdx": ${JSON.stringify(`file:${NEXT_MDX_TARBALL}`)},`
142+
)
143+
console.log(
144+
` "@next/env": ${JSON.stringify(`file:${NEXT_ENV_TARBALL}`)},`
145+
)
146+
console.log(
147+
` "@next/bundle-analyzer": ${JSON.stringify(
148+
`file:${NEXT_BA_TARBALL}`
149+
)}`
150+
)
151+
console.log(` }`)
152+
console.log(` }`)
153+
console.log()
154+
console.log('Add the following dependencies to your workspace package.json:')
155+
console.log(` "dependencies": {`)
156+
console.log(` "@next/swc": ${JSON.stringify(`file:${NEXT_SWC_TARBALL}`)},`)
157+
console.log(` ...`)
158+
console.log(` }`)
159+
console.log()
160+
})()

‎scripts/pack-util.cjs

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
const { execSync, execFileSync, spawn } = require('child_process')
2+
const { existsSync } = require('fs')
3+
const { join } = require('path')
4+
5+
function exec(title, command, opts) {
6+
if (Array.isArray(command)) {
7+
logCommand(title, command)
8+
return execFileSync(command[0], command.slice(1), {
9+
stdio: 'inherit',
10+
...opts,
11+
})
12+
} else {
13+
logCommand(title, command)
14+
return execSync(command, {
15+
stdio: 'inherit',
16+
...opts,
17+
})
18+
}
19+
}
20+
21+
exports.exec = exec
22+
23+
function execAsyncWithOutput(title, command, opts) {
24+
logCommand(title, command)
25+
const proc = spawn(command[0], command.slice(1), {
26+
encoding: 'utf8',
27+
stdio: ['inherit', 'pipe', 'pipe'],
28+
...opts,
29+
})
30+
const stdout = []
31+
proc.stdout.on('data', (data) => {
32+
process.stdout.write(data)
33+
stdout.push(data)
34+
})
35+
const stderr = []
36+
proc.stderr.on('data', (data) => {
37+
process.stderr.write(data)
38+
stderr.push(data)
39+
})
40+
return new Promise((resolve, reject) => {
41+
proc.on('exit', (code) => {
42+
if (code === 0) {
43+
return resolve({
44+
stdout: Buffer.concat(stdout),
45+
stderr: Buffer.concat(stderr),
46+
})
47+
}
48+
const err = new Error(
49+
`Command failed with exit code ${code}: ${prettyCommand(command)}`
50+
)
51+
err.code = code
52+
err.stdout = Buffer.concat(stdout)
53+
err.stderr = Buffer.concat(stderr)
54+
reject(err)
55+
})
56+
})
57+
}
58+
59+
exports.execAsyncWithOutput = execAsyncWithOutput
60+
61+
function prettyCommand(command) {
62+
if (Array.isArray(command)) command = command.join(' ')
63+
return command.replace(/ -- .*/, ' -- …')
64+
}
65+
66+
function logCommand(title, command) {
67+
if (command) {
68+
const pretty = prettyCommand(command)
69+
console.log(`\n\x1b[1;4m${title}\x1b[0m\n> \x1b[1m${pretty}\x1b[0m\n`)
70+
} else {
71+
console.log(`\n\x1b[1;4m${title}\x1b[0m\n`)
72+
}
73+
}
74+
75+
exports.logCommand = logCommand
76+
77+
function booleanArg(args, name) {
78+
const index = args.indexOf(name)
79+
if (index === -1) return false
80+
args.splice(index, 1)
81+
return true
82+
}
83+
84+
exports.booleanArg = booleanArg
85+
86+
const DEFAULT_GLOBS = ['**', '!target', '!node_modules', '!crates', '!.turbo']
87+
const FORCED_GLOBS = ['package.json', 'README*', 'LICENSE*', 'LICENCE*']
88+
async function packageFiles(path) {
89+
const { globby } = await import('globby')
90+
const { files = DEFAULT_GLOBS, main, bin } = require(`${path}/package.json`)
91+
92+
const allFiles = files.concat(
93+
FORCED_GLOBS,
94+
main ?? [],
95+
Object.values(bin ?? {})
96+
)
97+
const isGlob = (f) => f.includes('*') || f.startsWith('!')
98+
const simpleFiles = allFiles
99+
.filter((f) => !isGlob(f) && existsSync(join(path, f)))
100+
.map((f) => f.replace(/^\.\//, ''))
101+
const globFiles = allFiles.filter(isGlob)
102+
const globbedFiles = await globby(globFiles, { cwd: path })
103+
const packageFiles = [...globbedFiles, ...simpleFiles].sort()
104+
const set = new Set()
105+
return packageFiles.filter((f) => {
106+
if (set.has(f)) return false
107+
// We add the full path, but check for parent directories too.
108+
// This catches the case where the whole directory is added and then a single file from the directory.
109+
// The sorting before ensures that the directory comes before the files inside of the directory.
110+
set.add(f)
111+
while (f.includes('/')) {
112+
f = f.replace(/\/[^/]+$/, '')
113+
if (set.has(f)) return false
114+
}
115+
return true
116+
})
117+
}
118+
exports.packageFiles = packageFiles

‎scripts/unpack-next.cjs

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/usr/bin/env node
2+
3+
const { exec } = require('./pack-util.cjs')
4+
const fs = require('fs')
5+
const path = require('path')
6+
7+
const CWD = process.cwd()
8+
const TARBALLS = `${CWD}/tarballs`
9+
10+
const PROJECT_DIR = path.resolve(process.argv[2])
11+
12+
function realPathIfAny(path) {
13+
try {
14+
return fs.realpathSync(path)
15+
} catch {
16+
return null
17+
}
18+
}
19+
const packages = {
20+
next: realPathIfAny(`${PROJECT_DIR}/node_modules/next`),
21+
'next-swc': realPathIfAny(`${PROJECT_DIR}/node_modules/@next/swc`),
22+
'next-mdx': realPathIfAny(`${PROJECT_DIR}/node_modules/@next/mdx`),
23+
'next-bundle-analyzer': realPathIfAny(
24+
`${PROJECT_DIR}/node_modules/@next/bundle-anlyzer`
25+
),
26+
}
27+
28+
for (const [key, path] of Object.entries(packages)) {
29+
if (!path) continue
30+
exec(`Unpack ${key}`, `tar -xf '${TARBALLS}/${key}.tar' -C '${path}'`)
31+
}

0 commit comments

Comments
 (0)