Add env handling for experimental tracing (#68516)

This moves the NEXT_PUBLIC env inlining step out of the webpack
compilation so that the experimental tracing doesn't need to invalidate
based on these values allowing better cache re-use.

Closes: NDX-127

Author: ijjk

packages/next/src/build/flying-shuttle/inline-static-env.ts

@@ -0,0 +1,53 @@
+import fs from 'fs'
+import path from 'path'
+import { promisify } from 'util'
+import globOriginal from 'next/dist/compiled/glob'
+import { getNextPublicEnvironmentVariables } from '../webpack/plugins/define-env-plugin'
+import { Sema } from 'next/dist/compiled/async-sema'
+
+const glob = promisify(globOriginal)
+
+export async function inlineStaticEnv({ distDir }: { distDir: string }) {
+  const staticEnv = getNextPublicEnvironmentVariables()
+
+  const serverDir = path.join(distDir, 'server')
+  const serverChunks = await glob('**/*.js', {
+    cwd: serverDir,
+  })
+  const clientDir = path.join(distDir, 'static')
+  const clientChunks = await glob('**/*.js', {
+    cwd: clientDir,
+  })
+
+  const inlineSema = new Sema(8)
+
+  for (const [parentDir, files] of [
+    [serverDir, serverChunks],
+    [clientDir, clientChunks],
+  ] as const) {
+    await Promise.all(
+      files.map(async (file) => {
+        await inlineSema.acquire()
+        const filepath = path.join(parentDir, file)
+        const content = await fs.promises.readFile(filepath, 'utf8')
+        // TODO: should we support process.env['NEXT_PUBLIC_KEY'] format?
+        await fs.promises.writeFile(
+          filepath,
+          content.replace(/['"`]?[\w]{1,}\.env\.[\w]{1,}['"`]?/g, (match) => {
+            const matchParts = match.split('.')
+            let normalizedMatch = `process.env.${matchParts[2].replace(
+              // remove ending quote if present
+              /['"`]$/,
+              ''
+            )}`
+            if (staticEnv[normalizedMatch]) {
+              return JSON.stringify(staticEnv[normalizedMatch])
+            }
+            return match
+          })
+        )
+        inlineSema.release()
+      })
+    )
+  }
+}

packages/next/src/build/index.ts

@@ -194,6 +194,7 @@ import {
 } from './flying-shuttle/detect-changed-entries'
 import { storeShuttle } from './flying-shuttle/store-shuttle'
 import { stitchBuilds } from './flying-shuttle/stitch-builds'
+import { inlineStaticEnv } from './flying-shuttle/inline-static-env'
 
 interface ExperimentalBypassForInfo {
   experimentalBypassFor?: RouteHas[]
@@ -2543,6 +2544,9 @@ export default async function build(
             distDir,
             shuttleDir,
           })
+
+          console.log('inlining static env')
+          await inlineStaticEnv({ distDir })
         }
       }
 

packages/next/src/build/webpack/plugins/define-env-plugin.ts

@@ -139,11 +139,24 @@ export function getDefineEnv({
   isNodeServer,
   middlewareMatchers,
 }: DefineEnvPluginOptions): SerializedDefineEnv {
+  const nextPublicEnv = getNextPublicEnvironmentVariables()
+
+  if (config.experimental.flyingShuttle) {
+    // we delay inlining these values until after the build
+    // with flying shuttle enabled so we can update them
+    // without invalidating entries
+    for (const key in nextPublicEnv) {
+      // we inline as the key itself to avoid process.env
+      // mangling by webpack/minifier making replacing unsafe
+      nextPublicEnv[key] = key
+    }
+  }
+
   const defineEnv: DefineEnv = {
     // internal field to identify the plugin config
     __NEXT_DEFINE_ENV: true,
 
-    ...getNextPublicEnvironmentVariables(),
+    ...nextPublicEnv,
     ...getNextConfigEnv(config),
     ...(!isEdgeServer
       ? {}

test/e2e/app-dir/app/app/dashboard/deployments/[id]/page.js

@@ -22,6 +22,7 @@ export default function DeploymentsPage(props) {
   return (
     <>
       <p>hello from app/dashboard/deployments/[id]. ID is: {data.id}</p>
+      <p id="my-env">{process.env.NEXT_PUBLIC_TEST_ID}</p>
     </>
   )
 }

test/e2e/app-dir/app/flying-shuttle.test.ts

@@ -13,17 +13,22 @@ import { nextTestSetup, isNextStart } from 'e2e-utils'
       it('should skip for non-next start', () => {})
       return
     }
-
     const { next } = nextTestSetup({
       files: __dirname,
       dependencies: {
         nanoid: '4.0.1',
       },
       env: {
-        NEXT_PRIVATE_FLYING_SHUTTLE: 'true',
+        NEXT_PRIVATE_FLYING_SHUTTLE: '1',
       },
     })
     let initialConfig: Record<string, any> = {}
+    let testEnvId = ''
+
+    beforeEach(() => {
+      testEnvId = Math.random() + ''
+      next.env['NEXT_PUBLIC_TEST_ID'] = testEnvId
+    })
 
     beforeAll(async () => {
       const manifest = await next.readJSON(
@@ -123,6 +128,12 @@ import { nextTestSetup, isNextStart } from 'e2e-utils'
     })
 
     async function checkAppPagesNavigation() {
+      // ensure we inlined NEXT_PUBLIC_ properly
+      const index$ = await next.render$('/')
+      const deployments$ = await next.render$('/dashboard/deployments/123')
+      expect(index$('#my-env').text()).toContain(testEnvId)
+      expect(deployments$('#my-env').text()).toContain(testEnvId)
+
       const testPaths = [
         { path: '/', content: 'hello from pages/index', type: 'pages' },
         {
@@ -163,6 +174,12 @@ import { nextTestSetup, isNextStart } from 'e2e-utils'
           ).toContain(content)
         })
 
+        if (path === '/' || path === '/dashboard/deployments/123') {
+          expect(await browser.elementByCss('#my-env').text()).toContain(
+            testEnvId
+          )
+        }
+
         const checkNav = async (testPath: (typeof testPaths)[0]) => {
           await browser.eval(`window.next.router.push("${testPath.path}")`)
 

test/e2e/app-dir/app/pages/index.js

@@ -16,6 +16,7 @@ export default function Page() {
       <Link href="/dashboard">Dashboard</Link>
       <p id="react-version">{React.version}</p>
       <Button>Click me!</Button>
+      <p id="my-env">{process.env.NEXT_PUBLIC_TEST_ID}</p>
     </>
   )
 }