Merge branch 'master' into use-cloudfront-native-http-compression

Author: danielcondemarin

README.md

@@ -431,6 +431,7 @@ The fourth cache behaviour handles next API requests `api/*`.
 | logLambdaExecutionTimes  | `boolean`         | `false`                                                                                                                    | Logs to CloudWatch the default handler performance metrics.                                                                                                                                                                                                                                                                                                                      |
 | minifyHandlers           | `boolean`         | `false`                                                                                                                    | Use minified handlers to reduce code size.                                                                                                                                                                                                                                                                                                                                       |
 | enableHTTPCompression    | `boolean`         | `false`                                                                                                                    | When set to `true` the Lambda@Edge functions for SSR and API requests will use Gzip to compress the response. Note that you shouldn't need to enable this because CloudFront will compress responses for you out of the box.                                                                                                                                                     |
+| deploy                   | `boolean`         | `true`                                                                                                                     | Whether to deploy resources to AWS. Useful if you just need the Lambdas and assets but want to deploy them yourself (available in latest alpha).                                                                                                                                                                                                                                 |
 
 Custom inputs can be configured like this:
 

packages/libs/lambda-at-edge/CHANGELOG.md

@@ -3,6 +3,10 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [1.7.0-alpha.13](https://github.com/serverless-nextjs/serverless-next.js/compare/@sls-next/[email protected]...@sls-next/[email protected]) (2020-10-18)
+
+**Note:** Version bump only for package @sls-next/lambda-at-edge
+
 # [1.7.0-alpha.12](https://github.com/serverless-nextjs/serverless-next.js/compare/@sls-next/[email protected]...@sls-next/[email protected]) (2020-10-16)
 
 ### Bug Fixes

packages/libs/lambda-at-edge/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.7.0-alpha.12",
+  "version": "1.7.0-alpha.13",
   "description": "Provides handlers that can be used in CloudFront Lambda@Edge to deploy next.js applications to the edge",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -39,8 +39,10 @@
     "@types/execa": "^2.0.0",
     "@types/fs-extra": "^9.0.1",
     "@types/jsonwebtoken": "^8.5.0",
+    "@types/klaw": "^3.0.1",
     "@types/node": "^14.0.14",
     "@types/path-to-regexp": "^1.7.0",
+    "klaw": "^3.0.0",
     "rollup": "^2.26.6",
     "rollup-plugin-node-externals": "^2.2.0",
     "rollup-plugin-terser": "^7.0.2",

packages/libs/lambda-at-edge/src/build.ts

@@ -17,9 +17,14 @@ import pathToRegexStr from "./lib/pathToRegexStr";
 import normalizeNodeModules from "./lib/normalizeNodeModules";
 import createServerlessConfig from "./lib/createServerlessConfig";
 import { isTrailingSlashRedirect } from "./routing/redirector";
+import readDirectoryFiles from "./lib/readDirectoryFiles";
+import filterOutDirectories from "./lib/filterOutDirectories";
+import { PrerenderManifest } from "next/dist/build";
+import { Item } from "klaw";
 
 export const DEFAULT_LAMBDA_CODE_DIR = "default-lambda";
 export const API_LAMBDA_CODE_DIR = "api-lambda";
+export const ASSETS_DIR = "assets";
 
 type BuildOptions = {
   args?: string[];
@@ -47,6 +52,7 @@ const defaultBuildOptions = {
 
 class Builder {
   nextConfigDir: string;
+  nextStaticDir: string;
   dotNextDir: string;
   serverlessDir: string;
   outputDir: string;
@@ -55,9 +61,11 @@ class Builder {
   constructor(
     nextConfigDir: string,
     outputDir: string,
-    buildOptions?: BuildOptions
+    buildOptions?: BuildOptions,
+    nextStaticDir?: string
   ) {
     this.nextConfigDir = path.resolve(nextConfigDir);
+    this.nextStaticDir = path.resolve(nextStaticDir ?? nextConfigDir);
     this.dotNextDir = path.join(this.nextConfigDir, ".next");
     this.serverlessDir = path.join(this.dotNextDir, "serverless");
     this.outputDir = outputDir;
@@ -466,6 +474,175 @@ class Builder {
     };
   }
 
+  /**
+   * Build static assets such as client-side JS, public files, static pages, etc.
+   * Note that the upload to S3 is done in a separate deploy step.
+   */
+  async buildStaticAssets(routesManifest: RoutesManifest) {
+    const basePath = routesManifest.basePath;
+    const nextConfigDir = this.nextConfigDir;
+    const nextStaticDir = this.nextStaticDir;
+
+    const dotNextDirectory = path.join(this.nextConfigDir, ".next");
+
+    const assetOutputDirectory = path.join(this.outputDir, ASSETS_DIR);
+
+    const normalizedBasePath = basePath ? basePath.slice(1) : "";
+    const withBasePath = (key: string): string =>
+      path.join(normalizedBasePath, key);
+
+    const copyIfExists = async (
+      source: string,
+      destination: string
+    ): Promise<void> => {
+      if (await fse.pathExists(source)) {
+        await fse.copy(source, destination);
+      }
+    };
+
+    const buildStaticFiles = await readDirectoryFiles(
+      path.join(dotNextDirectory, "static")
+    );
+
+    const staticFileAssets = buildStaticFiles
+      .filter(filterOutDirectories)
+      .map(async (fileItem: Item) => {
+        const source = fileItem.path;
+        const destination = path.join(
+          assetOutputDirectory,
+          withBasePath(
+            path
+              .relative(path.resolve(nextConfigDir), source)
+              .replace(/^.next/, "_next")
+          )
+        );
+
+        return copyIfExists(source, destination);
+      });
+
+    const pagesManifest = await fse.readJSON(
+      path.join(dotNextDirectory, "serverless/pages-manifest.json")
+    );
+
+    const htmlPageAssets = Object.values(pagesManifest)
+      .filter((pageFile) => (pageFile as string).endsWith(".html"))
+      .map((relativePageFilePath) => {
+        const source = path.join(
+          dotNextDirectory,
+          `serverless/${relativePageFilePath}`
+        );
+        const destination = path.join(
+          assetOutputDirectory,
+          withBasePath(
+            `static-pages/${(relativePageFilePath as string).replace(
+              /^pages\//,
+              ""
+            )}`
+          )
+        );
+
+        return copyIfExists(source, destination);
+      });
+
+    const prerenderManifest: PrerenderManifest = await fse.readJSON(
+      path.join(dotNextDirectory, "prerender-manifest.json")
+    );
+
+    const prerenderManifestJSONPropFileAssets = Object.keys(
+      prerenderManifest.routes
+    ).map((key) => {
+      const source = path.join(
+        dotNextDirectory,
+        `serverless/pages/${
+          key.endsWith("/") ? key + "index.json" : key + ".json"
+        }`
+      );
+      const destination = path.join(
+        assetOutputDirectory,
+        withBasePath(prerenderManifest.routes[key].dataRoute.slice(1))
+      );
+
+      return copyIfExists(source, destination);
+    });
+
+    const prerenderManifestHTMLPageAssets = Object.keys(
+      prerenderManifest.routes
+    ).map((key) => {
+      const relativePageFilePath = key.endsWith("/")
+        ? path.join(key, "index.html")
+        : key + ".html";
+
+      const source = path.join(
+        dotNextDirectory,
+        `serverless/pages/${relativePageFilePath}`
+      );
+      const destination = path.join(
+        assetOutputDirectory,
+        withBasePath(path.join("static-pages", relativePageFilePath))
+      );
+
+      return copyIfExists(source, destination);
+    });
+
+    const fallbackHTMLPageAssets = Object.values(
+      prerenderManifest.dynamicRoutes || {}
+    )
+      .filter(({ fallback }) => {
+        return !!fallback;
+      })
+      .map((routeConfig) => {
+        const fallback = routeConfig.fallback as string;
+
+        const source = path.join(
+          dotNextDirectory,
+          `serverless/pages/${fallback}`
+        );
+
+        const destination = path.join(
+          assetOutputDirectory,
+          withBasePath(path.join("static-pages", fallback))
+        );
+
+        return copyIfExists(source, destination);
+      });
+
+    const buildPublicOrStaticDirectory = async (
+      directory: "public" | "static"
+    ) => {
+      const directoryPath = path.join(nextStaticDir, directory);
+      if (!(await fse.pathExists(directoryPath))) {
+        return Promise.resolve([]);
+      }
+
+      const files = await readDirectoryFiles(directoryPath);
+
+      return files.filter(filterOutDirectories).map((fileItem: Item) => {
+        const source = fileItem.path;
+        const destination = path.join(
+          assetOutputDirectory,
+          withBasePath(
+            path.relative(path.resolve(nextStaticDir), fileItem.path)
+          )
+        );
+
+        return fse.copy(source, destination);
+      });
+    };
+
+    const publicDirAssets = await buildPublicOrStaticDirectory("public");
+    const staticDirAssets = await buildPublicOrStaticDirectory("static");
+
+    return Promise.all([
+      ...staticFileAssets, // .next/static
+      ...htmlPageAssets, // prerendered html pages
+      ...prerenderManifestJSONPropFileAssets, // SSG json files
+      ...prerenderManifestHTMLPageAssets, // SSG html files
+      ...fallbackHTMLPageAssets, // fallback files
+      ...publicDirAssets, // public dir
+      ...staticDirAssets // static dir
+    ]);
+  }
+
   async cleanupDotNext(): Promise<void> {
     const exists = await fse.pathExists(this.dotNextDir);
 
@@ -492,6 +669,7 @@ class Builder {
 
     await fse.emptyDir(join(this.outputDir, DEFAULT_LAMBDA_CODE_DIR));
     await fse.emptyDir(join(this.outputDir, API_LAMBDA_CODE_DIR));
+    await fse.emptyDir(join(this.outputDir, ASSETS_DIR));
 
     const { restoreUserConfig } = await createServerlessConfig(
       cwd,
@@ -529,6 +707,13 @@ class Builder {
     if (hasAPIPages) {
       await this.buildApiLambda(apiBuildManifest);
     }
+
+    // Copy static assets to .serverless_nextjs directory
+    const routesManifest = require(join(
+      this.dotNextDir,
+      "routes-manifest.json"
+    ));
+    await this.buildStaticAssets(routesManifest);
   }
 
   /**

packages/libs/lambda-at-edge/src/lib/filterOutDirectories.ts

@@ -0,0 +1,3 @@
+import { Item } from "klaw";
+
+export default (fileItem: Item): boolean => !fileItem.stats.isDirectory();

packages/libs/lambda-at-edge/src/lib/readDirectoryFiles.ts

@@ -0,0 +1,21 @@
+import klaw, { Item } from "klaw";
+import fse from "fs-extra";
+
+const readDirectoryFiles = async (directory: string): Promise<Array<Item>> => {
+  const directoryExists = await fse.pathExists(directory);
+  if (!directoryExists) {
+    return Promise.resolve([]);
+  }
+
+  const items: Item[] = [];
+  return new Promise((resolve, reject) => {
+    klaw(directory.trim())
+      .on("data", (item: Item) => items.push(item))
+      .on("end", () => {
+        resolve(items);
+      })
+      .on("error", reject);
+  });
+};
+
+export default readDirectoryFiles;

packages/libs/lambda-at-edge/tests/build/build-no-api.test.ts

@@ -4,10 +4,7 @@ import execa from "execa";
 import Builder from "../../src/build";
 import { DEFAULT_LAMBDA_CODE_DIR, API_LAMBDA_CODE_DIR } from "../../src/build";
 import { cleanupDir, removeNewLineChars } from "../test-utils";
-import {
-  OriginRequestDefaultHandlerManifest,
-  OriginRequestApiHandlerManifest
-} from "../../types";
+import { OriginRequestDefaultHandlerManifest } from "../../types";
 
 jest.mock("execa");
 
@@ -60,6 +57,9 @@ describe("Builder Tests (no API routes)", () => {
       expect(fseEmptyDirSpy).toBeCalledWith(
         expect.stringContaining(join(".test_sls_next_output", "api-lambda"))
       );
+      expect(fseEmptyDirSpy).toBeCalledWith(
+        expect.stringContaining(join(".test_sls_next_output", "assets"))
+      );
     });
   });
 

packages/libs/lambda-at-edge/tests/build/build.test.ts

@@ -1,8 +1,11 @@
 import { join } from "path";
 import fse from "fs-extra";
 import execa from "execa";
-import Builder from "../../src/build";
-import { DEFAULT_LAMBDA_CODE_DIR, API_LAMBDA_CODE_DIR } from "../../src/build";
+import Builder, {
+  ASSETS_DIR,
+  DEFAULT_LAMBDA_CODE_DIR,
+  API_LAMBDA_CODE_DIR
+} from "../../src/build";
 import { cleanupDir, removeNewLineChars } from "../test-utils";
 import {
   OriginRequestDefaultHandlerManifest,
@@ -80,6 +83,9 @@ describe("Builder Tests", () => {
         expect(fseEmptyDirSpy).toBeCalledWith(
           expect.stringContaining(join(".test_sls_next_output", "api-lambda"))
         );
+        expect(fseEmptyDirSpy).toBeCalledWith(
+          expect.stringContaining(join(".test_sls_next_output", "assets"))
+        );
       });
     });
 
@@ -266,6 +272,40 @@ describe("Builder Tests", () => {
         expect(countLines(apiHandler.toString())).toBeGreaterThan(100); // Arbitrary choice
       });
     });
+
+    describe("Assets Artefact Files", () => {
+      it("copies all assets", async () => {
+        const publicFiles = await fse.readdir(
+          join(outputDir, `${ASSETS_DIR}/public`)
+        );
+        expect(publicFiles).toEqual(["favicon.ico", "sub", "sw.js"]);
+
+        const staticFiles = await fse.readdir(
+          join(outputDir, `${ASSETS_DIR}/static`)
+        );
+        expect(staticFiles).toEqual(["donotdelete.txt"]);
+
+        const nextDataFiles = await fse.readdir(
+          join(outputDir, `${ASSETS_DIR}/_next/data/zsWqBqLjpgRmswfQomanp`)
+        );
+        expect(nextDataFiles).toEqual(["contact.json", "index.json"]);
+
+        const nextStaticFiles = await fse.readdir(
+          join(outputDir, `${ASSETS_DIR}/_next/static`)
+        );
+        expect(nextStaticFiles).toEqual(["chunks"]);
+
+        const staticPagesFiles = await fse.readdir(
+          join(outputDir, `${ASSETS_DIR}/static-pages`)
+        );
+        expect(staticPagesFiles).toEqual([
+          "about.html",
+          "contact.html",
+          "index.html",
+          "terms.html"
+        ]);
+      });
+    });
   });
 
   describe("Minified handlers build", () => {

packages/libs/lambda-at-edge/tests/build/simple-app-fixture/.next/static/chunks/chunk1.js

@@ -1 +1,5 @@
-{}
+{
+  "version": 2,
+  "routes": {
+  }
+}