chore(ci): add caching to local docker build (#708)

* .ci: Improve dockerfile to include caching

Improves dockerfile / .ci/build.sh script to output lambda zips in a
central location as well as adds buildkit caching to the docker build so
that you don't have to re-download the node_modules of each zip directly
each time you build with .ci/build.sh

This also makes improvements in that if there are only changes in one
directory then the cache will remain and the remaining lambdas will not
be re-built.

Signed-off-by: Eli Uriegas <[email protected]>

* Add comment about memory size

Author: seemethere

.ci/Dockerfile

@@ -1,13 +1,26 @@
-FROM node:12
-
+#syntax=docker/dockerfile:1.2
+FROM node:12 as build
+WORKDIR /lambda
 RUN apt-get update \
-    && apt-get install -y zip \
-    && rm -rf /var/lib/apt/lists/*
+        && apt-get install -y zip \
+        && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /lambda
+FROM build as runner-binaries-syncer
+COPY modules/runner-binaries-syncer/lambdas/runner-binaries-syncer /lambda
+RUN --mount=type=cache,target=/lambda/node_modules,id=runner-binaries-syncer \
+        yarn install && yarn dist
 
-COPY . /lambda
+FROM build as runners
+COPY modules/runners/lambdas/runners /lambda
+RUN --mount=type=cache,target=/lambda/node_modules,id=runners \
+        yarn install && yarn dist
 
-RUN yarn install \
-    && yarn run dist
+FROM build as webhook
+COPY modules/webhook/lambdas/webhook /lambda
+RUN --mount=type=cache,target=/lambda/node_modules,id=webhook \
+        yarn install && yarn dist
 
+FROM scratch as final
+COPY --from=runner-binaries-syncer /lambda/runner-binaries-syncer.zip /runner-binaries-syncer.zip
+COPY --from=runners                /lambda/runners.zip                /runners.zip
+COPY --from=webhook                /lambda/webhook.zip                /webhook.zip

.ci/build.sh

@@ -1,14 +1,22 @@
 #!/usr/bin/env bash
 set -e
 
-lambdaSrcDirs=("modules/runner-binaries-syncer/lambdas/runner-binaries-syncer" "modules/runners/lambdas/runners" "modules/webhook/lambdas/webhook")
-repoRoot=$(dirname $(dirname $(realpath ${BASH_SOURCE[0]})))
-
-for lambdaDir in ${lambdaSrcDirs[@]}; do
-    cd "$repoRoot/${lambdaDir}"
-    docker build -t lambda -f ../../../../.ci/Dockerfile .
-    docker create --name lambda lambda
-    zipName=$(basename "$PWD")
-    docker cp lambda:/lambda/${zipName}.zip ${zipName}.zip
-    docker rm lambda
-done
+# NOTE: This build requires docker buildkit integration which was introduced
+#       in Docker v19.03+ and at least 4GB of memory available to the 
+#       docker daemon
+
+set -eou pipefail
+
+TOP_DIR=$(git rev-parse --show-toplevel)
+OUTPUT_DIR=${OUTPUT_DIR:-${TOP_DIR}/lambda_output}
+
+mkdir -p "${OUTPUT_DIR}"
+
+(
+    set -x
+    DOCKER_BUILDKIT=1 docker build \
+        --target=final \
+        --output=type=local,dest="${OUTPUT_DIR}" \
+        -f "${TOP_DIR}/.ci/Dockerfile" \
+        "${TOP_DIR}"
+)