Add support for building Swift-DocC-Render in Swift CI

ci_test.py

@@ -20,14 +20,14 @@
 import os
 
 
-def run_command(cmd, log_file=None):
+def run_command(cmd, log_file=None, cwd=None):
     print("Running: {}".format(cmd))
     sys.stdout.flush()
     if log_file:
         file = open(log_file, "w")
-        p = subprocess.Popen(cmd, shell=True, stdout=file, stderr=file)
+        p = subprocess.Popen(cmd, shell=True, stdout=file, stderr=file, cwd=cwd)
     else:
-        p = subprocess.Popen(cmd, shell=True)
+        p = subprocess.Popen(cmd, shell=True, cwd=cwd)
 
     (output, err) = p.communicate()
     return p.wait()
@@ -45,8 +45,7 @@ def get_dockerfiles():
         filename = file_info['filename']
         print(filename)
         if "Dockerfile" in filename:
-            file_dir = filename.replace("Dockerfile", "")
-            dockerfiles.append(file_dir)
+            dockerfiles.append(filename)
     return dockerfiles
 
 
@@ -69,12 +68,25 @@ def main():
         sys.stdout.flush()
         log_file = dockerfile.replace(docker_dir,"").replace("/", "_")
         log_file = "{}.log".format(log_file)
-        cmd = "docker build --no-cache=true {}".format(dockerfile)
+
+        # Get the root directory for this collection of Dockerfiles. For example, this could be
+        # 'swift-ci' or 'nightly-main'.
+        #
+        # This will allow the test runner to build the Dockerfile from this directory
+        # which may contain additional scripts necessary for the build process.
+        dockerfile_path_components = dockerfile.split(os.sep)
+        dockerfile_root_dir = os.path.join(docker_dir, dockerfile_path_components.pop(0))
+
+        # Join the remaining path components as the relative path to this Dockerfile, from the root
+        # directory
+        dockerfile_relative_path = os.sep.join(dockerfile_path_components)
+
+        cmd = "docker build --no-cache=true -f {} .".format(dockerfile_relative_path)
         if "buildx" in dockerfile:
             # if "buildx" is part of the path, we want to use the new buildx build system and build
             # for both amd64 and arm64.
-            cmd = "docker buildx build --platform linux/arm64,linux/amd64 --no-cache=true {}".format(dockerfile)
-        status = run_command(cmd, log_file)
+            cmd = "docker buildx build --platform linux/arm64,linux/amd64 --no-cache=true -f {} .".format(dockerfile_relative_path)
+        status = run_command(cmd, log_file, dockerfile_root_dir)
         results[dockerfile] = status
         if status != 0:
             suite_status = False

swift-ci/install_node.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2021 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+
+# This script is used to install Node.js in the Docker containers used 
+# to build & qualify Swift toolchains on Linux.
+#
+# Node is specifically required to build swift-docc-render, the template used by swift-docc to
+# render documentation archives.
+
+set -e
+
+NODE_VERSION='v14.17.4'
+
+ARCH=`uname -m`
+if [ ${ARCH} == 'x86_64' ]; then
+  NODE_DISTRO='linux-x64'
+elif [ ${ARCH} == 'aarch64' ]; then
+  NODE_DISTRO='linux-arm64'
+else
+  echo >&2 "Unsupported CPU architecture. Unable to install required Node dependency."
+  exit 1
+fi
+
+NODE_FILENAME=node-$NODE_VERSION-$NODE_DISTRO
+NODE_COMPRESSED_FILENAME=$NODE_FILENAME.tar.gz
+NODE_URL=https://nodejs.org/dist/$NODE_VERSION/$NODE_COMPRESSED_FILENAME
+
+NODE_SHASUM_FILENAME=SHASUMS256.txt
+NODE_SHASUM_URL=https://nodejs.org/dist/$NODE_VERSION/$NODE_SHASUM_FILENAME
+
+if [ -x "$(command -v curl)" ]; then
+  curl -o $NODE_COMPRESSED_FILENAME $NODE_URL
+  curl -o $NODE_SHASUM_FILENAME $NODE_SHASUM_URL
+elif [ -x "$(command -v python)" ]; then
+  python -c "from urllib import urlretrieve; urlretrieve('$NODE_URL', '$NODE_COMPRESSED_FILENAME')"
+  python -c "from urllib import urlretrieve; urlretrieve('$NODE_SHASUM_URL', '$NODE_SHASUM_FILENAME')"
+else
+  echo >&2 "No download command found; install curl or python."
+  exit 1
+fi
+
+if [ -x "$(command -v shasum)" ]; then
+  SHA_CMD="shasum"
+elif [ -x "$(command -v sha256sum)" ]; then
+  SHA_CMD="sha256sum"
+else
+  echo >&2 "No sha command found; install shasum or sha256sum."
+  exit 1
+fi
+
+if grep $NODE_COMPRESSED_FILENAME $NODE_SHASUM_FILENAME | $SHA_CMD -c -; then
+  echo "Node.js binary verified successfully."
+else
+  echo >&2 "Node.js binary could not be verified."
+  exit 1
+fi
+
+mkdir -p /usr/local/lib/nodejs
+tar -xf $NODE_COMPRESSED_FILENAME -C /usr/local/lib/nodejs
+mv /usr/local/lib/nodejs/$NODE_FILENAME /usr/local/lib/nodejs/node

swift-ci/master/amazon-linux/2/Dockerfile

@@ -40,6 +40,11 @@ RUN yum -y install \
   wget             \
   which
 
+# Install the current version of Node.js that swift-docc-render depends on:
+COPY install_node.sh /
+RUN /install_node.sh
+ENV PATH "$PATH:/usr/local/lib/nodejs/node/bin"
+
 RUN mkdir -p /usr/local/lib/python3.7/site-packages/
 
 RUN easy_install-3.7 six

swift-ci/master/centos/7/Dockerfile

@@ -39,6 +39,11 @@ RUN yum install --enablerepo=centosplus  -y \
     which               \
     zlib-devel
 
+# Install the current version of Node.js that swift-docc-render depends on:
+COPY install_node.sh /
+RUN /install_node.sh
+ENV PATH "$PATH:/usr/local/lib/nodejs/node/bin"
+
 RUN echo -e ". /opt/rh/sclo-git25/enable\n. /opt/rh/llvm-toolset-7/enable\n. /opt/rh/devtoolset-8/enable\n" >> /home/build-user/.bashrc
 
 RUN sed -i -e 's/\*__block/\*__libc_block/g' /usr/include/unistd.h

swift-ci/master/centos/8/Dockerfile

@@ -37,6 +37,11 @@ RUN yum install --enablerepo=powertools -y \
     tar                   \
     which
 
+# Install the current version of Node.js that swift-docc-render depends on
+COPY install_node.sh /
+RUN /install_node.sh
+ENV PATH "$PATH:/usr/local/lib/nodejs/node/bin"
+
 RUN ln -s /usr/bin/python2 /usr/bin/python
 
 USER build-user

swift-ci/master/debian/10/Dockerfile

@@ -33,6 +33,11 @@ RUN apt-get -y update && apt-get -y --no-install-recommends install \
   uuid-dev		\
   ca-certificates
 
+# Install the current version of Node.js that swift-docc-render depends on:
+COPY install_node.sh /
+RUN /install_node.sh
+RUN echo "export PATH=$PATH:/usr/local/lib/nodejs/node/bin" >> /home/build-user/.bashrc
+
 USER build-user
 
 WORKDIR /home/build-user

swift-ci/master/debian/9/Dockerfile

@@ -31,6 +31,11 @@ RUN apt-get --no-install-recommends -y install \
   ca-certificates	\
   uuid-dev
 
+# Install the current version of Node.js that swift-docc-render depends on:
+COPY install_node.sh /
+RUN /install_node.sh
+RUN echo "export PATH=$PATH:/usr/local/lib/nodejs/node/bin" >> /home/build-user/.bashrc
+
 USER build-user
 
 WORKDIR /home/build-user

swift-ci/master/fedora/31/Dockerfile

@@ -26,6 +26,11 @@ RUN dnf -y update && dnf install -y \
     rsync                       \
     swig 
 
+# Install the current version of Node.js that swift-docc-render depends on:
+COPY install_node.sh /
+RUN /install_node.sh
+ENV PATH "$PATH:/usr/local/lib/nodejs/node/bin"
+
 USER build-user
 
 WORKDIR /home/build-user

swift-ci/master/fedora/32/Dockerfile

@@ -26,6 +26,11 @@ RUN dnf -y update && dnf install -y \
     rsync                       \
     swig 
 
+# Install the current version of Node.js that swift-docc-render depends on:
+COPY install_node.sh /
+RUN /install_node.sh
+ENV PATH "$PATH:/usr/local/lib/nodejs/node/bin"
+
 USER build-user
 
 WORKDIR /home/build-user

swift-ci/master/fedora/rawhide/Dockerfile

@@ -26,6 +26,11 @@ RUN dnf -y update && dnf install -y \
     rsync                       \
     swig 
 
+# Install the current version of Node.js that swift-docc-render depends on:
+COPY install_node.sh /
+RUN /install_node.sh
+ENV PATH "$PATH:/usr/local/lib/nodejs/node/bin"
+
 USER build-user
 
 WORKDIR /home/build-user

swift-ci/master/ubuntu/18.04/Dockerfile

@@ -30,6 +30,11 @@ RUN apt -y update && apt -y install \
   tzdata                \
   unzip                 \
   uuid-dev
+
+# Install the current version of Node.js that swift-docc-render depends on:
+COPY install_node.sh /
+RUN /install_node.sh
+ENV PATH "$PATH:/usr/local/lib/nodejs/node/bin"
 
 USER build-user
 

swift-ci/master/ubuntu/20.04/Dockerfile

@@ -32,6 +32,11 @@ RUN apt-get -y update && apt-get -y install \
   unzip                 \
   uuid-dev
 
+# Install the current version of Node.js that swift-docc-render depends on:
+COPY install_node.sh /
+RUN /install_node.sh
+ENV PATH "$PATH:/usr/local/lib/nodejs/node/bin"
+
 USER build-user
 
 WORKDIR /home/build-user