Create the initial version of the pull workflow to install Executorch pip package

.ci/docker/README.md

@@ -1,7 +1,9 @@
 # Docker images for Executorch CI
 
 This directory contains everything needed to build the Docker images
-that are used in Executorch CI.
+that are used in Executorch CI. The content of this directory are copied
+from PyTorch CI https://github.com/pytorch/pytorch/tree/main/.ci/docker.
+It also uses the same directory structure as PyTorch.
 
 ## Contents
 

.ci/docker/build.sh

@@ -7,15 +7,19 @@ shift
 
 echo "Building ${IMAGE_NAME} Docker image"
 
-OS="ubuntu"
+OS=ubuntu
 OS_VERSION=22.04
 CLANG_VERSION=12
+PYTHON_VERSION=3.10
+MINICONDA_VERSION='23.5.1-0'
 
 docker build \
   --no-cache \
   --progress=plain \
   --build-arg "OS_VERSION=${OS_VERSION}" \
   --build-arg "CLANG_VERSION=${CLANG_VERSION}" \
+  --build-arg "PYTHON_VERSION=${PYTHON_VERSION}" \
+  --build-arg "MINICONDA_VERSION=${MINICONDA_VERSION}" \
   -f "${OS}"/Dockerfile \
   "$@" \
   .

.ci/docker/common/install_base.sh

@@ -16,7 +16,8 @@ install_ubuntu() {
     jq \
     vim \
     unzip \
-    gdb
+    gdb \
+    rsync
 
   # Cleanup package manager
   apt-get autoclean && apt-get clean

.ci/docker/common/install_conda.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -ex
+
+# shellcheck source=/dev/null
+source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
+
+install_miniconda() {
+  BASE_URL="https://repo.anaconda.com/miniconda"
+  CONDA_FILE="Miniconda3-py${PYTHON_VERSION//./}_${MINICONDA_VERSION}-Linux-x86_64.sh"
+
+  mkdir -p /opt/conda
+  chown ci-user:ci-user /opt/conda
+
+  pushd /tmp
+  wget -q "${BASE_URL}/${CONDA_FILE}"
+  # Install miniconda
+  as_ci_user bash "${CONDA_FILE}" -b -f -p "/opt/conda"
+  # Clean up the download file
+  rm "${CONDA_FILE}"
+  popd
+
+  sed -e 's|PATH="\(.*\)"|PATH="/opt/conda/bin:\1"|g' -i /etc/environment
+  export PATH="/opt/conda/bin:$PATH"
+}
+
+install_python() {
+  pushd /opt/conda
+  # Install the correct Python version
+  as_ci_user conda create -n "py_${PYTHON_VERSION}" -y python="${PYTHON_VERSION}"
+  popd
+}
+
+install_pip_dependencies() {
+  pushd /opt/conda
+  # Install all Python dependencies, including PyTorch
+  pip_install -r /opt/conda/requirements-ci.txt
+  pip_install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu
+  popd
+}
+
+install_miniconda
+install_python
+install_pip_dependencies

.ci/docker/common/utils.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+as_ci_user() {
+  # NB: unsetting the environment variables works around a conda bug
+  #     https://github.com/conda/conda/issues/6576
+  # NB: Pass on PATH and LD_LIBRARY_PATH to sudo invocation
+  # NB: This must be run from a directory that the user has access to
+  sudo -E -H -u ci-user env -u SUDO_UID -u SUDO_GID -u SUDO_COMMAND -u SUDO_USER env "PATH=${PATH}" "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" "$@"
+}
+
+conda_install() {
+  # Ensure that the install command don't upgrade/downgrade Python
+  # This should be called as
+  #   conda_install pkg1 pkg2 ... [-c channel]
+  as_ci_user conda install -q -n "py_${PYTHON_VERSION}" -y python="${PYTHON_VERSION}" "$@"
+}
+
+conda_run() {
+  as_ci_user conda run -n "py_${PYTHON_VERSION}" --no-capture-output "$@"
+}
+
+pip_install() {
+  as_ci_user conda run -n "py_${PYTHON_VERSION}" pip install --progress-bar off "$@"
+}

.ci/docker/requirements-ci.txt

@@ -0,0 +1,5 @@
+flatbuffers==2.0
+mpmath==1.3.0
+PyYAML==6.0.1
+ruamel.yaml==0.17.32
+sympy==1.12

.ci/docker/ubuntu/Dockerfile

@@ -23,5 +23,15 @@ RUN bash ./install_buck.sh && rm install_buck.sh
 COPY ./common/install_user.sh install_user.sh
 RUN bash ./install_user.sh && rm install_user.sh
 
+# Install conda and other dependencies
+ARG MINICONDA_VERSION
+ARG PYTHON_VERSION
+ENV PYTHON_VERSION=$PYTHON_VERSION
+ENV PATH /opt/conda/envs/py_$PYTHON_VERSION/bin:/opt/conda/bin:$PATH
+COPY requirements-ci.txt /opt/conda/
+COPY ./common/install_conda.sh install_conda.sh
+COPY ./common/utils.sh utils.sh
+RUN bash ./install_conda.sh && rm install_conda.sh utils.sh /opt/conda/requirements-ci.txt
+
 USER ci-user
 CMD ["bash"]

.github/workflows/pull.yml

@@ -0,0 +1,33 @@
+name: pull
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
+  cancel-in-progress: true
+
+jobs:
+  buck-build-test:
+    name: buck-build-test
+    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
+    with:
+      runner: linux.2xlarge
+      docker-image: executorch-ubuntu-22.04-clang12
+      fetch-depth: 0
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      script: |
+        WORKSPACE=$(pwd)
+
+        pushd "${HOME}"
+        # Create the softlink to the workspace as install.sh requires to run from its parent directory
+        ln -s "${WORKSPACE}" executorch
+        # Install executorch
+        source executorch/install.sh
+
+        # Just print out the list of packages for debugging
+        pip list

install.sh

@@ -50,9 +50,12 @@ main() {
   # Uninstall older pip package if present.
   "${PIP}" uninstall -y executorch
 
-  # Install the tree as a pip package.
-  cd "${et_root}/../../"
-  "${PIP}" install .
+  (
+    # Install the tree as a pip package.
+    pushd "${et_root}/../../"
+    "${PIP}" install .
+    popd
+  )
 
   # Clean up.
   rm -rf "${pip_root}"