Add OSS CI workflow for MacOS M1 (#32)

Summary:
This builds and runs Executorch on MacOS M1, similar to the Linux x86_64 job

Pull Request resolved: #32

Differential Revision: D47929778

Pulled By: huydhn

fbshipit-source-id: 77177ce9087351d03aac4c4abccbdabe8189168f

Author: huydhn

.ci/docker/build.sh

@@ -5,7 +5,7 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-set -ex
+set -exu
 
 IMAGE_NAME="$1"
 shift
@@ -17,10 +17,8 @@ OS_VERSION=22.04
 CLANG_VERSION=12
 PYTHON_VERSION=3.10
 MINICONDA_VERSION=23.5.1-0
-
-# TODO: Pin PyTorch version for now until we have the CI in place to update this
-# safely
-TORCH_VERSION=2.1.0.dev20230731
+TORCH_VERSION=$(cat ci_commit_pins/pytorch.txt)
+BUCK2_VERSION=$(cat ci_commit_pins/buck2.txt)
 
 docker build \
   --no-cache \
@@ -30,6 +28,7 @@ docker build \
   --build-arg "PYTHON_VERSION=${PYTHON_VERSION}" \
   --build-arg "MINICONDA_VERSION=${MINICONDA_VERSION}" \
   --build-arg "TORCH_VERSION=${TORCH_VERSION}" \
+  --build-arg "BUCK2_VERSION=${BUCK2_VERSION}" \
   -f "${OS}"/Dockerfile \
   "$@" \
   .

.ci/docker/ci_commit_pins/buck2.txt

@@ -0,0 +1 @@
+2023-08-01

.ci/docker/ci_commit_pins/pytorch.txt

@@ -0,0 +1 @@
+2.1.0.dev20230731

.ci/docker/common/install_buck.sh

@@ -11,13 +11,14 @@ install_ubuntu() {
   apt-get update
   apt-get install -y zstd
 
-  wget -q https://github.com/facebook/buck2/releases/download/2023-07-18/buck2-x86_64-unknown-linux-gnu.zst
-  zstd -d buck2-x86_64-unknown-linux-gnu.zst -o buck2
+  BUCK2=buck2-x86_64-unknown-linux-gnu.zst
+  wget -q "https://github.com/facebook/buck2/releases/download/${BUCK2_VERSION}/${BUCK2}"
+  zstd -d "${BUCK2}" -o buck2
 
   chmod +x buck2
   mv buck2 /usr/bin/
 
-  rm buck2-x86_64-unknown-linux-gnu.zst
+  rm "${BUCK2}"
   # Cleanup package manager
   apt-get autoclean && apt-get clean
   rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

.ci/docker/requirements-ci.txt

@@ -1,4 +1,6 @@
 mpmath==1.3.0
+numpy==1.25.2
 PyYAML==6.0.1
 ruamel.yaml==0.17.32
 sympy==1.12
+zstd==1.5.5.1

.ci/docker/ubuntu/Dockerfile

@@ -16,6 +16,7 @@ COPY ./common/install_clang.sh install_clang.sh
 RUN bash ./install_clang.sh && rm install_clang.sh
 
 # Setup buck
+ARG BUCK2_VERSION
 COPY ./common/install_buck.sh install_buck.sh
 RUN bash ./install_buck.sh && rm install_buck.sh
 

.ci/scripts/setup-macos.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -exu
+
+install_buck() {
+  if ! command -v zstd &> /dev/null; then
+    brew install zstd
+  fi
+
+  if ! command -v wget &> /dev/null; then
+    brew install wget
+  fi
+
+  if ! command -v buck2 &> /dev/null; then
+    pushd .ci/docker
+
+    BUCK2=buck2-aarch64-apple-darwin.zst
+    BUCK2_VERSION=$(cat ci_commit_pins/buck2.txt)
+
+    wget -q "https://github.com/facebook/buck2/releases/download/${BUCK2_VERSION}/${BUCK2}"
+    zstd -d "${BUCK2}" -o buck2
+
+    chmod +x buck2
+    mv buck2 /opt/homebrew/bin
+
+    rm "${BUCK2}"
+    popd
+  fi
+}
+
+install_conda() {
+  pushd .ci/docker
+  # Install conda dependencies like flatbuffer
+  conda install --file conda-env-ci.txt
+  popd
+}
+
+install_pip_dependencies() {
+  pushd .ci/docker
+  # Install all Python dependencies, including PyTorch
+  pip install --progress-bar off -r requirements-ci.txt
+
+  TORCH_VERSION=$(cat ci_commit_pins/pytorch.txt)
+  pip install --progress-bar off --pre torch=="${TORCH_VERSION}" --index-url https://download.pytorch.org/whl/nightly/cpu
+  popd
+}
+
+install_buck
+install_conda
+install_pip_dependencies

.ci/scripts/test.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -exu
+
+install_executorch() {
+  which pip
+  # Install executorch, this assumes that Executorch is checked out in the
+  # current directory
+  pip install .
+  # Just print out the list of packages for debugging
+  pip list
+}
+
+build_and_test_executorch() {
+  # Build executorch runtime
+  buck2 build //examples/executor_runner:executor_runner
+
+  which python
+  # Export a test model
+  python -m examples.export.export_example --model_name="linear"
+  # Run test model
+  buck2 run //examples/executor_runner:executor_runner -- --model_path ./linear.pte
+}
+
+install_executorch
+build_and_test_executorch

.github/workflows/pull.yml

@@ -12,8 +12,8 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  buck-build-test:
-    name: buck-build-test
+  buck-build-test-linux:
+    name: buck-build-test-linux
     uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
     with:
       runner: linux.2xlarge
@@ -27,14 +27,23 @@ jobs:
         # here, as it's there in the container
         export PATH="/opt/conda/envs/py_${PYTHON_VERSION}/bin:${PATH}"
 
-        # Install executorch
-        pip3 install .
-        # Just print out the list of packages for debugging
-        pip3 list
+        # Build and test Executorch
+        bash .ci/scripts/test.sh
 
-        # Build executorch runtime
-        buck2 build //examples/executor_runner:executor_runner
-        # Export a test model
-        python3 -m examples.export.export_example --model_name="linear"
-        # Run test model
-        buck2 run //examples/executor_runner:executor_runner -- --model_path ./linear.pte
+  buck-build-test-macos:
+    name: buck-build-test-macos
+    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
+    with:
+      runner: macos-m1-12
+      submodules: 'true'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      script: |
+        WORKSPACE=$(pwd)
+
+        pushd "${WORKSPACE}/pytorch/executorch"
+        # Setup MacOS dependencies as there is no Docker support on MacOS atm
+        bash .ci/scripts/setup-macos.sh
+
+        # Build and test Executorch
+        bash .ci/scripts/test.sh
+        popd

setup.py

@@ -16,7 +16,7 @@
 def custom_command():
     src_dst_list = [
         ("schema/scalar_type.fbs", "exir/serialize/scalar_type.fbs"),
-        ("schema/schema.fbs", "exir/serialize/program.fbs"),
+        ("schema/program.fbs", "exir/serialize/program.fbs"),
     ]
     for src, dst in src_dst_list:
         print(f"copying from {src} to {dst}")