Setup CI to run tests on Simulator. (#839)

Summary:
Pull Request resolved: #839

Preliminary CI rule, waiting for all the changes to get in before landing

Reviewed By: huydhn

Differential Revision: D50181786

fbshipit-source-id: 55785bb346dd71e08d6af27ac6e46b4a1794a213

Author: shoumikhin

.github/workflows/pull.yml

@@ -114,6 +114,29 @@ jobs:
         # Test selective build
         PYTHON_EXECUTABLE=python bash examples/selective_build/test_selective_build.sh "${BUILD_TOOL}"
 
+  test-demo-ios:
+    name: test-demo-ios
+    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
+    strategy:
+      matrix:
+        include:
+          - build-tool: cmake
+      fail-fast: false
+    with:
+      runner: macos-latest-xlarge
+      submodules: 'true'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 60
+      script: |
+        WORKSPACE=$(pwd)
+        pushd "${WORKSPACE}/pytorch/executorch"
+        BUILD_TOOL=${{ matrix.build-tool }}
+        # Setup MacOS dependencies as there is no Docker support on MacOS atm
+        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}"
+        # Build and test iOS Demo App
+        PYTHON_EXECUTABLE=python sh build/test_ios_ci.sh
+        popd
+
   unittest:
     uses: ./.github/workflows/_unittest.yml
     with:

build/build_apple_frameworks.sh

@@ -15,6 +15,7 @@ MODE="Debug"
 TOOLCHAIN=""
 BUCK2="/tmp/buck2"
 PYTHON=$(which python3)
+FLATC=""
 IOS_DEPLOYMENT_TARGET="17.0"
 COREML=OFF
 MPS=OFF
@@ -37,6 +38,7 @@ usage() {
   echo "  --toolchain=FILE     Cmake toolchain file. Default: '\$SOURCE_ROOT_DIR/third-party/pytorch/cmake/iOS.cmake'"
   echo "  --buck2=FILE         Buck2 executable path. Default: '/tmp/buck2'"
   echo "  --python=FILE        Python executable path. Default: Path of python3 found in the current \$PATH"
+  echo "  --flatc=FILE         FlatBuffers Compiler executable path. Default: '\$SOURCE_ROOT_DIR/third-party/flatbuffers/cmake-out/flatc'"
   echo "  --coreml             Include this flag to build Core ML backend."
   echo "  --mps                Include this flag to build Metal Performance Shaders backend."
   echo "  --xnnpack            Include this flag to build XNNPACK backend."
@@ -54,6 +56,7 @@ for arg in "$@"; do
       --toolchain=*) TOOLCHAIN="${arg#*=}" ;;
       --buck2=*) BUCK2="${arg#*=}" ;;
       --python=*) PYTHON="${arg#*=}" ;;
+      --flatc=*) FLATC="${arg#*=}" ;;
       --ios-deployment-target=*) IOS_DEPLOYMENT_TARGET="${arg#*=}" ;;
       --coreml) COREML=ON ;;
       --mps) MPS=ON ;;
@@ -78,6 +81,10 @@ if [[ -z "$TOOLCHAIN" ]]; then
 fi
 [[ -f "$TOOLCHAIN" ]] || { echo >&2 "Toolchain file $TOOLCHAIN does not exist."; exit 1; }
 
+if [[ -z "$FLATC" ]]; then
+    FLATC="$SOURCE_ROOT_DIR/third-party/flatbuffers/cmake-out/flatc"
+fi
+
 check_command() {
   command -v "$1" >/dev/null 2>&1 || { echo >&2 "$1 is not installed"; exit 1; }
 }
@@ -86,13 +93,10 @@ check_command cmake
 check_command rsync
 check_command "$BUCK2"
 check_command "$PYTHON"
+check_command "$FLATC"
 
 echo "Building libraries"
 
-if [[ -z "$SOURCE_ROOT_DIR" ]]; then
-    SOURCE_ROOT_DIR=$(pwd)
-fi
-
 rm -rf "$OUTPUT" && mkdir -p "$OUTPUT" && cd "$OUTPUT" || exit 1
 
 cmake_build() {
@@ -102,9 +106,11 @@ cmake_build() {
     mkdir "$platform" && cd "$platform" || exit 1
     cmake "$SOURCE_ROOT_DIR" -G Xcode \
         -DCMAKE_TOOLCHAIN_FILE="$TOOLCHAIN" \
+        -DCMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LANGUAGE_STANDARD="c++17" \
+        -DCMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LIBRARY="libc++" \
         -DBUCK2="$BUCK2" \
         -DPYTHON_EXECUTABLE="$PYTHON" \
-        -DFLATC_EXECUTABLE="$SOURCE_ROOT_DIR/third-party/flatbuffers/cmake-out/flatc" \
+        -DFLATC_EXECUTABLE="$FLATC" \
         -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
         -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY="$(pwd)" \
         -DIOS_DEPLOYMENT_TARGET="$IOS_DEPLOYMENT_TARGET" \

build/test_ios.sh

@@ -15,32 +15,27 @@ set -e
 
 OUTPUT="${1:-executorch}"
 EXIT_STATUS=0
+APP_PATH="examples/demo-apps/apple_ios/ExecuTorchDemo/ExecuTorchDemo"
+MODEL_NAME="mv3"
+SIMULATOR_NAME="executorch"
 
-report() {
-  if [ $EXIT_STATUS -eq 0 ]; then
-    say "SUCCEEDED"
-  else
-    say "FAILED"
-  fi
-}
-
-cleanup() {
-  if xcrun simctl list | grep -q 'executorch'; then
+finish() {
+  EXIT_STATUS=$?
+  if xcrun simctl list | grep -q "$SIMULATOR_NAME"; then
     say "Deleting Simulator"
-    xcrun simctl delete executorch
+    xcrun simctl delete "$SIMULATOR_NAME"
   fi
-
   if [ -d "$OUTPUT" ]; then
     popd > /dev/null
     say "Deleting Output Directory"
     rm -rf "$OUTPUT"
   fi
-}
-
-finish() {
-  EXIT_STATUS=$?
-  cleanup
-  report
+  if [ $EXIT_STATUS -eq 0 ]; then
+    say "SUCCEEDED"
+  else
+    say "FAILED"
+  fi
+  exit $EXIT_STATUS
 }
 
 trap finish EXIT
@@ -86,31 +81,31 @@ say "Installing MPS Backend Requirements"
 
 say "Exporting Models"
 
-python3 -m examples.portable.scripts.export --model_name="mv3"
-python3 -m examples.apple.coreml.scripts.export_and_delegate --model_name="mv3"
-python3 -m examples.apple.mps.scripts.mps_example --model_name="mv3"
-python3 -m examples.xnnpack.aot_compiler --model_name="mv3" --delegate
+python3 -m examples.portable.scripts.export --model_name="$$MODEL_NAME"
+python3 -m examples.apple.coreml.scripts.export_and_delegate --model_name="$$MODEL_NAME"
+python3 -m examples.apple.mps.scripts.mps_example --model_name="$$MODEL_NAME"
+python3 -m examples.xnnpack.aot_compiler --model_name="$$MODEL_NAME" --delegate
 
-mkdir -p examples/demo-apps/apple_ios/ExecuTorchDemo/ExecuTorchDemo/Resources/Models/MobileNet/
-mv mv3*.pte examples/demo-apps/apple_ios/ExecuTorchDemo/ExecuTorchDemo/Resources/Models/MobileNet/
+mkdir -p "$APP_PATH/Resources/Models/MobileNet/"
+mv $MODEL_NAME*.pte "$APP_PATH/Resources/Models/MobileNet/"
 
 say "Downloading Labels"
 
 curl https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt \
-  -o examples/demo-apps/apple_ios/ExecuTorchDemo/ExecuTorchDemo/Resources/Models/MobileNet/imagenet_classes.txt
+  -o "$APP_PATH/Resources/Models/MobileNet/imagenet_classes.txt"
 
 say "Building Frameworks"
 
 ./build/build_apple_frameworks.sh --buck2="$(realpath .venv/bin/buck2)" --Release --coreml --mps --xnnpack
-mv cmake-out examples/demo-apps/apple_ios/ExecuTorchDemo/ExecuTorchDemo/Frameworks
+mv cmake-out "$APP_PATH/Frameworks"
 
 say "Creating Simulator"
 
-xcrun simctl create executorch "iPhone 15"
+xcrun simctl create "$SIMULATOR_NAME" "iPhone 15"
 
 say "Running Tests"
 
-xcodebuild clean test \
-  -project examples/demo-apps/apple_ios/ExecuTorchDemo/ExecuTorchDemo.xcodeproj \
-  -scheme App \
-  -destination name=executorch
+xcodebuild test \
+  -project "$APP_PATH.xcodeproj" \
+  -scheme MobileNetClassifierTest \
+  -destination name="$SIMULATOR_NAME"

build/test_ios_ci.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -e
+
+APP_PATH="examples/demo-apps/apple_ios/ExecuTorchDemo/ExecuTorchDemo"
+MODEL_NAME="mv3"
+SIMULATOR_NAME="executorch"
+
+finish() {
+  EXIT_STATUS=$?
+  if xcrun simctl list | grep -q "$SIMULATOR_NAME"; then
+    say "Deleting Simulator"
+    xcrun simctl delete "$SIMULATOR_NAME"
+  fi
+  if [ $EXIT_STATUS -eq 0 ]; then
+    say "SUCCEEDED"
+  else
+    say "FAILED"
+  fi
+  exit $EXIT_STATUS
+}
+
+trap finish EXIT
+
+say() {
+  echo -e "\033[1m\n\t** $1 **\n\033[0m"
+}
+
+say "Installing Requirements"
+
+./install_requirements.sh
+
+say "Exporting Models"
+
+python3 -m examples.portable.scripts.export --model_name="$MODEL_NAME"
+python3 -m examples.xnnpack.aot_compiler --model_name="$MODEL_NAME" --delegate
+
+mkdir -p "$APP_PATH/Resources/Models/MobileNet/"
+mv $MODEL_NAME*.pte "$APP_PATH/Resources/Models/MobileNet/"
+
+say "Downloading Labels"
+
+curl https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt \
+  -o "$APP_PATH/Resources/Models/MobileNet/imagenet_classes.txt"
+
+say "Building Frameworks"
+
+./build/build_apple_frameworks.sh --buck2="$(which buck2)" --flatc="$(which flatc)" --xnnpack
+mv cmake-out "$APP_PATH/Frameworks"
+
+say "Creating Simulator"
+
+xcrun simctl create "$SIMULATOR_NAME" "iPhone 14"
+
+say "Running Tests"
+
+xcodebuild test \
+  -project "$APP_PATH.xcodeproj" \
+  -scheme MobileNetClassifierTest \
+  -destination name="$SIMULATOR_NAME" \
+  -skip-testing:MobileNetClassifierTest/MobileNetClassifierTest/testV3WithCoreMLBackend \
+  -skip-testing:MobileNetClassifierTest/MobileNetClassifierTest/testV3WithMPSBackend