Switch Apple benchmark workflow to use the generic ET benchmark iOS app

.github/workflows/apple-perf.yml

@@ -204,22 +204,19 @@ jobs:
           if-no-files-found: ignore
           path: ${{ runner.temp }}/artifacts/
 
-  build-llm-demo:
-    name: build-llm-demo
+  build-benchmark-app:
+    name: build-benchmark-app
     uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
     needs:
       - set-parameters
     secrets: inherit
-    strategy:
-      matrix:
-          tokenizer: [bpe]
     with:
       runner: macos-latest-xlarge
       python-version: '3.11'
       submodules: 'true'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       upload-artifact: ios-apps
-      secrets-env: BUILD_CERTIFICATE_BASE64 BUILD_PROVISION_PROFILE_BASE64 KEYCHAIN_PASSWORD
+      secrets-env: BUILD_CERTIFICATE_BASE64 EXECUTORCH_BENCHMARK_BUILD_PROVISION_PROFILE_BASE64 KEYCHAIN_PASSWORD
       timeout: 90
       script: |
         set -eux
@@ -234,7 +231,7 @@ jobs:
         export ARTIFACTS_DIR_NAME=artifacts-to-be-uploaded
 
         # Setup Apple certificate for iOS development
-        BUILD_PROVISION_PROFILE_BASE64="${SECRET_BUILD_PROVISION_PROFILE_BASE64}" \
+        BUILD_PROVISION_PROFILE_BASE64="${SECRET_EXECUTORCH_BENCHMARK_BUILD_PROVISION_PROFILE_BASE64}" \
         BUILD_CERTIFICATE_BASE64="${SECRET_BUILD_CERTIFICATE_BASE64}" \
         KEYCHAIN_PASSWORD="${SECRET_KEYCHAIN_PASSWORD}" \
         .ci/scripts/setup-ios.sh
@@ -248,11 +245,38 @@ jobs:
           backends/apple/mps/install_requirements.sh
         echo "::endgroup::"
 
+        echo "::group::Build ExecuTorch iOS frameworks"
+        FRAMEWORKS=(
+          "executorch"
+          "backend_coreml"
+          "backend_mps"
+          "backend_xnnpack"
+          "kernels_custom"
+          "kernels_optimized"
+          "kernels_portable"
+          "kernels_quantized"
+        )
+
+        # Build Release iOS Frameworks
+        PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \
+          build/build_apple_frameworks.sh --coreml --custom --mps --optimized --portable --quantized --xnnpack
+
+        mkdir -p extension/apple/Benchmark/Frameworks
+        for FRAMEWORK in "${FRAMEWORKS[@]}"; do (
+          cp -r "cmake-out/${FRAMEWORK}.xcframework" extension/apple/Benchmark/Frameworks/
+        ) done
+        echo "::endgroup::"
+
+        # NB: Although exported models can be copied to this directory and bundled together with the
+        # app, we don't use this in CI and rely on AWS extra data parameter to make the model and the
+        # tokenizer available to the benchmark. This decouples the app and the model. We just need to
+        # create the directory here to pass the build
+        mkdir -p extension/apple/Benchmark/Models
         ${CONDA_RUN} --no-capture-output \
-          build/build_apple_llm_demo.sh ${{ matrix.tokenizer }} ${ARTIFACTS_DIR_NAME}
+          build/build_apple_llm_demo.sh ${ARTIFACTS_DIR_NAME}
 
-  upload-ios-apps:
-    needs: build-llm-demo
+  upload-benchmark-app:
+    needs: build-benchmark-app
     runs-on: linux.2xlarge
     steps:
       - name: Download the apps from GitHub
@@ -281,7 +305,7 @@ jobs:
   benchmark-on-device:
     needs:
       - set-parameters
-      - upload-ios-apps
+      - upload-benchmark-app
       - upload-models
     permissions:
       id-token: write
@@ -302,7 +326,7 @@ jobs:
       project-arn: arn:aws:devicefarm:us-west-2:308535385114:project:02a2cf0f-6d9b-45ee-ba1a-a086587469e6
       device-pool-arn: ${{ matrix.device }}
       # Uploaded to S3 from the previous job
-      ios-ipa-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/LLaMAPerfBenchmark.ipa
-      ios-xctestrun-zip: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/LLaMAPerfBenchmark.xctestrun.zip
+      ios-ipa-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/Benchmark.ipa
+      ios-xctestrun-zip: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/Benchmark.xctestrun.zip
       test-spec: ${{ inputs.test_spec || 'https://ossci-ios.s3.amazonaws.com/executorch/default-ios-device-farm-appium-test-spec.yml' }}
       extra-data: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/${{ matrix.model }}_${{ matrix.delegate }}/model.zip

.github/workflows/apple.yml

@@ -15,7 +15,7 @@ on:
       - build/build_apple_frameworks.sh
       - build/create_frameworks.sh
       - build/test_ios_ci.sh
-      - examples/demo-apps/apple/**
+      - examples/demo-apps/apple_ios/**
       - extension/apple/**
       - extension/module/**
   workflow_dispatch:
@@ -35,7 +35,7 @@ concurrency:
 
 jobs:
   build-demo-ios:
-    name: test-demo-ios
+    name: build-demo-ios
     uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
     secrets: inherit
     with:

build/build_apple_llm_demo.sh

@@ -7,50 +7,33 @@
 
 set -euo pipefail
 
-TOKENIZER="${1:-bpe}"
-ARTIFACTS_DIR_NAME="$2"
-
-APP_PATH="examples/demo-apps/apple_ios/LLaMA/LLaMA"
-
-if [[ "${TOKENIZER}" = "bpe" ]]; then
-  xcodebuild build-for-testing \
-    -project "${APP_PATH}.xcodeproj" \
-    -scheme LLaMAPerfBenchmark \
-    -destination platform="iOS" \
-    -allowProvisioningUpdates \
-    DEVELOPMENT_TEAM=78E7V7QP35 \
-    CODE_SIGN_STYLE=Manual \
-    PROVISIONING_PROFILE_SPECIFIER=iLLaMA \
-    CODE_SIGN_IDENTITY="iPhone Distribution" \
-    CODE_SIGNING_REQUIRED=No \
-    CODE_SIGNING_ALLOWED=No \
-    GCC_PREPROCESSOR_DEFINITIONS="DEBUG=1 ET_USE_TIKTOKEN=0"
-else
-  xcodebuild build-for-testing \
-    -project "${APP_PATH}.xcodeproj" \
-    -scheme LLaMAPerfBenchmark \
-    -destination platform="iOS" \
-    -allowProvisioningUpdates \
-    DEVELOPMENT_TEAM=78E7V7QP35 \
-    CODE_SIGN_STYLE=Manual \
-    PROVISIONING_PROFILE_SPECIFIER=iLLaMA \
-    CODE_SIGN_IDENTITY="iPhone Distribution" \
-    CODE_SIGNING_REQUIRED=No \
-    CODE_SIGNING_ALLOWED=No
-fi
+ARTIFACTS_DIR_NAME="$1"
+APP_PATH="extension/apple/Benchmark/Benchmark"
+
+xcodebuild build-for-testing \
+  -project "${APP_PATH}.xcodeproj" \
+  -scheme Benchmark \
+  -destination "platform=iOS" \
+  -sdk iphoneos \
+  -allowProvisioningUpdates \
+  DEVELOPMENT_TEAM=78E7V7QP35 \
+  CODE_SIGN_STYLE=Manual \
+  PROVISIONING_PROFILE_SPECIFIER="ExecuTorch Benchmark" \
+  CODE_SIGN_IDENTITY="iPhone Distribution" \
+  CODE_SIGNING_REQUIRED=No \
+  CODE_SIGNING_ALLOWED=No
 
 # The hack to figure out where the xctest package locates
 BUILD_DIR=$(xcodebuild -showBuildSettings -project "$APP_PATH.xcodeproj" -json | jq -r ".[0].buildSettings.BUILD_DIR")
 
 # Prepare the demo app, debug mode here is the default from xcodebuild and match
 # with what we have in the test spec
-# TODO (huydhn): See if we can switch to release mode here
-MODE="Debug"
+MODE="Release"
 PLATFORM="iphoneos"
 pushd "${BUILD_DIR}/${MODE}-${PLATFORM}"
 
 rm -rf Payload && mkdir Payload
-APP_NAME=LLaMAPerfBenchmark
+APP_NAME=Benchmark
 
 ls -lah
 cp -r "${APP_NAME}.app" Payload && zip -vr "${APP_NAME}.ipa" Payload

examples/demo-apps/apple_ios/default-ios-device-farm-appium-test-spec.yml

@@ -11,8 +11,10 @@ phases:
   pre_test:
     commands:
       - mkdir $DEVICEFARM_TEST_PACKAGE_PATH/Debug-iphoneos
+      - mkdir $DEVICEFARM_TEST_PACKAGE_PATH/Release-iphoneos
       - unzip $DEVICEFARM_APP_PATH -d /tmp
-      - mv /tmp/Payload/*.app $DEVICEFARM_TEST_PACKAGE_PATH/Debug-iphoneos/
+      - cp -r /tmp/Payload/*.app $DEVICEFARM_TEST_PACKAGE_PATH/Debug-iphoneos/
+      - cp -r /tmp/Payload/*.app $DEVICEFARM_TEST_PACKAGE_PATH/Release-iphoneos/
 
   # The test phase includes commands that run your test suite execution.
   test:

extension/apple/Benchmark/Benchmark.xcodeproj/project.pbxproj

@@ -411,7 +411,8 @@
 				PRODUCT_BUNDLE_IDENTIFIER = org.pytorch.executorch.Benchmark;
 				PRODUCT_NAME = Benchmark;
 				SDKROOT = auto;
-				SUPPORTED_PLATFORMS = "iphoneos iphonesimulator macosx";
+				SUPPORTED_PLATFORMS = "iphoneos iphonesimulator";
+				SUPPORTS_MACCATALYST = NO;
 				SWIFT_EMIT_LOC_STRINGS = YES;
 				SWIFT_VERSION = 5.0;
 				TARGETED_DEVICE_FAMILY = "1,2";
@@ -444,7 +445,8 @@
 				PRODUCT_BUNDLE_IDENTIFIER = org.pytorch.executorch.Benchmark;
 				PRODUCT_NAME = Benchmark;
 				SDKROOT = auto;
-				SUPPORTED_PLATFORMS = "iphoneos iphonesimulator macosx";
+				SUPPORTED_PLATFORMS = "iphoneos iphonesimulator";
+				SUPPORTS_MACCATALYST = NO;
 				SWIFT_EMIT_LOC_STRINGS = YES;
 				SWIFT_VERSION = 5.0;
 				TARGETED_DEVICE_FAMILY = "1,2";
@@ -467,7 +469,8 @@
 				PRODUCT_BUNDLE_IDENTIFIER = org.pytorch.executorch.BenchmarkTests;
 				PRODUCT_NAME = "$(TARGET_NAME)";
 				SDKROOT = auto;
-				SUPPORTED_PLATFORMS = "iphoneos iphonesimulator macosx";
+				SUPPORTED_PLATFORMS = "iphoneos iphonesimulator";
+				SUPPORTS_MACCATALYST = NO;
 				SWIFT_EMIT_LOC_STRINGS = NO;
 				SWIFT_VERSION = 5.0;
 				TARGETED_DEVICE_FAMILY = "1,2";
@@ -491,7 +494,8 @@
 				PRODUCT_BUNDLE_IDENTIFIER = org.pytorch.executorch.BenchmarkTests;
 				PRODUCT_NAME = "$(TARGET_NAME)";
 				SDKROOT = auto;
-				SUPPORTED_PLATFORMS = "iphoneos iphonesimulator macosx";
+				SUPPORTED_PLATFORMS = "iphoneos iphonesimulator";
+				SUPPORTS_MACCATALYST = NO;
 				SWIFT_EMIT_LOC_STRINGS = NO;
 				SWIFT_VERSION = 5.0;
 				TARGETED_DEVICE_FAMILY = "1,2";