Move benchmark apps to extension/benchmark dir

Author: Guang Yang

.github/workflows/android-perf.yml

@@ -292,7 +292,7 @@ jobs:
               --output-dir benchmark-results \
               --repo ${{ github.repository }} \
               --head-branch ${{ github.head_ref || github.ref_name }} \
-              --workflow-name ${{ github.workflow }} \
+              --workflow-name "${{ github.workflow }}" \
               --workflow-run-id ${{ github.run_id }} \
               --workflow-run-attempt ${{ github.run_attempt }}
           done

.github/workflows/android.yml

@@ -15,6 +15,7 @@ on:
       - install_requirements.sh
       - examples/demo-apps/android/**
       - extension/android/**
+      - extension/benchmark/android/**
       - extension/module/**
   workflow_dispatch:
 

.github/workflows/apple-perf.yml

@@ -235,17 +235,17 @@ jobs:
         PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \
           build/build_apple_frameworks.sh --coreml --custom --mps --optimized --portable --quantized --xnnpack
 
-        mkdir -p extension/apple/Benchmark/Frameworks
+        mkdir -p extension/benchmark/apple/Benchmark/Frameworks
         for FRAMEWORK in "${FRAMEWORKS[@]}"; do (
-          cp -r "cmake-out/${FRAMEWORK}.xcframework" extension/apple/Benchmark/Frameworks/
+          cp -r "cmake-out/${FRAMEWORK}.xcframework" extension/benchmark/apple/Benchmark/Frameworks/
         ) done
         echo "::endgroup::"
 
         # NB: Although exported models can be copied to this directory and bundled together with the
         # app, we don't use this in CI and rely on AWS extra data parameter to make the model and the
         # tokenizer available to the benchmark. This decouples the app and the model. We just need to
         # create the directory here to pass the build
-        mkdir -p extension/apple/Benchmark/Models
+        mkdir -p extension/benchmark/apple/Benchmark/Models
         ${CONDA_RUN} --no-capture-output \
           build/build_apple_llm_demo.sh ${ARTIFACTS_DIR_NAME}
 

.github/workflows/apple.yml

@@ -18,6 +18,7 @@ on:
       - build/test_ios_ci.sh
       - examples/demo-apps/apple_ios/**
       - extension/apple/**
+      - extension/benchmark/apple/**
       - extension/module/**
   workflow_dispatch:
 
@@ -272,14 +273,14 @@ jobs:
         PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \
           build/build_apple_frameworks.sh --coreml --custom --mps --optimized --portable --quantized --xnnpack
 
-        mkdir -p extension/apple/Benchmark/Frameworks
+        mkdir -p extension/benchmark/apple/Benchmark/Frameworks
         for FRAMEWORK in "${FRAMEWORKS[@]}"; do (
-          cp -r "cmake-out/${FRAMEWORK}.xcframework" extension/apple/Benchmark/Frameworks/
+          cp -r "cmake-out/${FRAMEWORK}.xcframework" extension/benchmark/apple/Benchmark/Frameworks/
         ) done
         echo "::endgroup::"
 
         echo "::group::Build ExecuTorch benchmark app"
-        mkdir -p extension/apple/Benchmark/Models
+        mkdir -p extension/benchmark/apple/Benchmark/Models
         ${CONDA_RUN} --no-capture-output \
           build/build_apple_llm_demo.sh ${ARTIFACTS_DIR_NAME}
         echo "::endgroup::"

.github/workflows/lint.yml

@@ -66,7 +66,7 @@ jobs:
         FILES_NEEDS_FORMAT=$(/opt/google-java-format -n extension/android/src/main/java/org/pytorch/executorch/*.java \
           examples/demo-apps/android/ExecuTorchDemo/app/src/main/java/com/example/executorchdemo/*.java \
           examples/demo-apps/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/*.java \
-          extension/android/benchmark/app/src/main/java/org/pytorch/minibench/*.java)
+          extension/benchmark/android/benchmark/app/src/main/java/org/pytorch/minibench/*.java)
         if [ -n "$FILES_NEEDS_FORMAT" ]; then
           echo "Warning: The following files need formatting. Please use google-java-format."
           echo "Use a binary from https://github.com/google/google-java-format/releases/"

.github/workflows/upload-android-test-specs.yml

@@ -4,13 +4,13 @@ on:
   pull_request:
     paths:
       - .github/workflows/upload-android-test-specs.yml
-      - extension/android/benchmark/android-llm-device-farm-test-spec.yml
+      - extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml
   push:
     branches:
       - main
     paths:
       - .github/workflows/upload-android-test-specs.yml
-      - extension/android/benchmark/android-llm-device-farm-test-spec.yml
+      - extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml
 
 concurrency:
   # NB: This concurency group needs to be different than the one used in android-perf, otherwise
@@ -32,7 +32,7 @@ jobs:
             ${{ github.repository }}/${{ github.run_id }}/artifacts
           retention-days: 1
           if-no-files-found: error
-          path: extension/android/benchmark/android-llm-device-farm-test-spec.yml
+          path: extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml
 
   validate-android-test-spec:
     needs: upload-android-test-spec-for-validation
@@ -77,7 +77,7 @@ jobs:
 
       - name: Upload the spec to S3 ossci-android bucket
         shell: bash
-        working-directory: extension/android/benchmark/
+        working-directory: extension/benchmark/android/benchmark/
         env:
           SPEC_FILE: android-llm-device-farm-test-spec.yml
         run: |

.github/workflows/upload-apple-test-specs.yml

@@ -4,13 +4,13 @@ on:
   pull_request:
     paths:
       - .github/workflows/upload-apple-test-specs.yml
-      - examples/demo-apps/apple_ios/default-ios-device-farm-appium-test-spec.yml
+      - extension/benchmark/apple/Benchmark/default-ios-device-farm-appium-test-spec.yml
   push:
     branches:
       - main
     paths:
       - .github/workflows/upload-apple-test-specs.yml
-      - examples/demo-apps/apple_ios/default-ios-device-farm-appium-test-spec.yml
+      - extension/benchmark/apple/Benchmark/default-ios-device-farm-appium-test-spec.yml
 
 concurrency:
   # NB: This concurency group needs to be different than the one used in apple-perf, otherwise
@@ -32,7 +32,7 @@ jobs:
             ${{ github.repository }}/${{ github.run_id }}/artifacts
           retention-days: 1
           if-no-files-found: error
-          path: examples/demo-apps/apple_ios/default-ios-device-farm-appium-test-spec.yml
+          path: extension/benchmark/apple/Benchmark/default-ios-device-farm-appium-test-spec.yml
 
   validate-apple-test-spec:
     needs: upload-apple-test-spec-for-validation
@@ -78,7 +78,7 @@ jobs:
 
       - name: Upload the spec to S3 ossci-ios bucket
         shell: bash
-        working-directory: examples/demo-apps/apple_ios
+        working-directory: extension/benchmark/apple/Benchmark/
         env:
           SPEC_FILE: default-ios-device-farm-appium-test-spec.yml
         run: |

build/build_android_llm_demo.sh

@@ -108,9 +108,9 @@ build_android_demo_apps() {
   ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew build assembleAndroidTest
   popd
 
-  mkdir -p extension/android/benchmark/app/libs
-  cp ${BUILD_AAR_DIR}/executorch.aar extension/android/benchmark/app/libs
-  pushd extension/android/benchmark
+  mkdir -p extension/benchmark/android/benchmark/app/libs
+  cp ${BUILD_AAR_DIR}/executorch.aar extension/benchmark/android/benchmark/app/libs
+  pushd extension/benchmark/android/benchmark
   ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew build assembleAndroidTest
   popd
 }
@@ -135,8 +135,8 @@ collect_artifacts_to_be_uploaded() {
   # Collect MiniBench APK
   MINIBENCH_APP_DIR="${ARTIFACTS_DIR_NAME}/minibench"
   mkdir -p "${MINIBENCH_APP_DIR}"
-  cp extension/android/benchmark/app/build/outputs/apk/debug/*.apk "${MINIBENCH_APP_DIR}"
-  cp extension/android/benchmark/app/build/outputs/apk/androidTest/debug/*.apk "${MINIBENCH_APP_DIR}"
+  cp extension/benchmark/android/benchmark/app/build/outputs/apk/debug/*.apk "${MINIBENCH_APP_DIR}"
+  cp extension/benchmark/android/benchmark/app/build/outputs/apk/androidTest/debug/*.apk "${MINIBENCH_APP_DIR}"
 }
 
 BUILD_AAR_DIR="$(mktemp -d)"

build/build_apple_llm_demo.sh

@@ -8,7 +8,7 @@
 set -euo pipefail
 
 ARTIFACTS_DIR_NAME="$1"
-APP_PATH="extension/apple/Benchmark/Benchmark"
+APP_PATH="extension/benchmark/apple/Benchmark/Benchmark"
 
 xcodebuild build-for-testing \
   -project "${APP_PATH}.xcodeproj" \

extension/android/benchmark/.gitignore renamed to extension/benchmark/android/benchmark/.gitignore

@@ -0,0 +1,45 @@
+---
+AccessModifierOffset: -1
+AlignEscapedNewlinesLeft: true
+AlignTrailingComments: false
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: false
+BinPackParameters: false
+BreakBeforeBinaryOperators: false
+BreakBeforeBraces: Attach
+BreakConstructorInitializersBeforeComma: false
+ColumnLimit:     80
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+ExperimentalAutoDetectBinPacking: true
+IndentCaseLabels: false
+IndentFunctionDeclarationAfterType: false
+IndentWidth:     2
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakBeforeFirstCallParameter: 10
+PenaltyBreakComment: 60
+PenaltyBreakFirstLessLess: 20
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Right
+SpaceAfterControlStatementKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles: false
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+Standard:        Cpp11
+TabWidth:        8
+UseTab:          Never
+...

extension/android/benchmark/README.md renamed to extension/benchmark/android/benchmark/README.md

@@ -66,10 +66,10 @@
 		032A73FD2CAFBB7800932D36 /* tiktoken.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = tiktoken.h; sourceTree = "<group>"; };
 		032A73FE2CAFBB7800932D36 /* tiktoken.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = tiktoken.cpp; sourceTree = "<group>"; };
 		032A73FF2CAFBB7800932D36 /* tokenizer.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = tokenizer.h; sourceTree = "<group>"; };
-		032A74212CAFC1B300932D36 /* runner.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = runner.h; path = ../../../examples/models/llama2/runner/runner.h; sourceTree = SOURCE_ROOT; };
-		032A74222CAFC1B300932D36 /* runner.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = runner.cpp; path = ../../../examples/models/llama2/runner/runner.cpp; sourceTree = SOURCE_ROOT; };
-		032A74242CAFC34800932D36 /* llama_tiktoken.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = llama_tiktoken.h; path = ../../../examples/models/llama2/tokenizer/llama_tiktoken.h; sourceTree = SOURCE_ROOT; };
-		032A74252CAFC34800932D36 /* llama_tiktoken.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = llama_tiktoken.cpp; path = ../../../examples/models/llama2/tokenizer/llama_tiktoken.cpp; sourceTree = SOURCE_ROOT; };
+		032A74212CAFC1B300932D36 /* runner.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = runner.h; path = ../../../../examples/models/llama2/runner/runner.h; sourceTree = SOURCE_ROOT; };
+		032A74222CAFC1B300932D36 /* runner.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = runner.cpp; path = ../../../../examples/models/llama2/runner/runner.cpp; sourceTree = SOURCE_ROOT; };
+		032A74242CAFC34800932D36 /* llama_tiktoken.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = llama_tiktoken.h; path = ../../../../examples/models/llama2/tokenizer/llama_tiktoken.h; sourceTree = SOURCE_ROOT; };
+		032A74252CAFC34800932D36 /* llama_tiktoken.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = llama_tiktoken.cpp; path = ../../../../examples/models/llama2/tokenizer/llama_tiktoken.cpp; sourceTree = SOURCE_ROOT; };
 		037C96A02C8A570B00B3DF38 /* Tests.xctestplan */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = Tests.xctestplan; sourceTree = "<group>"; };
 		03B0118B2CAC567900054791 /* DynamicTestCase.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = DynamicTestCase.h; sourceTree = "<group>"; };
 		03B0118C2CAC567900054791 /* DynamicTestCase.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = DynamicTestCase.m; sourceTree = "<group>"; };
@@ -158,7 +158,7 @@
 				032A73DF2CAFBB7800932D36 /* util.h */,
 			);
 			name = runner;
-			path = ../../llm/runner;
+			path = ../../../llm/runner;
 			sourceTree = SOURCE_ROOT;
 		};
 		032A73E92CAFBB7800932D36 /* sampler */ = {
@@ -168,7 +168,7 @@
 				032A73E62CAFBB7800932D36 /* sampler.cpp */,
 			);
 			name = sampler;
-			path = ../../llm/sampler;
+			path = ../../../llm/sampler;
 			sourceTree = SOURCE_ROOT;
 		};
 		032A74022CAFBB7800932D36 /* tokenizer */ = {
@@ -184,7 +184,7 @@
 				032A73FF2CAFBB7800932D36 /* tokenizer.h */,
 			);
 			name = tokenizer;
-			path = ../../llm/tokenizer;
+			path = ../../../llm/tokenizer;
 			sourceTree = SOURCE_ROOT;
 		};
 		03B0118D2CAC567900054791 /* TestUtils */ = {
@@ -364,7 +364,7 @@
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 			shellPath = /bin/sh;
-			shellScript = "set -e\n\nif ! command -v cmake &> /dev/null\nthen\n    echo \"Cmake not found, please install Cmake. \\n1. Download Cmake.app from https://cmake.org/download with version > 3.19. \\n2. Install it to Applications/ folder and run sudo /Applications/CMake.app/Contents/bin/cmake-gui --install to install CMake commandline tools.\"\n    exit 1\nfi\n\nCMAKE_DIR=\"$TEMP_DIR/cmake\"\nrm -rf \"$CMAKE_DIR\"\n\nPLATFORM=\"SIMULATORARM64\"\nDEPLOYMENT_TARGET=\"17.0\"\n\nif [[ \"$PLATFORM_NAME\" == *\"iphoneos\"* ]]; then\n  PLATFORM=\"OS64\"\nelif [[ \"$PLATFORM_NAME\" == *\"macos\"* ]]; then\n  PLATFORM=\"MAC_ARM64\"\n  DEPLOYMENT_TARGET=\"10.15\"\nfi\n\ncmake_build() {\n    local src_dir=$1\n    local target=$2\n    shift 2\n    local extra_args=(\"$@\")\n    local build_dir=\"$CMAKE_DIR/build/$(basename \"$src_dir\")\"\n\n    mkdir -p \"$build_dir\" && cd \"$build_dir\"\n\n    if [[ \"$PLATFORM\" == \"MAC_ARM64\" ]]; then\n        extra_args+=(-DCMAKE_INSTALL_BUNDLEDIR=\"${CMAKE_DIR}/bin\")\n        extra_args+=(-DCMAKE_MACOSX_BUNDLE=OFF)\n    fi\n    cmake -G Xcode \\\n          -DCMAKE_BUILD_TYPE=\"Release\" \\\n          -DCMAKE_CXX_STANDARD=17 \\\n          -DCMAKE_TOOLCHAIN_FILE=\"$SRCROOT/../../../third-party/ios-cmake/ios.toolchain.cmake\" \\\n          -DCMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LANGUAGE_STANDARD=\"c++17\" \\\n          -DCMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LIBRARY=\"libc++\" \\\n          -DPLATFORM=\"$PLATFORM\" \\\n          -DDEPLOYMENT_TARGET=\"$DEPLOYMENT_TARGET\" \\\n          -DCMAKE_INSTALL_PREFIX=\"$CMAKE_DIR\" \\\n          \"${extra_args[@]}\" \\\n          \"$src_dir\"\n    cmake --build . --config \"Release\" --target \"$target\"\n    if [[ \"$target\" == \"install\" ]]; then\n        cmake --install . --prefix \"$CMAKE_DIR\"\n    fi\n}\n\ncmake_build \"$SRCROOT/../../llm/third-party/abseil-cpp\" \"install\" \\\n    -DABSL_PROPAGATE_CXX_STD=ON\n\ncmake_build \"$SRCROOT/../../llm/third-party/re2\" \"install\"\n\ncmake_build \"$SRCROOT/../../llm/third-party/sentencepiece\" \"sentencepiece-static\" \\\n    -DSPM_ENABLE_SHARED=OFF\n\necho \"$(find $CMAKE_DIR/lib -name \"*.a\" | sed -E 's|^.*/lib([^/]+)\\.a|-l\\1|g' | tr '\\n' ' ')\" > \"$CMAKE_DIR/linker_flags\"\n";
+			shellScript = "set -e\n\nif ! command -v cmake &> /dev/null\nthen\n    echo \"Cmake not found, please install Cmake. \\n1. Download Cmake.app from https://cmake.org/download with version > 3.19. \\n2. Install it to Applications/ folder and run sudo /Applications/CMake.app/Contents/bin/cmake-gui --install to install CMake commandline tools.\"\n    exit 1\nfi\n\nCMAKE_DIR=\"$TEMP_DIR/cmake\"\nrm -rf \"$CMAKE_DIR\"\n\nPLATFORM=\"SIMULATORARM64\"\nDEPLOYMENT_TARGET=\"17.0\"\n\nif [[ \"$PLATFORM_NAME\" == *\"iphoneos\"* ]]; then\n  PLATFORM=\"OS64\"\nelif [[ \"$PLATFORM_NAME\" == *\"macos\"* ]]; then\n  PLATFORM=\"MAC_ARM64\"\n  DEPLOYMENT_TARGET=\"10.15\"\nfi\n\ncmake_build() {\n    local src_dir=$1\n    local target=$2\n    shift 2\n    local extra_args=(\"$@\")\n    local build_dir=\"$CMAKE_DIR/build/$(basename \"$src_dir\")\"\n\n    mkdir -p \"$build_dir\" && cd \"$build_dir\"\n\n    if [[ \"$PLATFORM\" == \"MAC_ARM64\" ]]; then\n        extra_args+=(-DCMAKE_INSTALL_BUNDLEDIR=\"${CMAKE_DIR}/bin\")\n        extra_args+=(-DCMAKE_MACOSX_BUNDLE=OFF)\n    fi\n    cmake -G Xcode \\\n          -DCMAKE_BUILD_TYPE=\"Release\" \\\n          -DCMAKE_CXX_STANDARD=17 \\\n          -DCMAKE_TOOLCHAIN_FILE=\"$SRCROOT/../../../../third-party/ios-cmake/ios.toolchain.cmake\" \\\n          -DCMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LANGUAGE_STANDARD=\"c++17\" \\\n          -DCMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LIBRARY=\"libc++\" \\\n          -DPLATFORM=\"$PLATFORM\" \\\n          -DDEPLOYMENT_TARGET=\"$DEPLOYMENT_TARGET\" \\\n          -DCMAKE_INSTALL_PREFIX=\"$CMAKE_DIR\" \\\n          \"${extra_args[@]}\" \\\n          \"$src_dir\"\n    cmake --build . --config \"Release\" --target \"$target\"\n    if [[ \"$target\" == \"install\" ]]; then\n        cmake --install . --prefix \"$CMAKE_DIR\"\n    fi\n}\n\ncmake_build \"$SRCROOT/../../../llm/third-party/abseil-cpp\" \"install\" \\\n    -DABSL_PROPAGATE_CXX_STD=ON\n\ncmake_build \"$SRCROOT/../../../llm/third-party/re2\" \"install\"\n\ncmake_build \"$SRCROOT/../../../llm/third-party/sentencepiece\" \"sentencepiece-static\" \\\n    -DSPM_ENABLE_SHARED=OFF\n\necho \"$(find $CMAKE_DIR/lib -name \"*.a\" | sed -E 's|^.*/lib([^/]+)\\.a|-l\\1|g' | tr '\\n' ' ')\" > \"$CMAKE_DIR/linker_flags\"\n";
 		};
 /* End PBXShellScriptBuildPhase section */