Move test spec file (#5218)

Summary:
Use the new minibench for both LLM and generic model benchmarking.

Pull Request resolved: #5218

Reviewed By: guangy10

Differential Revision: D62478090

Pulled By: kirklandsign

fbshipit-source-id: 62a1b7a5cc0212031453e4962122db51c34e0559

Author: kirklandsign

.github/workflows/android-perf.yml

@@ -292,8 +292,8 @@ jobs:
       # Unlike models there are limited numbers of build flavor for apps, and the model controls whether it should build with bpe/tiktoken tokenizer.
       # It's okay to build all possible apps with all possible flavors in job "build-llm-demo". However, in this job, once a model is given, there is only
       # one app+flavor that could load and run the model.
-      android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo/app-debug.apk
-      android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo/app-debug-androidTest.apk
+      android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/minibench/app-debug.apk
+      android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/minibench/app-debug-androidTest.apk
       # NB: Need to set the default spec here so that it works for periodic too
       test-spec: ${{ inputs.test_spec || 'https://ossci-android.s3.amazonaws.com/executorch/android-llm-device-farm-test-spec.yml' }}
       # Uploaded to S3 from the previous job

.github/workflows/upload-android-test-specs.yml

@@ -4,13 +4,13 @@ on:
   pull_request:
     paths:
       - .github/workflows/upload-android-test-specs.yml
-      - examples/demo-apps/android/LlamaDemo/android-llm-device-farm-test-spec.yml
+      - extension/android/benchmark/android-llm-device-farm-test-spec.yml
   push:
     branches:
       - main
     paths:
       - .github/workflows/upload-android-test-specs.yml
-      - examples/demo-apps/android/LlamaDemo/android-llm-device-farm-test-spec.yml
+      - extension/android/benchmark/android-llm-device-farm-test-spec.yml
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
@@ -30,7 +30,7 @@ jobs:
             ${{ github.repository }}/${{ github.run_id }}/artifact
           retention-days: 1
           if-no-files-found: error
-          path: examples/demo-apps/android/LlamaDemo/android-llm-device-farm-test-spec.yml
+          path: extension/android/benchmark/android-llm-device-farm-test-spec.yml
 
   validate-android-test-spec:
     needs: upload-android-test-spec-for-validation
@@ -75,7 +75,7 @@ jobs:
 
       - name: Upload the spec to S3 ossci-android bucket
         shell: bash
-        working-directory: examples/demo-apps/android/LlamaDemo/
+        working-directory: extension/android/benchmark/
         env:
           SPEC_FILE: android-llm-device-farm-test-spec.yml
         run: |

build/build_android_llm_demo.sh

@@ -111,7 +111,7 @@ build_android_demo_apps() {
   mkdir -p extension/android/benchmark/app/libs
   cp ${BUILD_AAR_DIR}/executorch.aar extension/android/benchmark/app/libs
   pushd extension/android/benchmark
-  ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew build
+  ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew build assembleAndroidTest
   popd
 }
 
@@ -136,6 +136,7 @@ collect_artifacts_to_be_uploaded() {
   MINIBENCH_APP_DIR="${ARTIFACTS_DIR_NAME}/minibench"
   mkdir -p "${MINIBENCH_APP_DIR}"
   cp extension/android/benchmark/app/build/outputs/apk/debug/*.apk "${MINIBENCH_APP_DIR}"
+  cp extension/android/benchmark/app/build/outputs/apk/androidTest/debug/*.apk "${MINIBENCH_APP_DIR}"
 }
 
 BUILD_AAR_DIR="$(mktemp -d)"

examples/demo-apps/android/LlamaDemo/android-llm-device-farm-test-spec.yml renamed to extension/android/benchmark/android-llm-device-farm-test-spec.yml

@@ -69,7 +69,7 @@ phases:
       # Run the new generic benchmark activity https://developer.android.com/tools/adb#am
       - echo "Run LLM benchmark"
       - |
-        adb -s $DEVICEFARM_DEVICE_UDID shell am start -W -n com.example.executorchllamademo/.LlmBenchmarkRunner \
+        adb -s $DEVICEFARM_DEVICE_UDID shell am start -W -n org.pytorch.minibench/.LlmBenchmarkActivity \
         --es "model_dir" "/data/local/tmp/llama" \
         --es "tokenizer_path" "/data/local/tmp/llama/tokenizer.bin"
 
@@ -82,12 +82,12 @@ phases:
         MAX_ATTEMPT=10
         while [ -z "${BENCHMARK_RESULTS}" ] && [ $ATTEMPT -lt $MAX_ATTEMPT ]; do
           echo "Waiting for benchmark results..."
-          BENCHMARK_RESULTS=$(adb -s $DEVICEFARM_DEVICE_UDID shell run-as com.example.executorchllamademo cat files/benchmark_results.json)
+          BENCHMARK_RESULTS=$(adb -s $DEVICEFARM_DEVICE_UDID shell run-as org.pytorch.minibench cat files/benchmark_results.json)
           sleep 30
           ((ATTEMPT++))
         done
 
-        adb -s $DEVICEFARM_DEVICE_UDID shell run-as com.example.executorchllamademo ls -la files/
+        adb -s $DEVICEFARM_DEVICE_UDID shell run-as org.pytorch.minibench ls -la files/
         # Trying to pull the file using adb ends up with permission error, but this works too, so why not
         echo "${BENCHMARK_RESULTS}" > $DEVICEFARM_LOG_DIR/benchmark_results.json