Merge branch 'main' into oss_model_execution

Author: neuropilot-captain

.ci/docker/ci_commit_pins/pytorch.txt

@@ -1 +1 @@
-c42ac54d9e817bf0a0366eb78e6c8beba4d5eff5
+00e3eea170ce5db8ea9c62ce5e48f13886cd6d20

.ci/docker/ci_commit_pins/torchao.txt

@@ -0,0 +1 @@
+0916b5b29b092afcbf2b898caae49abe80662bac

.ci/scripts/build-qnn-sdk.sh

@@ -6,11 +6,12 @@
 # LICENSE file in the root directory of this source tree.
 
 set -eux
+set -o xtrace
 
 build_qnn_backend() {
   echo "Start building qnn backend."
   export ANDROID_NDK_ROOT=/opt/ndk
-  export QNN_SDK_ROOT=/tmp/qnn/2.23.0.240531
+  export QNN_SDK_ROOT=/tmp/qnn/2.25.0.240728
   export EXECUTORCH_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
 
   bash backends/qualcomm/scripts/build.sh --skip_aarch64 --job_number 2 --release
@@ -28,6 +29,7 @@ set_up_aot() {
       -DQNN_SDK_ROOT=${QNN_SDK_ROOT} \
       -DEXECUTORCH_BUILD_SDK=ON \
       -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
+      -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
       -DEXECUTORCH_ENABLE_EVENT_TRACER=ON \
       -DPYTHON_EXECUTABLE=python3 \
       -DEXECUTORCH_SEPARATE_FLATCC_HOST_PROJECT=OFF

.ci/scripts/build_llama_android.sh

@@ -22,8 +22,9 @@ install_executorch_and_backend_lib() {
     -DANDROID_PLATFORM=android-23 \
     -DCMAKE_INSTALL_PREFIX=cmake-android-out \
     -DCMAKE_BUILD_TYPE=Release \
-    -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
     -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
+    -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
+    -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
     -DEXECUTORCH_BUILD_XNNPACK=ON \
     -DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON \
     -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \

.ci/scripts/setup-qnn-deps.sh

@@ -7,14 +7,18 @@
 
 set -ex
 
+verify_pkg_installed() {
+  echo $(dpkg-query -W --showformat='${Status}\n' $1|grep "install ok installed")
+}
+
 install_qnn() {
   echo "Start installing qnn."
   QNN_INSTALLATION_DIR=/tmp/qnn
   mkdir -p "${QNN_INSTALLATION_DIR}"
 
-  curl -Lo /tmp/v2.23.0.24.06.24.zip "https://softwarecenter.qualcomm.com/api/download/software/qualcomm_neural_processing_sdk/v2.23.0.24.06.24.zip"
+  curl -Lo /tmp/v2.25.0.24.07.28.zip "https://softwarecenter.qualcomm.com/api/download/software/qualcomm_neural_processing_sdk/v2.25.0.240728.zip"
   echo "Finishing downloading qnn sdk."
-  unzip -qo /tmp/v2.23.0.24.06.24.zip -d /tmp
+  unzip -qo /tmp/v2.25.0.24.07.28.zip -d /tmp
   echo "Finishing unzip qnn sdk."
 
 
@@ -26,4 +30,22 @@ install_qnn() {
   ls -lah "${QNN_INSTALLATION_DIR}"
 }
 
+setup_libc++() {
+  sudo apt-get update
+  pkgs_to_check=('libc++-dev')
+  j=0
+  while [ $j -lt ${#pkgs_to_check[*]} ]; do
+    install_status=$(verify_pkg_installed ${pkgs_to_check[$j]})
+    if [ "$install_status" == "" ]; then
+      sudo apt-get install -y ${pkgs_to_check[$j]}
+      if [[ $? -ne 0 ]]; then
+        echo "ERROR: Failed to install required packages for libc++"
+        exit 1
+      fi
+    fi
+    j=$(( $j +1));
+  done
+}
+
+setup_libc++
 install_qnn

.ci/scripts/test_llama.sh

@@ -75,7 +75,7 @@ echo "COREML option ${COREML}"
 if [[ "${MODE}" =~ .*qnn.* ]]; then
   QNN=ON
   export EXECUTORCH_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
-  export QNN_SDK_ROOT=/tmp/qnn/2.23.0.240531
+  export QNN_SDK_ROOT=/tmp/qnn/2.25.0.240728
   export LD_LIBRARY_PATH="${QNN_SDK_ROOT}/lib/x86_64-linux-clang"
   export PYTHONPATH=".."
   cp schema/program.fbs exir/_serialize/program.fbs
@@ -107,8 +107,9 @@ cmake_install_executorch_libraries() {
     retry cmake \
         -DCMAKE_INSTALL_PREFIX=cmake-out \
         -DCMAKE_BUILD_TYPE=Debug \
-        -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
         -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
+        -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
+        -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
         -DEXECUTORCH_BUILD_KERNELS_CUSTOM="$CUSTOM" \
         -DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON \
         -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \

.ci/scripts/test_llava.sh

@@ -9,47 +9,97 @@ set -exu
 # shellcheck source=/dev/null
 
 BUILD_TYPE=${1:-Debug}
+TARGET_OS=${2:-Native}
+BUILD_DIR=${3:-cmake-out}
 
-echo "Building with BUILD_TYPE: $BUILD_TYPE"
+echo "Building with BUILD_TYPE: $BUILD_TYPE, TARGET_OS: $TARGET_OS, BUILD_DIR: $BUILD_DIR"
 
 if [[ -z "${PYTHON_EXECUTABLE:-}" ]]; then
-  PYTHON_EXECUTABLE=python3
+    PYTHON_EXECUTABLE=python3
 fi
 
+TARGET_OS_lower="$(echo "${TARGET_OS}" | awk '{print tolower($0)}')"
+if [[ "${TARGET_OS_lower}" == "android" ]]; then
+    if [[ -z "${ANDROID_NDK}" ]]; then
+        echo "Set ANDROID_NDK environment variable to build for Android."
+        exit 1
+    fi
+fi
+
+# Number of processes for a parallel build
+NPROC=8
+if hash nproc &> /dev/null; then NPROC=$(nproc); fi
+
+EXECUTORCH_COMMON_CMAKE_ARGS="                      \
+        -DCMAKE_INSTALL_PREFIX=${BUILD_DIR}         \
+        -DCMAKE_BUILD_TYPE=${BUILD_TYPE}            \
+        -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON      \
+        -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
+        -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON      \
+        -DEXECUTORCH_BUILD_KERNELS_CUSTOM=ON        \
+        -DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON     \
+        -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON     \
+        -DEXECUTORCH_BUILD_XNNPACK=ON               \
+        -DEXECUTORCH_DO_NOT_USE_CXX11_ABI=ON        \
+        -DEXECUTORCH_XNNPACK_SHARED_WORKSPACE=ON"
+
 cmake_install_executorch_libraries() {
-    cmake                                               \
-        -DCMAKE_INSTALL_PREFIX=cmake-out                \
-        -DCMAKE_BUILD_TYPE=${BUILD_TYPE}                \
-        -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON          \
-        -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON     \
-        -DEXECUTORCH_BUILD_KERNELS_CUSTOM=ON            \
-        -DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON         \
-        -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON         \
-        -DEXECUTORCH_BUILD_XNNPACK=ON                   \
-        -DEXECUTORCH_DO_NOT_USE_CXX11_ABI=ON            \
-        -DEXECUTORCH_XNNPACK_SHARED_WORKSPACE=ON        \
-        -Bcmake-out .
-
-
-    cmake --build cmake-out -j9 --target install --config ${BUILD_TYPE}
+    cmake                               \
+        ${EXECUTORCH_COMMON_CMAKE_ARGS} \
+        -B${BUILD_DIR} .
+
+    cmake --build ${BUILD_DIR} -j${NPROC} --target install --config ${BUILD_TYPE}
+}
+
+cmake_install_executorch_libraries_for_android() {
+    cmake                                                                       \
+        -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \
+        -DANDROID_ABI=arm64-v8a                                                 \
+        -DANDROID_PLATFORM=android-23                                           \
+        ${EXECUTORCH_COMMON_CMAKE_ARGS}                                         \
+        -B${BUILD_DIR} .
+
+    cmake --build ${BUILD_DIR} -j${NPROC} --target install --config ${BUILD_TYPE}
 }
 
+
+LLAVA_COMMON_CMAKE_ARGS="                        \
+        -DPYTHON_EXECUTABLE="$PYTHON_EXECUTABLE" \
+        -DCMAKE_INSTALL_PREFIX=${BUILD_DIR}      \
+        -DCMAKE_BUILD_TYPE=${BUILD_TYPE}         \
+        -DEXECUTORCH_BUILD_KERNELS_CUSTOM=ON     \
+        -DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON  \
+        -DEXECUTORCH_BUILD_XNNPACK=ON"
+
 cmake_build_llava_runner() {
     dir=examples/models/llava
     python_lib=$($PYTHON_EXECUTABLE -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')
 
-    cmake                                       \
-        -DCMAKE_INSTALL_PREFIX=cmake-out        \
-        -DCMAKE_BUILD_TYPE=${BUILD_TYPE}         \
-        -DEXECUTORCH_BUILD_KERNELS_CUSTOM=ON    \
-        -DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON \
-        -DEXECUTORCH_BUILD_XNNPACK=ON           \
-        -DCMAKE_PREFIX_PATH="$python_lib"       \
-        -Bcmake-out/${dir}                      \
+    cmake                                 \
+        ${LLAVA_COMMON_CMAKE_ARGS}        \
+        -DCMAKE_PREFIX_PATH="$python_lib" \
+        -B${BUILD_DIR}/${dir}             \
         ${dir}
 
+    cmake --build ${BUILD_DIR}/${dir} -j${NPROC} --config ${BUILD_TYPE}
+}
+
 
-    cmake --build cmake-out/${dir} -j9 --config ${BUILD_TYPE}
+cmake_build_llava_runner_for_android() {
+    dir=examples/models/llava
+    python_lib=$($PYTHON_EXECUTABLE -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')
+
+    cmake                                                                       \
+        -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \
+        -DANDROID_ABI=arm64-v8a                                                 \
+        -DANDROID_PLATFORM=android-23                                           \
+        ${LLAVA_COMMON_CMAKE_ARGS}                                              \
+        -DCMAKE_PREFIX_PATH="$python_lib"                                       \
+        -DLLAVA_RUNNER_NO_TORCH_DUMMY_IMAGE=ON                                  \
+        -B${BUILD_DIR}/${dir}                                                   \
+        ${dir}
+
+    cmake --build ${BUILD_DIR}/${dir} -j${NPROC} --config ${BUILD_TYPE}
 }
 
 # only export the one without custom op for now since it's
@@ -61,7 +111,7 @@ export_llava() {
 # Download a new image with different size, to test if the model can handle different image sizes
 prepare_image_tensor() {
     echo "Downloading image"
-    curl -o basketball.jpg https://upload.wikimedia.org/wikipedia/commons/7/73/Chicago_Bulls_and_New_Jersey_Nets%2C_March_28%2C_1991.jpg 
+    curl -o basketball.jpg https://upload.wikimedia.org/wikipedia/commons/7/73/Chicago_Bulls_and_New_Jersey_Nets%2C_March_28%2C_1991.jpg
     $PYTHON_EXECUTABLE -m executorch.examples.models.llava.image_util --image-path basketball.jpg --output-path image.pt
 }
 
@@ -80,18 +130,29 @@ run_and_verify() {
         echo "tokenizer.bin is missing."
         exit 1
     fi
-    RUNTIME_ARGS="--model_path=llava.pte \
-     --tokenizer_path=tokenizer.bin \
-     --image_path=image.pt \
-     --prompt=ASSISTANT: \
-     --temperature=0 \
-     --seq_len=650"
-    cmake-out/examples/models/llava/llava_main ${RUNTIME_ARGS} > result.txt
+
+
+
+    RUNTIME_ARGS="--model_path=llava.pte    \
+        --tokenizer_path=tokenizer.bin      \
+        --image_path=image.pt               \
+        --prompt=ASSISTANT:                 \
+        --temperature=0                     \
+        --seq_len=650"
+
+    if [[ "${TARGET_OS_lower}" == "android" ]]; then
+        echo "Transfer relevant files to the phone via ADB and run llava_main with following args,"
+        echo "$ llava_main ${RUNTIME_ARGS} "
+        exit 0;
+    fi
+
+    ${BUILD_DIR}/examples/models/llava/llava_main ${RUNTIME_ARGS} > result.txt
+
     # verify result.txt
     RESULT=$(cat result.txt)
     # set the expected prefix to be the same as prompt because there's a bug in sdpa_with_kv_cache that causes <unk> tokens.
     if [[ "$(uname)" == "Darwin" ]]; then
-        EXPECTED_PREFIX="ASSISTANT: image captures a basketball game in progress on a basketball court. There are several players on the court, with one player in the foreground holding a basketball, and"
+        EXPECTED_PREFIX="ASSISTANT: image captures a basketball game in progress, with several players on the court. One of the players is dribbling the ball, while the others are in various"
     else
         # set the expected prefix to be the same as prompt because there's a bug in sdpa_with_kv_cache that causes <unk> tokens.
         EXPECTED_PREFIX="ASSISTANT:"
@@ -109,8 +170,20 @@ run_and_verify() {
     fi
 }
 
-cmake_install_executorch_libraries
-cmake_build_llava_runner
+# Step1. Build stuff
+if [[ "${TARGET_OS_lower}" == "android" ]]; then
+    cmake_install_executorch_libraries_for_android
+    cmake_build_llava_runner_for_android
+elif [[ "${TARGET_OS_lower}" == "native" ]]; then
+    cmake_install_executorch_libraries
+    cmake_build_llava_runner
+else
+    echo "Invalid TARGET_OS ($2): ${TARGET_OS}"
+fi
+
+# Step2. Generate the PTE
 export_llava
+
+# Step3. Run
 prepare_image_tensor
 run_and_verify

.ci/scripts/test.sh renamed to .ci/scripts/test_model.sh

@@ -156,9 +156,40 @@ test_model_with_qnn() {
   export PYTHONPATH=$EXECUTORCH_ROOT/..
 
   if [[ "${MODEL_NAME}" == "dl3" ]]; then
-    "${PYTHON_EXECUTABLE}" -m examples.qualcomm.scripts.deeplab_v3 -b ${CMAKE_OUTPUT_DIR} -m SM8550 --compile_only --download
-    EXPORTED_MODEL=./deeplab_v3/dlv3_qnn.pte
+    EXPORT_SCRIPT=deeplab_v3
+    EXPORTED_MODEL_NAME=dlv3_qnn.pte
+  elif [[ "${MODEL_NAME}" == "mv3" ]]; then
+    EXPORT_SCRIPT=mobilenet_v3
+    EXPORTED_MODEL_NAME=mv3_qnn.pte
+  elif [[ "${MODEL_NAME}" == "mv2" ]]; then
+    EXPORT_SCRIPT=mobilenet_v2
+    EXPORTED_MODEL_NAME=mv2_qnn.pte
+  elif [[ "${MODEL_NAME}" == "ic4" ]]; then
+    EXPORT_SCRIPT=inception_v4
+    EXPORTED_MODEL_NAME=ic4_qnn.pte
+  elif [[ "${MODEL_NAME}" == "ic3" ]]; then
+    EXPORT_SCRIPT=inception_v3
+    EXPORTED_MODEL_NAME=ic3_qnn.pte
+  elif [[ "${MODEL_NAME}" == "vit" ]]; then
+    EXPORT_SCRIPT=torchvision_vit
+    EXPORTED_MODEL_NAME=vit_qnn.pte
   fi
+
+  # Use SM8450 for S22, SM8550 for S23, and SM8560 for S24
+  QNN_CHIPSET=SM8450
+
+  "${PYTHON_EXECUTABLE}" -m examples.qualcomm.scripts.${EXPORT_SCRIPT} -b ${CMAKE_OUTPUT_DIR} -m ${QNN_CHIPSET} --compile_only
+  EXPORTED_MODEL=./${EXPORT_SCRIPT}/${EXPORTED_MODEL_NAME}
+}
+
+test_model_with_coreml() {
+  if [[ "${BUILD_TOOL}" == "buck2" ]]; then
+    echo "coreml doesn't support buck2."
+    exit 1
+  fi
+
+  "${PYTHON_EXECUTABLE}" -m examples.apple.coreml.scripts.export --model_name="${MODEL_NAME}"
+  EXPORTED_MODEL=$(find "." -type f -name "${MODEL_NAME}*.pte" -print -quit)
 }
 
 if [[ "${BACKEND}" == "portable" ]]; then
@@ -170,6 +201,12 @@ elif [[ "${BACKEND}" == "qnn" ]]; then
   if [[ $? -eq 0 ]]; then
     prepare_artifacts_upload
   fi
+elif [[ "${BACKEND}" == "coreml" ]]; then
+  echo "Testing ${MODEL_NAME} with coreml..."
+  test_model_with_coreml
+  if [[ $? -eq 0 ]]; then
+    prepare_artifacts_upload
+  fi
 elif [[ "${BACKEND}" == "xnnpack" ]]; then
   echo "Testing ${MODEL_NAME} with xnnpack..."
   test_model_with_xnnpack true true

.github/pytorch-probot.yml

@@ -1,5 +1,7 @@
 # The schema is from https://github.com/pytorch/pytorch/blob/main/.github/pytorch-probot.yml
 ciflow_push_tags:
+- ciflow/android
+- ciflow/apple
 - ciflow/nightly
 - ciflow/trunk
 - ciflow/binaries