Portable selected op workaround for arm backend (#1122)

Summary:
Arm uses portable yaml, and it only needs softmax. However, now we disabled root op list, and can only support yaml, so this breaks arm build.

We temporarily support yaml and ops list while we make a selective build rule for arm.

Pull Request resolved: #1122

Reviewed By: larryliu0820

Differential Revision: D50936359

Pulled By: kirklandsign

fbshipit-source-id: b220d3526f2a40769ebd06b492a3563da299920d

Author: kirklandsign

.ci/docker/ubuntu/Dockerfile

@@ -54,7 +54,7 @@ RUN rm install_linter.sh utils.sh requirements-lintrunner.txt
 ARG ARM_SDK
 COPY --chown=ci-user:ci-user ./arm /opt/arm
 # Set up ARM SDK if needed
-RUN if [ -n "${ARM_SDK}" ]; then git config --global user.email "ci@example.com"; git config --global user.name "OSS CI"; bash /opt/arm/setup.sh --i-agree-to-the-contained-eula /opt/arm-sdk; chown -R ci-user:ci-user /opt/arm-sdk; fi
+RUN if [ -n "${ARM_SDK}" ]; then git config --global user.email "ossci@example.com"; git config --global user.name "OSS CI"; bash /opt/arm/setup.sh --i-agree-to-the-contained-eula /opt/arm-sdk; chown -R ci-user:ci-user /opt/arm-sdk; fi
 
 USER ci-user
 CMD ["bash"]

.github/workflows/pull.yml

@@ -106,26 +106,6 @@ jobs:
         # Test selective build
         PYTHON_EXECUTABLE=python bash examples/selective_build/test_selective_build.sh "${BUILD_TOOL}"
 
-  test-arm-backend-delegation:
-    name: test-arm-backend-delegation
-    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
-    with:
-      runner: linux.2xlarge
-      docker-image: executorch-ubuntu-22.04-arm-sdk
-      submodules: 'true'
-      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-      script: |
-        # The generic Linux job chooses to use base env, not the one setup by the image
-        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
-        conda activate "${CONDA_ENV}"
-
-        source .ci/scripts/utils.sh
-        install_flatc_from_source
-        install_executorch
-        # Test selective build
-        source /opt/arm-sdk/setup_path.sh
-        PYTHON_EXECUTABLE=python bash examples/arm/run.sh /opt/arm-sdk buck2
-
   unittest:
     uses: ./.github/workflows/_unittest.yml
     with:

examples/arm/CMakeLists.txt

@@ -31,7 +31,7 @@ set(_common_compile_options -Wno-deprecated-declarations -fPIC)
 # Let files say "include <executorch/path/to/header.h>".
 set(_common_include_directories ${EXECUTORCH_ROOT}/..)
 
-find_package(executorch CONFIG REQUIRED)
+find_package(executorch CONFIG REQUIRED HINTS ${CMAKE_INSTALL_PREFIX})
 target_include_directories(executorch INTERFACE ${_common_include_directories})
 
 include(${EXECUTORCH_ROOT}/build/Utils.cmake)
@@ -40,5 +40,6 @@ include(${EXECUTORCH_ROOT}/build/Codegen.cmake)
 # Generate C++ bindings to register kernels into both PyTorch (for AOT) and
 # Executorch (for runtime). Here select all ops in functions.yaml
 gen_selected_ops("" "${EXECUTORCH_SELECT_OPS_LIST}" "")
-generate_bindings_for_kernels(${EXECUTORCH_ROOT}/kernels/portable/functions.yaml "")
-gen_operators_lib("arm_ops_lib" portable_kernels executorch)
+generate_bindings_for_kernels(
+  ${EXECUTORCH_ROOT}/kernels/portable/functions.yaml "")
+gen_operators_lib("portable_ops_lib" portable_kernels executorch)

examples/arm/aot_arm_compiler.py

@@ -13,7 +13,7 @@
 import torch
 import torch._export as export
 
-from executorch.backends.arm.arm_backend import ArmPartitioner
+from executorch.backends.arm.arm_partitioner import ArmPartitioner
 from executorch.exir import EdgeCompileConfig
 
 from ..portable.utils import export_to_edge, save_pte_program

examples/arm/ethos-u-setup/core_platform/patches/0009-Executorch-Use-libportable_ops_lib-instead.patch

@@ -0,0 +1,25 @@
+From 8ee58a2ac02693ec5f261c866fc333d268f253d4 Mon Sep 17 00:00:00 2001
+From: Hansong Zhang <[email protected]>
+Date: Thu, 2 Nov 2023 14:48:03 -0700
+Subject: [PATCH] Use libportable_ops_lib instead
+
+---
+ applications/executorch_tests/CMakeLists.txt | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/applications/executorch_tests/CMakeLists.txt b/applications/executorch_tests/CMakeLists.txt
+index a07f22f..174b911 100644
+--- a/applications/executorch_tests/CMakeLists.txt
++++ b/applications/executorch_tests/CMakeLists.txt
+@@ -43,7 +43,7 @@ message("**********************")
+ 
+ set(LIB_ET_RUNTIME "${ET_BUILD_DIR_PATH}/libexecutorch.a")
+ set(LIB_ET_ETHOS "${ET_BUILD_DIR_PATH}/backends/arm/libexecutorch_delegate_ethos_u.a")
+-set(LIB_ET_OP_REGISTRATION "${ET_BUILD_DIR_PATH}/examples/arm/libarm_ops_lib.a")
++set(LIB_ET_OP_REGISTRATION "${ET_BUILD_DIR_PATH}/examples/arm/libportable_ops_lib.a")
+ set(LIB_ET_OP_KERNELS "${ET_BUILD_DIR_PATH}/kernels/portable/libportable_kernels.a")
+ 
+ add_custom_target(
+-- 
+2.39.3
+

examples/arm/run.sh

@@ -54,14 +54,16 @@ function generate_pte_file() {
 
 # build ExecuTorch Libraries
 function build_executorch() {
+    set -x
+
     [[ -d "${et_build_dir}" ]] \
         && echo "[${FUNCNAME[0]}] Warn: using already existing build-dir for executorch: ${et_build_dir}!!"
     mkdir -p "${et_build_dir}"
 
     cd "${et_root_dir}"
     cmake                                                 \
         -DBUCK2=${buck2}                                  \
-        -DCMAKE_INSTALL_PREFIX=cmake-out                  \
+        -DCMAKE_INSTALL_PREFIX=${et_build_dir}            \
         -DEXECUTORCH_BUILD_EXECUTOR_RUNNER=OFF            \
         -DCMAKE_BUILD_TYPE=Release                        \
         -DEXECUTORCH_ENABLE_LOGGING=ON                    \
@@ -74,18 +76,19 @@ function build_executorch() {
     echo "[${FUNCNAME[0]}] Configured CMAKE"
 
     n=$(nproc)
-    cmake --build ${et_build_dir} --target install --config Release -- -j"$((n - 5))"
+    cmake --build ${et_build_dir} -j"$((n - 5))" --target install --config Release
 
     cmake                                                 \
-        -DCMAKE_INSTALL_PREFIX=cmake-out                  \
+        -DCMAKE_INSTALL_PREFIX=${et_build_dir}            \
         -DCMAKE_BUILD_TYPE=Release                        \
         -DEXECUTORCH_SELECT_OPS_LIST="aten::_softmax.out" \
         -DCMAKE_TOOLCHAIN_FILE="${toolchain_cmake}"       \
         -B"${et_build_dir}"/examples/arm                  \
-        -Dexecutorch_DIR="${et_root_dir}"/build           \
         "${et_root_dir}"/examples/arm
     cmake --build ${et_build_dir}/examples/arm -- -j"$((n - 5))"
 
+    set +x
+
     cd "${et_build_dir}"
     echo "[${FUNCNAME[0]}] Generated static libraries for ExecuTorch:"
     find . -name "*.a" -exec ls -al {} \;