Qualcomm AI Engine Direct Backend (#490)

Summary:
Hi there,

This commit adds QNN backend. Please check setup.md and README under `backends/qnn` for more details.

We add pybind dependency at the top level, because we guess you might need it in the near future. If it's not the case, please feel free to tell us. We'll move it under `backends/qnn`.

Below is some SoC model mapping:

- `SM8550` -> `Snapdragon 8 Gen 2`
- `SM8475` -> `Snapdragon 8 Gen 1+`
- `SM8450` -> `Snapdragon 8 Gen 1`

Please feel free to contact us if you find any problem.

---------

Pull Request resolved: #490

Reviewed By: cccclai

Differential Revision: D49686980

Pulled By: kimishpatel

fbshipit-source-id: 5398137bd015a056de80f55a921f4d718df08a24

Co-authored-by: shewu-quic <[email protected]>
Co-authored-by: chunit-quic <[email protected]>
Co-authored-by: chiwwang <[email protected]>
Co-authored-by: harshs-qti <[email protected]>

Author: 5 people

.gitmodules

@@ -13,6 +13,9 @@
 [submodule "third-party/googletest"]
 	path = third-party/googletest
 	url = https://github.com/google/googletest.git
+[submodule "third-party/pybind11"]
+	path = third-party/pybind11
+	url = https://github.com/pybind/pybind11.git
 [submodule "backends/xnnpack/third-party/pthreadpool"]
 	path = backends/xnnpack/third-party/pthreadpool
 	url = https://github.com/Maratyszcza/pthreadpool.git

CMakeLists.txt

@@ -307,6 +307,12 @@ if(EXECUTORCH_BUILD_XNNPACK)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/backends/xnnpack)
 endif()
 
+option(EXECUTORCH_BUILD_QNN "Build the backends/qualcomm directory" OFF)
+if(EXECUTORCH_BUILD_QNN)
+  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/backends/qualcomm)
+  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/examples/qualcomm)
+endif()
+
 # Add selective build subdirectory
 if(BUILD_SELECTIVE_BUILD_TEST)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/examples/selective_build)

LICENSE

@@ -4,6 +4,7 @@ For "ExecuTorch" software
 
 Copyright (c) Meta Platforms, Inc. and affiliates.
 Copyright 2023 Arm Limited and/or its affiliates.
+Copyright (c) Qualcomm Innovation Center, Inc.
 
 Redistribution and use in source and binary forms, with or without modification,
 are permitted provided that the following conditions are met:

backends/qualcomm/CMakeLists.txt

@@ -0,0 +1,233 @@
+# Copyright (c) Qualcomm Innovation Center, Inc.
+# All rights reserved
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+#
+# get path
+#
+get_filename_component(EXECUTORCH_SOURCE_DIR
+                       "${CMAKE_CURRENT_LIST_DIR}/../.."
+                       ABSOLUTE)
+get_filename_component(QNN_EXECUTORCH_ROOT_DIR
+                       ${CMAKE_CURRENT_LIST_DIR}
+                       ABSOLUTE)
+# Let files say "include <executorch/path/to/header.h>".
+get_filename_component(_common_include_directories
+                       "${EXECUTORCH_SOURCE_DIR}/.."
+                       ABSOLUTE)
+
+if(NOT DEFINED QNN_SDK_ROOT)
+    message(FATAL_ERROR
+            "Please define QNN_SDK_ROOT, e.g. cmake <..> -DQNN_SDK_ROOT=<...>")
+elseif(CMAKE_TOOLCHAIN_FILE MATCHES ".*ios\.toolchain\.cmake$")
+    message(FATAL_ERROR
+            "ios is not supported by Qualcomm AI Engine Direct")
+endif()
+
+message(STATUS "Using qnn sdk root ${QNN_SDK_ROOT}")
+message(STATUS "Using EXECUTORCH_SOURCE_DIR ${EXECUTORCH_SOURCE_DIR}")
+
+if(${ANDROID})
+    find_library(android_log log)
+endif()
+
+add_compile_options("-Wall" "-Werror" "-Wno-sign-compare")
+
+# GNU emit wanring for ignored attributes
+# Unfortunately, we use [[maybe_unused]] which can be ignored by GNU.
+# So we make it a warning, not an error in GNU.
+if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    add_compile_options("-Wno-error=attributes")
+endif()
+
+if(CMAKE_BUILD_TYPE STREQUAL "Release")
+    # strip symbols
+    add_link_options("-s")
+    # hide dynamic symbols
+    set(CMAKE_C_VISIBILITY_PRESET hidden)
+    set(CMAKE_CXX_VISIBILITY_PRESET hidden)
+
+    # --gc-sections is added by torch.
+    add_compile_definitions("-O3" "-ffunction-sections" "-fdata-sections")
+endif()
+
+
+include_directories(
+    BEFORE
+    ${_common_include_directories}
+)
+include_directories(
+    BEFORE
+    ${QNN_SDK_ROOT}/include/QNN
+)
+
+#
+# declare targets
+#
+add_library(executorch_backend INTERFACE)
+add_library(qnn_executorch_backend STATIC)
+add_library(qnn_executorch_header INTERFACE)
+add_library(qnn_manager STATIC)
+add_library(qnn_function_interface INTERFACE)
+add_library(qnn_implementation STATIC)
+add_library(qnn_sys_function_interface INTERFACE)
+add_library(qnn_sys_implementation STATIC)
+add_library(qnn_logger STATIC)
+add_library(qnn_device STATIC)
+add_library(qnn_context STATIC)
+add_library(qnn_backend_cache STATIC)
+add_library(qnn_graph STATIC)
+add_library(qnn_backend STATIC)
+add_library(qnn_factory STATIC)
+add_library(qnn_header INTERFACE)
+add_library(qnn_logging STATIC)
+add_library(wrappers STATIC)
+add_library(utils STATIC)
+
+#
+# declare dependency
+#
+target_link_libraries(wrappers
+    PRIVATE
+    qnn_header
+)
+target_link_libraries(qnn_function_interface
+    INTERFACE
+    qnn_header
+)
+target_link_libraries(qnn_implementation
+    PRIVATE
+    qnn_function_interface
+    qnn_header
+    qnn_logging
+    ${CMAKE_DL_LIBS}
+)
+target_link_libraries(qnn_sys_function_interface
+    INTERFACE
+    qnn_header
+)
+target_link_libraries(qnn_sys_implementation
+    PRIVATE
+    qnn_sys_function_interface
+    qnn_header
+    qnn_logging
+    ${CMAKE_DL_LIBS}
+)
+target_link_libraries(qnn_logger
+    PRIVATE
+    qnn_implementation
+    ${android_log}
+)
+target_link_libraries(qnn_backend
+    PRIVATE
+    qnn_implementation
+    qnn_logger
+)
+target_link_libraries(qnn_device
+    PRIVATE
+    qnn_implementation
+    qnn_logger
+    utils
+)
+target_link_libraries(qnn_backend_cache
+    PRIVATE
+    qnn_sys_implementation
+)
+target_link_libraries(qnn_context
+    PRIVATE
+    qnn_implementation
+    qnn_logger
+    qnn_backend
+    qnn_device
+    qnn_backend_cache
+)
+target_link_libraries(qnn_graph
+    PRIVATE
+    qnn_implementation
+    qnn_context
+)
+target_link_libraries(qnn_factory
+    PUBLIC
+    qnn_header
+    PRIVATE
+    qnn_backend
+    qnn_device
+    qnn_context
+    qnn_graph
+)
+target_link_libraries(qnn_manager
+    PRIVATE
+    qnn_factory
+    wrappers
+)
+target_link_libraries(qnn_executorch_backend
+    PRIVATE
+    qnn_executorch_header
+    qnn_manager
+    executorch
+)
+
+#
+# add linker option
+#
+target_link_options_shared_lib(qnn_executorch_backend)
+
+#
+# add sources
+#
+add_subdirectory(
+    ${QNN_EXECUTORCH_ROOT_DIR}/runtime
+    ${CMAKE_CURRENT_BINARY_DIR}/qnn_executorch
+)
+add_subdirectory(
+    ${QNN_EXECUTORCH_ROOT_DIR}/runtime/backends
+    ${CMAKE_CURRENT_BINARY_DIR}/qnn_executorch/backends
+)
+add_subdirectory(
+    ${QNN_EXECUTORCH_ROOT_DIR}/aot/wrappers
+    ${CMAKE_CURRENT_BINARY_DIR}/qnn_executorch/wrappers
+)
+
+# QNN pybind
+if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64")
+    add_subdirectory(${EXECUTORCH_SOURCE_DIR}/third-party/pybind11
+                    ${CMAKE_CURRENT_BINARY_DIR}/pybind11)
+    add_library(PyQnnManagerAdaptor MODULE)
+    add_library(PyQnnWrapperAdaptor MODULE)
+
+    target_link_libraries(PyQnnManagerAdaptor
+        PRIVATE
+        pybind11::module
+        pybind11::lto
+        qnn_manager
+        qnn_executorch_header
+        executorch
+    )
+    target_link_libraries(PyQnnWrapperAdaptor
+        PRIVATE
+        pybind11::module
+        pybind11::lto
+        wrappers
+        qnn_logging
+        qnn_executorch_header
+    )
+
+    pybind11_extension(PyQnnManagerAdaptor)
+    pybind11_extension(PyQnnWrapperAdaptor)
+    if(NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo)
+        # Strip unnecessary sections of the binary
+        pybind11_strip(PyQnnManagerAdaptor)
+        pybind11_strip(PyQnnWrapperAdaptor)
+    endif()
+
+    add_subdirectory(
+        ${QNN_EXECUTORCH_ROOT_DIR}/aot/python
+        ${CMAKE_CURRENT_BINARY_DIR}/qnn_executorch/python
+    )
+endif()

backends/qualcomm/README.md

@@ -0,0 +1,56 @@
+# Qualcomm AI Engine Direct Backend
+
+Disclaimer: At present, we do not offer any backward compatibility guarantees
+for any APIs. We are currently in a pre-alpha development phase, and as such,
+we reserve the right to modify interfaces and implementations.
+
+This backend is implemented on the top of
+[Qualcomm AI Engine Direct SDK](https://developer.qualcomm.com/software/qualcomm-ai-engine-direct-sdk).
+Please follow [setup](setup.md) to setup environment, build, and run executorch models by this backend.
+
+## Delegate Options
+
+Please check `generate_qnn_executorch_compiler_spec()` in
+[utils.py](./utils/utils.py) for supported SoC and inference type.
+
+### Supported Chipset
+- Snapdragon 8 Gen 1
+- Snapdragon 8 Gen 1+
+- Snapdragon 8 Gen 2
+
+### Supported Inference Type
+- Quantized
+- FP16
+
+## Directory Structure
+
+```
+backends/qualcomm
+├── aot # Codes for generating QNN context binary (AoT Part).
+|   ├── wrappers # Wrapper of QNN data structures for ease of use.
+|   └── python # Python interface for using QNN libraries.
+├── builders # Codes for lowering each operators (AoT Part).
+├── partition # QNN Partitioner (AoT Part).
+├── passes # Various passes helping lower models to QNN backend (AoT Part).
+├── python # Places to put pybind artifacts for accessing QNN APIs, structures, etc (AoT Part).
+├── runtime # Here is QNN runtime responsbile for compiling a model on x64.
+|   |       # Meanwhile, this is also the runtime responsbile for executing compiled
+|   |       # models on a device.
+|   └── backends # Backends supported by QNN.
+|       └── htpbackend
+|           ├── aarch64 # Configuration required to run on device. (Device Part).
+|           └── x86_64 # Configuration required to compile graph on host. (AoT Part).
+├── scripts # Misc supporting scripts, not related to core functionality.
+├── tests # Unit tests and model tests go here.
+└── utils # Miscellaneous utilities.
+
+examples/backend
+└── qualcomm # Examples to run QNN backends.
+```
+
+## Examples
+
+Please see this [README.md](../../examples/backend/qualcomm/README.md).
+
+Further, an example build script is provided as [build.sh](scripts/build.sh).
+

backends/qualcomm/aot/python/CMakeLists.txt

@@ -0,0 +1,20 @@
+# Copyright (c) Qualcomm Innovation Center, Inc.
+# All rights reserved
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+
+# PyQnnManagerAdaptor
+target_sources(PyQnnManagerAdaptor
+    PUBLIC
+    ${CMAKE_CURRENT_LIST_DIR}/PyQnnManagerAdaptor.cpp
+)
+
+# PyQnnWrapperAdaptor
+target_sources(PyQnnWrapperAdaptor
+    PUBLIC
+    ${CMAKE_CURRENT_LIST_DIR}/PyQnnWrapperAdaptor.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/PyQnnWrapperAdaptor.h
+)
+