Qualcomm AI Engine Direct Backend

.gitmodules

@@ -13,6 +13,9 @@
 [submodule "third-party/googletest"]
 	path = third-party/googletest
 	url = https://github.com/google/googletest.git
+[submodule "third-party/pybind11"]
+	path = third-party/pybind11
+	url = https://github.com/pybind/pybind11.git
 [submodule "backends/xnnpack/third-party/pthreadpool"]
 	path = backends/xnnpack/third-party/pthreadpool
 	url = https://github.com/Maratyszcza/pthreadpool.git

CMakeLists.txt

@@ -300,6 +300,12 @@ if(EXECUTORCH_BUILD_XNNPACK)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/backends/xnnpack)
 endif()
 
+option(EXECUTORCH_BUILD_QNN "Build the backends/qualcomm directory" OFF)
+if(EXECUTORCH_BUILD_QNN)
+  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/backends/qualcomm)
+  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/examples/qualcomm)
+endif()
+
 # Add selective build subdirectory
 if(BUILD_SELECTIVE_BUILD_TEST)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/examples/selective_build)

LICENSE

@@ -4,6 +4,7 @@ For "ExecuTorch" software
 
 Copyright (c) Meta Platforms, Inc. and affiliates.
 Copyright 2023 Arm Limited and/or its affiliates.
+Copyright (c) Qualcomm Innovation Center, Inc.
 
 Redistribution and use in source and binary forms, with or without modification,
 are permitted provided that the following conditions are met:

backends/qualcomm/CMakeLists.txt

@@ -0,0 +1,233 @@
+# Copyright (c) Qualcomm Innovation Center, Inc.
+# All rights reserved
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+#
+# get path
+#
+get_filename_component(EXECUTORCH_SOURCE_DIR
+                       "${CMAKE_CURRENT_LIST_DIR}/../.."
+                       ABSOLUTE)
+get_filename_component(QNN_EXECUTORCH_ROOT_DIR
+                       ${CMAKE_CURRENT_LIST_DIR}
+                       ABSOLUTE)
+# Let files say "include <executorch/path/to/header.h>".
+get_filename_component(_common_include_directories
+                       "${EXECUTORCH_SOURCE_DIR}/.."
+                       ABSOLUTE)
+
+if(NOT DEFINED QNN_SDK_ROOT)
+    message(FATAL_ERROR
+            "Please define QNN_SDK_ROOT, e.g. cmake <..> -DQNN_SDK_ROOT=<...>")
+elseif(CMAKE_TOOLCHAIN_FILE MATCHES ".*ios\.toolchain\.cmake$")
+    message(FATAL_ERROR
+            "ios is not supported by Qualcomm AI Engine Direct")
+endif()
+
+message(STATUS "Using qnn sdk root ${QNN_SDK_ROOT}")
+message(STATUS "Using EXECUTORCH_SOURCE_DIR ${EXECUTORCH_SOURCE_DIR}")
+
+if(${ANDROID})
+    find_library(android_log log)
+endif()
+
+add_compile_options("-Wall" "-Werror" "-Wno-sign-compare")
+
+# GNU emit wanring for ignored attributes
+# Unfortunately, we use [[maybe_unused]] which can be ignored by GNU.
+# So we make it a warning, not an error in GNU.
+if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    add_compile_options("-Wno-error=attributes")
+endif()
+
+if(CMAKE_BUILD_TYPE STREQUAL "Release")
+    # strip symbols
+    add_link_options("-s")
+    # hide dynamic symbols
+    set(CMAKE_C_VISIBILITY_PRESET hidden)
+    set(CMAKE_CXX_VISIBILITY_PRESET hidden)
+
+    # --gc-sections is added by torch.
+    add_compile_definitions("-O3" "-ffunction-sections" "-fdata-sections")
+endif()
+
+
+include_directories(
+    BEFORE
+    ${_common_include_directories}
+)
+include_directories(
+    BEFORE
+    ${QNN_SDK_ROOT}/include/QNN
+)
+
+#
+# declare targets
+#
+add_library(executorch_backend INTERFACE)
+add_library(qnn_executorch_backend STATIC)
+add_library(qnn_executorch_header INTERFACE)
+add_library(qnn_manager STATIC)
+add_library(qnn_function_interface INTERFACE)
+add_library(qnn_implementation STATIC)
+add_library(qnn_sys_function_interface INTERFACE)
+add_library(qnn_sys_implementation STATIC)
+add_library(qnn_logger STATIC)
+add_library(qnn_device STATIC)
+add_library(qnn_context STATIC)
+add_library(qnn_backend_cache STATIC)
+add_library(qnn_graph STATIC)
+add_library(qnn_backend STATIC)
+add_library(qnn_factory STATIC)
+add_library(qnn_header INTERFACE)
+add_library(qnn_logging STATIC)
+add_library(wrappers STATIC)
+add_library(utils STATIC)
+
+#
+# declare dependency
+#
+target_link_libraries(wrappers
+    PRIVATE
+    qnn_header
+)
+target_link_libraries(qnn_function_interface
+    INTERFACE
+    qnn_header
+)
+target_link_libraries(qnn_implementation
+    PRIVATE
+    qnn_function_interface
+    qnn_header
+    qnn_logging
+    ${CMAKE_DL_LIBS}
+)
+target_link_libraries(qnn_sys_function_interface
+    INTERFACE
+    qnn_header
+)
+target_link_libraries(qnn_sys_implementation
+    PRIVATE
+    qnn_sys_function_interface
+    qnn_header
+    qnn_logging
+    ${CMAKE_DL_LIBS}
+)
+target_link_libraries(qnn_logger
+    PRIVATE
+    qnn_implementation
+    ${android_log}
+)
+target_link_libraries(qnn_backend
+    PRIVATE
+    qnn_implementation
+    qnn_logger
+)
+target_link_libraries(qnn_device
+    PRIVATE
+    qnn_implementation
+    qnn_logger
+    utils
+)
+target_link_libraries(qnn_backend_cache
+    PRIVATE
+    qnn_sys_implementation
+)
+target_link_libraries(qnn_context
+    PRIVATE
+    qnn_implementation
+    qnn_logger
+    qnn_backend
+    qnn_device
+    qnn_backend_cache
+)
+target_link_libraries(qnn_graph
+    PRIVATE
+    qnn_implementation
+    qnn_context
+)
+target_link_libraries(qnn_factory
+    PUBLIC
+    qnn_header
+    PRIVATE
+    qnn_backend
+    qnn_device
+    qnn_context
+    qnn_graph
+)
+target_link_libraries(qnn_manager
+    PRIVATE
+    qnn_factory
+    wrappers
+)
+target_link_libraries(qnn_executorch_backend
+    PRIVATE
+    qnn_executorch_header
+    qnn_manager
+    executorch
+)
+
+#
+# add linker option
+#
+target_link_options_shared_lib(qnn_executorch_backend)
+
+#
+# add sources
+#
+add_subdirectory(
+    ${QNN_EXECUTORCH_ROOT_DIR}/runtime
+    ${CMAKE_CURRENT_BINARY_DIR}/qnn_executorch
+)
+add_subdirectory(
+    ${QNN_EXECUTORCH_ROOT_DIR}/runtime/backends
+    ${CMAKE_CURRENT_BINARY_DIR}/qnn_executorch/backends
+)
+add_subdirectory(
+    ${QNN_EXECUTORCH_ROOT_DIR}/aot/wrappers
+    ${CMAKE_CURRENT_BINARY_DIR}/qnn_executorch/wrappers
+)
+
+# QNN pybind
+if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64")
+    add_subdirectory(${EXECUTORCH_SOURCE_DIR}/third-party/pybind11
+                    ${CMAKE_CURRENT_BINARY_DIR}/pybind11)
+    add_library(PyQnnManagerAdaptor MODULE)
+    add_library(PyQnnWrapperAdaptor MODULE)
+
+    target_link_libraries(PyQnnManagerAdaptor
+        PRIVATE
+        pybind11::module
+        pybind11::lto
+        qnn_manager
+        qnn_executorch_header
+        executorch
+    )
+    target_link_libraries(PyQnnWrapperAdaptor
+        PRIVATE
+        pybind11::module
+        pybind11::lto
+        wrappers
+        qnn_logging
+        qnn_executorch_header
+    )
+
+    pybind11_extension(PyQnnManagerAdaptor)
+    pybind11_extension(PyQnnWrapperAdaptor)
+    if(NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo)
+        # Strip unnecessary sections of the binary
+        pybind11_strip(PyQnnManagerAdaptor)
+        pybind11_strip(PyQnnWrapperAdaptor)
+    endif()
+
+    add_subdirectory(
+        ${QNN_EXECUTORCH_ROOT_DIR}/aot/python
+        ${CMAKE_CURRENT_BINARY_DIR}/qnn_executorch/python
+    )
+endif()

backends/qualcomm/README.md

@@ -0,0 +1,56 @@
+# Qualcomm AI Engine Direct Backend
+
+Disclaimer: At present, we do not offer any backward compatibility guarantees
+for any APIs. We are currently in a pre-alpha development phase, and as such,
+we reserve the right to modify interfaces and implementations.
+
+This backend is implemented on the top of
+[Qualcomm AI Engine Direct SDK](https://developer.qualcomm.com/software/qualcomm-ai-engine-direct-sdk).
+Please follow [setup](setup.md) to setup environment, build, and run executorch models by this backend.
+
+## Delegate Options
+
+Please check `generate_qnn_executorch_compiler_spec()` in
+[utils.py](./utils/utils.py) for supported SoC and inference type.
+
+### Supported Chipset
+- Snapdragon 8 Gen 1
+- Snapdragon 8 Gen 1+
+- Snapdragon 8 Gen 2
+
+### Supported Inference Type
+- Quantized
+- FP16
+
+## Directory Structure
+
+```
+backends/qualcomm
+├── aot # Codes for generating QNN context binary (AoT Part).
+|   ├── wrappers # Wrapper of QNN data structures for ease of use.
+|   └── python # Python interface for using QNN libraries.
+├── builders # Codes for lowering each operators (AoT Part).
+├── partition # QNN Partitioner (AoT Part).
+├── passes # Various passes helping lower models to QNN backend (AoT Part).
+├── python # Places to put pybind artifacts for accessing QNN APIs, structures, etc (AoT Part).
+├── runtime # Here is QNN runtime responsbile for compiling a model on x64.
+|   |       # Meanwhile, this is also the runtime responsbile for executing compiled
+|   |       # models on a device.
+|   └── backends # Backends supported by QNN.
+|       └── htpbackend
+|           ├── aarch64 # Configuration required to run on device. (Device Part).
+|           └── x86_64 # Configuration required to compile graph on host. (AoT Part).
+├── scripts # Misc supporting scripts, not related to core functionality.
+├── tests # Unit tests and model tests go here.
+└── utils # Miscellaneous utilities.
+
+examples/backend
+└── qualcomm # Examples to run QNN backends.
+```
+
+## Examples
+
+Please see this [README.md](../../examples/backend/qualcomm/README.md).
+
+Further, an example build script is provided as [build.sh](scripts/build.sh).
+

backends/qualcomm/aot/python/CMakeLists.txt

@@ -0,0 +1,20 @@
+# Copyright (c) Qualcomm Innovation Center, Inc.
+# All rights reserved
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+
+# PyQnnManagerAdaptor
+target_sources(PyQnnManagerAdaptor
+    PUBLIC
+    ${CMAKE_CURRENT_LIST_DIR}/PyQnnManagerAdaptor.cpp
+)
+
+# PyQnnWrapperAdaptor
+target_sources(PyQnnWrapperAdaptor
+    PUBLIC
+    ${CMAKE_CURRENT_LIST_DIR}/PyQnnWrapperAdaptor.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/PyQnnWrapperAdaptor.h
+)
+