Adds CoreML backend

CMakeLists.txt

@@ -295,6 +295,18 @@ if(EXECUTORCH_BUILD_XNNPACK)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/backends/xnnpack)
 endif()
 
+# Add COREML subdirectory
+if(EXECUTORCH_BUILD_COREML_DELGATE)
+  # CoreML delegate library can only be built with iOS toolchain
+  if(CMAKE_TOOLCHAIN_FILE MATCHES ".*ios\.toolchain\.cmake$")
+    add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/backends/coreml)
+  else()
+    message(
+      FATAL_ERROR "executorch: Building CoreML delegate requires iOS toolchain"
+    )
+  endif()
+endif()
+
 # Add selective build subdirectory
 if(BUILD_SELECTIVE_BUILD_TEST)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/examples/selective_build)

backends/coreml/.clang-format

@@ -0,0 +1,29 @@
+BasedOnStyle: WebKit
+BreakBeforeBraces: Attach
+AllowShortIfStatementsOnASingleLine: false
+BreakBeforeBinaryOperators: None
+BreakConstructorInitializers: BeforeColon
+IndentCaseLabels: true
+SortIncludes: true
+SpaceBeforeRangeBasedForLoopColon: false
+SpaceBeforeParens: ControlStatements
+AlignAfterOpenBracket: Align
+NamespaceIndentation: None
+MaxEmptyLinesToKeep: 2
+#AlignConsecutiveAssignments: true
+
+ColumnLimit: 120
+
+# When breaking up parameter/argument lists across multiple lines,
+# put only one per line instead of packing as many as possible.
+BinPackArguments: false
+BinPackParameters: false
+
+# Control of individual brace wrapping cases.
+BreakBeforeBraces: Custom
+BraceWrapping:
+  BeforeCatch: true
+
+# Options for aligning backslashes in escaped newlines.
+AlignEscapedNewlines: Left
+

backends/coreml/.gitignore

@@ -0,0 +1,23 @@
+# Mac OS X
+*.DS_Store
+
+cmake-ios-out/
+
+
+# Xcode
+*.pbxuser
+*.mode1v3
+*.mode2v3
+*.perspectivev3
+*.xcuserstate
+project.xcworkspace/
+xcuserdata/
+
+
+runtime/runner/build/
+runtime/libraries/
+runtime/inmemoryfs/build/
+xcode-runner-build/
+xcode-test-build/
+
+*.egg-info

backends/coreml/CMakeLists.txt

@@ -0,0 +1,105 @@
+# Copyright © 2023 Apple Inc. All rights reserved.
+
+cmake_minimum_required(VERSION 3.19)
+
+project(executorch_coreml_backend)
+
+enable_language(CXX)
+enable_language(OBJC)
+
+# Source root directory for executorch.
+if(NOT EXECUTORCH_ROOT)
+  set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../..)
+endif()
+
+# inmemoryfs sources
+set(INMEMORYFS_SOURCES
+  runtime/inmemoryfs/inmemory_filesystem.cpp
+  runtime/inmemoryfs/memory_buffer.cpp
+  runtime/inmemoryfs/memory_stream.cpp
+  runtime/inmemoryfs/reversed_memory_stream.cpp
+)
+
+# kvstore sources
+set(KVSTORE_SOURCES
+  runtime/kvstore/database.cpp
+  runtime/kvstore/sqlite_error.cpp
+  runtime/kvstore/key_value_store.cpp
+  runtime/kvstore/statement.cpp
+)
+
+# delegate sources
+set(DELEGATE_SOURCES
+  runtime/delegate/asset.mm
+  runtime/delegate/backend_delegate.mm
+  runtime/delegate/coreml_backend_delegate.mm
+  runtime/delegate/ETCoreMLAsset.mm
+  runtime/delegate/ETCoreMLAssetManager.mm
+  runtime/delegate/ETCoreMLLogging.mm
+  runtime/delegate/ETCoreMLModel.mm
+  runtime/delegate/ETCoreMLModelManager.mm
+  runtime/delegate/ETCoreMLStrings.mm
+  runtime/delegate/MLModel_Prewarm.mm
+  runtime/delegate/MLMultiArray_Copy.mm
+  runtime/delegate/multiarray.mm
+  runtime/delegate/serde_json.mm
+)
+
+# Define the delegate library
+add_library(coremldelegate)
+target_sources(
+  coremldelegate PRIVATE
+  ${INMEMORYFS_SOURCES}
+  ${KVSTORE_SOURCES}
+  ${DELEGATE_SOURCES}
+)
+
+target_include_directories(
+  coremldelegate PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/runtime/kvstore
+)
+target_include_directories(
+  coremldelegate PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/runtime/inmemoryfs
+)
+target_include_directories(
+  coremldelegate PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/runtime/delegate
+)
+target_include_directories(
+  coremldelegate PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/third-party/nlohmann
+)
+target_include_directories(
+  coremldelegate PRIVATE
+  ${EXECUTORCH_ROOT}/..
+)
+
+target_link_libraries(
+  coremldelegate PRIVATE
+  executorch
+  gflags
+)
+
+target_compile_options(coremldelegate PUBLIC "-fobjc-arc")
+
+set(
+  TARGET coremldelegate
+  APPEND_STRING PROPERTY COMPILE_FLAGS "-x objective-c++"
+)
+
+set(
+  TARGET coremldelegate
+  APPEND_STRING PROPERTY COMPILE_FLAGS "-Wno-null-character"
+)
+
+set(
+  TARGET coremldelegate
+  APPEND_STRING PROPERTY COMPILE_FLAGS "-Wno-receiver-expr"
+)
+
+set_property(
+  TARGET coremldelegate
+  PROPERTY CXX_STANDARD 17
+)
+

backends/coreml/README.md

@@ -0,0 +1,92 @@
+## CoreML Backend
+
+
+For setting up the **CoreML** backend please follow the instructions described in `backend/coreml/setup.md`. 
+
+## AOT 
+
+For delegating the Program to the **CoreML** backend, the client must be responsible for calling `to_backend` with the **CoreMLBackend** tag. 
+
+```
+from executorch.backends.coreml.compiler import CoreMLBackend
+
+# This will delegate the whole program to the CoreML backend. 
+lowered_module = to_backend("CoreMLBackend", edge.exported_program, [])
+
+```
+
+Currently, the **CoreML** backend would delegate the whole module to **CoreML**. If a specific op is not supported by the **CoreML** backend then the `to_backend` call would throw an exception. We will be adding a **Partitioner** for **CoreML** soon to resolve the issue. 
+
+The `preprocess` code is located at `backends/coreml/coreml_preprocess.py`. The implementation uses `coremltools` to convert the **ExportedProgram** to **MLPackage** which is flattened and returned as bytes to **ExecuTorch**.
+
+### Compute Units
+
+The `CoreML` runtime by default loads the model with compute units set to `MLComputeUnitsAll`. This can be overridden by providing `compute_units` at the preprocessing stage.
+
+**MLComputeUnitsCPUOnly**
+
+```
+# cpu is equivalent to MLComputeUnitsCPUOnly.
+lowered_module = to_backend("CoreMLBackend", edge.exported_program, [CompileSpec("compute_units", bytes("cpu", 'utf-8'))])
+
+```
+
+**MLComputeUnitsCPUAndGPU**
+
+```
+# gpu is equivalent to MLComputeUnitsCPUAndGPU.
+lowered_module = to_backend("CoreMLBackend", edge.exported_program, [CompileSpec("compute_units", bytes("gpu", 'utf-8'))])
+
+```
+
+
+**MLComputeUnitsCPUAndNeuralEngine**
+
+```
+# ane is equivalent to MLComputeUnitsCPUAndNeuralEngine.
+lowered_module = to_backend("CoreMLBackend", edge.exported_program, [CompileSpec("compute_units", bytes("ane", 'utf-8'))])
+
+```
+
+**MLComputeUnitsAll**
+
+```
+# all is equivalent to MLComputeUnitsAll.
+lowered_module = to_backend("CoreMLBackend", edge.exported_program, [CompileSpec("compute_units", bytes("all", 'utf-8'))])
+
+```
+
+
+## Runtime 
+
+Directory Structure.
+```
+
+executorch
+├── backends                        
+    ├── coreml                   #  CoreML backend implementation.                 
+        ├── runtime              #  CoreML runtime implementation.  
+            ├── inmemoryfs       #  In-Memory filesystem implementation. 
+            ├── kvstore          #  Persistent key-value store implementation.  
+            ├── delegate         #  Delegate implementation.  
+            ├── libraries        #  Linked libraries. 
+            ├── runner           #  Runner app implementation.  
+            ├── test             #  Test files and models. 
+
+```
+
+## Integration
+
+There are two steps required to integrate the **CoreML** backend.
+- Exporting the Program: The client must call `to_backend` with the `CoreMLBackend` tag. 
+
+```
+from executorch.backends.coreml.compiler import CoreMLBackend
+
+# This will delegate the whole program to the CoreML backend. 
+lowered_module = to_backend("CoreMLBackend", edge.exported_program, [])
+
+```
+- Running the delegated program: The client application must link with the `coremldelegate` library. Please follow the instructions described in the `backends/coreml/setup.md` to link the `coremldelegate` library. Once linked, there are no additional steps required. **ExecuTorch** would automatically call the **CoreML** delegate to execute the **CoreML** delegated part of the exported **Program**.
+
+

backends/coreml/compiler/__init__.py

@@ -0,0 +1,9 @@
+# Copyright © 2023 Apple Inc. All rights reserved.
+#
+# Please refer to the license found in the LICENSE file in the root directory of the source tree.
+
+from .coreml_preprocess import CoreMLBackend
+
+__all__ = [
+    CoreMLBackend,
+]

backends/coreml/compiler/coreml_preprocess.py

@@ -0,0 +1,76 @@
+#  Copyright © 2023 Apple Inc. All rights reserved.
+
+# CoreML backend for delegating a EdgeProgram to CoreML.
+
+import json
+import shutil
+import uuid
+
+from pathlib import Path
+
+from typing import final, List
+
+import coremltools as ct
+import executorchcoreml
+
+from executorch.exir.backend.backend_details import (
+    BackendDetails,
+    ExportedProgram,
+    PreprocessResult,
+)
+from executorch.exir.backend.compile_spec_schema import CompileSpec
+
+
+@final
+class CoreMLBackend(BackendDetails):
+    @staticmethod
+    def to_bytes(mlmodel):
+        dir_path = Path("tmp")
+        model_dir_path = dir_path / "lowered_module"
+        Path(model_dir_path).mkdir(parents=True, exist_ok=True)
+        model_path = model_dir_path / "model.mlpackage"
+        mlmodel.save(model_path)
+
+        # save model metdata
+        spec = mlmodel.get_spec()
+        input_names = [input.name for input in spec.description.input]
+        output_names = [output.name for output in spec.description.output]
+        identifier = uuid.uuid4()
+
+        model_metadata = {
+            "inputNames": input_names,
+            "outputNames": output_names,
+            "identifier": str(identifier),
+        }
+
+        # store metadata
+        model_metadata_path = Path(model_dir_path) / "metadata.json"
+        json_object = json.dumps(model_metadata)
+        with open(model_metadata_path, "w") as outfile:
+            outfile.write(json_object)
+
+        # flatten directory contents and convert it to bytes
+        flattened_bytes = executorchcoreml.flatten_directory_contents(
+            str(model_dir_path.resolve())
+        )
+        shutil.rmtree(str(model_dir_path.resolve()))
+        return flattened_bytes
+
+    @classmethod
+    # pyre-ignore
+    def preprocess(
+        cls,
+        edge_program: ExportedProgram,
+        module_compile_spec: List[CompileSpec],
+    ) -> PreprocessResult:
+        mlmodel = ct.convert(
+            model=edge_program,
+            source="pytorch",
+            convert_to="mlprogram",
+            pass_pipeline=ct.PassPipeline.DEFAULT,
+            skip_model_load=True,
+        )
+        flattened_bytes = CoreMLBackend.to_bytes(mlmodel)
+        return PreprocessResult(
+            processed_bytes=flattened_bytes,
+        )

backends/coreml/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+
+[tool.setuptools]
+py-modules = []
+
+[project]
+name = "executorchcoreml"
+version = "0.0.1"
+
+# Python dependencies required for development
+dependencies=[
+    "pybind11",
+    "numpy",
+]