CoreML backend changes.

.github/workflows/pull.yml

@@ -114,6 +114,34 @@ jobs:
         # Test selective build
         PYTHON_EXECUTABLE=python bash examples/selective_build/test_selective_build.sh "${BUILD_TOOL}"
 
+  test-coreml-delegate:
+    name: test-coreml-delegate
+    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
+    secrets: inherit
+    strategy:
+      matrix:
+        include:
+          - build-tool: cmake
+      fail-fast: false
+    with:
+      runner: macos-13-xlarge
+      submodules: 'true'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 60
+      secrets-env: TOKEN_COREML_PRIVATE_REPO
+      script: |
+        WORKSPACE=$(pwd)
+        pushd "${WORKSPACE}/pytorch/executorch"
+        BUILD_TOOL=${{ matrix.build-tool }}
+        # Setup MacOS dependencies as there is no Docker support on MacOS atm
+        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}"
+
+        # Build and test coreml delegate
+        PYTHON_EXECUTABLE=python sh backends/apple/coreml/scripts/install_requirements_internal.sh
+        PYTHON_EXECUTABLE=python sh backends/apple/coreml/scripts/build_tests.sh
+        PYTHON_EXECUTABLE=python sh backends/apple/coreml/scripts/run_tests.sh
+        popd
+
   unittest:
     uses: ./.github/workflows/_unittest.yml
     with:

CMakeLists.txt

@@ -346,6 +346,19 @@ if(EXECUTORCH_BUILD_MPS)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/examples/apple/mps)
 endif()
 
+# Build CoreML backend
+option(EXECUTORCH_BUILD_COREML "Build the backends/apple/coreml directory" OFF)
+if(EXECUTORCH_BUILD_COREML)
+  # CoreML delegate library can only be built with iOS toolchain
+  if(CMAKE_TOOLCHAIN_FILE MATCHES ".*(iOS\.)|(ios\.toolchain\.)cmake$")
+    add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/backends/apple/coreml)
+  else()
+    message(
+      FATAL_ERROR "executorch: Building CoreML delegate requires iOS toolchain"
+    )
+  endif()
+endif()
+
 # Add selective build subdirectory
 if(BUILD_SELECTIVE_BUILD_TEST)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/examples/selective_build)

backends/apple/coreml/.clang-format

@@ -0,0 +1,29 @@
+BasedOnStyle: WebKit
+BreakBeforeBraces: Attach
+AllowShortIfStatementsOnASingleLine: false
+BreakBeforeBinaryOperators: None
+BreakConstructorInitializers: BeforeColon
+IndentCaseLabels: true
+SortIncludes: true
+SpaceBeforeRangeBasedForLoopColon: false
+SpaceBeforeParens: ControlStatements
+AlignAfterOpenBracket: Align
+NamespaceIndentation: None
+MaxEmptyLinesToKeep: 2
+#AlignConsecutiveAssignments: true
+
+ColumnLimit: 120
+
+# When breaking up parameter/argument lists across multiple lines,
+# put only one per line instead of packing as many as possible.
+BinPackArguments: false
+BinPackParameters: false
+
+# Control of individual brace wrapping cases.
+BreakBeforeBraces: Custom
+BraceWrapping:
+  BeforeCatch: true
+
+# Options for aligning backslashes in escaped newlines.
+AlignEscapedNewlines: Left
+

backends/apple/coreml/.gitignore

@@ -0,0 +1,25 @@
+# Mac OS X
+*.DS_Store
+
+cmake-out/
+
+
+# Xcode
+*.pbxuser
+*.mode1v3
+*.mode2v3
+*.perspectivev3
+*.xcuserstate
+project.xcworkspace/
+xcuserdata/
+
+
+runtime/runner/build/
+runtime/libraries/
+runtime/inmemoryfs/build/
+runtime/test/models/
+third-party/
+
+xcode-build/
+
+*.egg-info

backends/apple/coreml/CMakeLists.txt

@@ -0,0 +1,109 @@
+# Copyright © 2023 Apple Inc. All rights reserved.
+
+cmake_minimum_required(VERSION 3.19)
+
+project(executorch_coreml_backend)
+
+enable_language(CXX)
+enable_language(OBJC)
+
+# Source root directory for executorch.
+if(NOT EXECUTORCH_ROOT)
+  set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../..)
+endif()
+
+# inmemoryfs sources
+set(INMEMORYFS_SOURCES
+  runtime/inmemoryfs/inmemory_filesystem.cpp
+  runtime/inmemoryfs/memory_buffer.cpp
+  runtime/inmemoryfs/memory_stream.cpp
+  runtime/inmemoryfs/reversed_memory_stream.cpp
+)
+
+# kvstore sources
+set(KVSTORE_SOURCES
+  runtime/kvstore/database.cpp
+  runtime/kvstore/sqlite_error.cpp
+  runtime/kvstore/key_value_store.cpp
+  runtime/kvstore/statement.cpp
+)
+
+# delegate sources
+set(DELEGATE_SOURCES
+  runtime/delegate/asset.mm
+  runtime/delegate/backend_delegate.mm
+  runtime/delegate/coreml_backend_delegate.mm
+  runtime/delegate/ETCoreMLAsset.mm
+  runtime/delegate/ETCoreMLAssetManager.mm
+  runtime/delegate/ETCoreMLLogging.mm
+  runtime/delegate/ETCoreMLModel.mm
+  runtime/delegate/ETCoreMLModelManager.mm
+  runtime/delegate/ETCoreMLStrings.mm
+  runtime/delegate/MLModel_Prewarm.mm
+  runtime/delegate/MLMultiArray_Copy.mm
+  runtime/delegate/multiarray.mm
+  runtime/delegate/serde_json.mm
+)
+
+# Define the delegate library
+add_library(coremldelegate)
+target_sources(
+  coremldelegate PRIVATE
+  ${INMEMORYFS_SOURCES}
+  ${KVSTORE_SOURCES}
+  ${DELEGATE_SOURCES}
+)
+target_include_directories(
+  coremldelegate PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/runtime/include
+)
+target_include_directories(
+  coremldelegate PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/runtime/kvstore
+)
+target_include_directories(
+  coremldelegate PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/runtime/inmemoryfs
+)
+target_include_directories(
+  coremldelegate PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/runtime/delegate
+)
+target_include_directories(
+  coremldelegate PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/third-party/nlohmann_json/single_include/nlohmann
+)
+target_include_directories(
+  coremldelegate PRIVATE
+  ${EXECUTORCH_ROOT}/..
+)
+
+target_link_libraries(
+  coremldelegate PRIVATE
+  executorch
+)
+
+target_compile_options(coremldelegate PRIVATE "-fobjc-arc")
+target_compile_options(coremldelegate PRIVATE "-fno-exceptions")
+target_compile_options(coremldelegate PRIVATE "-fno-rtti")
+target_compile_definitions(coremldelegate PRIVATE JSON_NOEXCEPTION=1)
+
+set(
+  TARGET coremldelegate
+  APPEND_STRING PROPERTY COMPILE_FLAGS "-x objective-c++"
+)
+
+set(
+  TARGET coremldelegate
+  APPEND_STRING PROPERTY COMPILE_FLAGS "-Wno-null-character"
+)
+
+set(
+  TARGET coremldelegate
+  APPEND_STRING PROPERTY COMPILE_FLAGS "-Wno-receiver-expr"
+)
+
+set_property(
+  TARGET coremldelegate
+  PROPERTY CXX_STANDARD 17
+)

backends/apple/coreml/README.md

@@ -0,0 +1,59 @@
+# ExecuTorch CoreML Delegate
+
+
+This subtree contains the CoreML Delegate implementation for ExecuTorch.
+CoreML is an optimized framework for running machine learning models on Apple devices. The delegate is the mechanism for leveraging the CoreML framework to accelerate operators when running on Apple devices.
+
+## Layout
+- `compiler/` : Lowers a module to CoreML backend.
+- `scripts/` : Scripts for installing dependencies and running tests.
+- `runtime/`: CoreML delegate runtime implementation.
+    - `inmemoryfs`: InMemory filesystem implementation used to serialize/de-serialize AOT blob.
+    - `kvstore`: Persistent Key-Value store implementation.
+    - `delegate`: Runtime implementation.
+    - `include` : Public headers.
+    - `tests` :  Tests for CoreML delegate.
+    - `workspace` : Xcode workspace for tests.
+- `third-party/`: External dependencies.
+
+## Help & Improvements
+If you have problems or questions or have suggestions for ways to make
+implementation and testing better, please create an issue on [github](https://www.github.com/pytorch/executorch/issues).
+
+## Delegation
+
+For delegating the Program to the **CoreML** backend, the client must be responsible for calling `to_backend` with the **CoreMLBackend** tag.
+
+```python
+import executorch.exir as exir
+import torch
+
+from executorch.exir.backend.backend_api import to_backend
+
+from executorch.backends.coreml.compiler import CoreMLBackend
+
+class LowerableSubModel(torch.nn.Module):
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, x):
+        return torch.sin(x)
+
+# Convert the lowerable module to Edge IR Representation
+to_be_lowered = LowerableSubModel()
+example_input = (torch.ones(1), )
+to_be_lowered_exir_submodule = exir.capture(to_be_lowered, example_input).to_edge()
+
+# Lower to CoreML backend
+lowered_module = to_backend('CoreMLBackend', to_be_lowered_exir_submodule, [])
+```
+
+Currently, the **CoreML** backend delegates the whole module to **CoreML**. If a specific op is not supported by the **CoreML** backend then the `to_backend` call would throw an exception. We will be adding a **CoreML Partitioner** to resolve the issue.
+
+The `to_backend` implementation is a thin wrapper over `coremltools`, `coremltools` is responsible for converting an **ExportedProgram** to a **MLModel**. The converted **MLModel** data is saved, flattened, and returned as bytes to **ExecuTorch**.
+
+## Runtime
+
+To execute a **CoreML** delegated **Program**, the client must link to the `coremldelegate` library. Once linked there are no additional steps required, **ExecuTorch** when running the **Program** would call the **CoreML** runtime to execute the **CoreML** delegated part of the **Program**.
+
+Please follow the instructions described in the [CoreML setup](/backends/apple/coreml/setup.md) to link the `coremldelegate` library.

backends/apple/coreml/compiler/__init__.py

@@ -0,0 +1,9 @@
+# Copyright © 2023 Apple Inc. All rights reserved.
+#
+# Please refer to the license found in the LICENSE file in the root directory of the source tree.
+
+from .coreml_preprocess import CoreMLBackend
+
+__all__ = [
+    CoreMLBackend,
+]

backends/apple/coreml/compiler/coreml_preprocess.py

@@ -0,0 +1,76 @@
+#  Copyright © 2023 Apple Inc. All rights reserved.
+
+# CoreML backend for delegating a EdgeProgram to CoreML.
+
+import json
+import shutil
+import uuid
+
+from pathlib import Path
+
+from typing import final, List
+
+import coremltools as ct
+import executorchcoreml
+
+from executorch.exir.backend.backend_details import (
+    BackendDetails,
+    ExportedProgram,
+    PreprocessResult,
+)
+from executorch.exir.backend.compile_spec_schema import CompileSpec
+
+
+@final
+class CoreMLBackend(BackendDetails):
+    @staticmethod
+    def to_bytes(mlmodel):
+        dir_path = Path("tmp")
+        model_dir_path = dir_path / "lowered_module"
+        Path(model_dir_path).mkdir(parents=True, exist_ok=True)
+        model_path = model_dir_path / "model.mlpackage"
+        mlmodel.save(model_path)
+
+        # save model metdata
+        spec = mlmodel.get_spec()
+        input_names = [input.name for input in spec.description.input]
+        output_names = [output.name for output in spec.description.output]
+        identifier = uuid.uuid4()
+
+        model_metadata = {
+            "inputNames": input_names,
+            "outputNames": output_names,
+            "identifier": str(identifier),
+        }
+
+        # store metadata
+        model_metadata_path = Path(model_dir_path) / "metadata.json"
+        json_object = json.dumps(model_metadata)
+        with open(model_metadata_path, "w") as outfile:
+            outfile.write(json_object)
+
+        # flatten directory contents and convert it to bytes
+        flattened_bytes = executorchcoreml.flatten_directory_contents(
+            str(model_dir_path.resolve())
+        )
+        shutil.rmtree(str(model_dir_path.resolve()))
+        return flattened_bytes
+
+    @classmethod
+    # pyre-ignore
+    def preprocess(
+        cls,
+        edge_program: ExportedProgram,
+        module_compile_spec: List[CompileSpec],
+    ) -> PreprocessResult:
+        mlmodel = ct.convert(
+            model=edge_program,
+            source="pytorch",
+            convert_to="mlprogram",
+            pass_pipeline=ct.PassPipeline.DEFAULT,
+            skip_model_load=True,
+        )
+        flattened_bytes = CoreMLBackend.to_bytes(mlmodel)
+        return PreprocessResult(
+            processed_bytes=flattened_bytes,
+        )