Integrate sdk (#2314)

Summary:
The PR adds support for logging profiling information and intermediate outputs. The required files are only included if CoreML delegate is built with`EXECUTORCH_BUILD_SDK` option.

- Profiling:  `ETCoreMLModelProfiler` adds profiling support, it uses the `MLComputePlan` API that's currently in beta but unlikely to change. The caveat is that it only works for macOS >= 14.4, iOS >= 17.4.
- Debugging: `ETCoreMLModelDebugger` adds support for getting intermediate tensors. The model spec is updated to include additional intermediate outputs, compiled, and then executed. It has a dependency on `protobuf` as it modifies the specification of the model. The logging part doesn't work yet, it would require ExecuTorch API (`log_tensor_delegate`).
- CLI: Added cli file for parsing CoreML delegate metadata.
- AOT: AOT integration is remaining but most likely it would be done in `coremltools`. `coremltools` will add a `json` file to `mlpackage` that would contain the operation path to debug handle mapping.

![Screenshot 2024-03-07 at 11 04 00 PM](https://github.com/pytorch/executorch/assets/709448/62bbc623-cd53-42cd-bec7-f7afca227f79)

Pull Request resolved: #2314

Reviewed By: cccclai

Differential Revision: D54754857

Pulled By: shoumikhin

fbshipit-source-id: 493da4bfb8f9a4d60d68a219cfed63178d731add

Author: cymbalrush

backends/apple/coreml/.gitignore

@@ -1,7 +1,7 @@
 # Mac OS X
 *.DS_Store
 
-cmake-out/
+*-cmake-out/
 
 
 # Xcode
@@ -18,6 +18,7 @@ runtime/runner/build/
 runtime/libraries/
 runtime/inmemoryfs/build/
 runtime/test/models/
+runtime/sdk/format/
 runtime/include/executorch/
 third-party/
 

backends/apple/coreml/CMakeLists.txt

@@ -38,6 +38,9 @@ set(DELEGATE_SOURCES
   runtime/delegate/coreml_backend_delegate.mm
   runtime/delegate/ETCoreMLAsset.mm
   runtime/delegate/ETCoreMLAssetManager.mm
+  runtime/delegate/ETCoreMLDefaultModelExecutor.mm
+  runtime/delegate/ETCoreMLModelLoader.mm
+  runtime/delegate/ETCoreMLModelCompiler.mm
   runtime/delegate/ETCoreMLLogging.mm
   runtime/delegate/ETCoreMLModel.mm
   runtime/delegate/ETCoreMLModelManager.mm
@@ -54,6 +57,57 @@ set(UTIL_SOURCES
   runtime/util/objc_json_serde.mm
 )
 
+# sdk sources
+set(SDK_SOURCES
+  runtime/sdk/ETCoreMLModelAnalyzer.mm
+  runtime/sdk/ETCoreMLModelStructurePath.mm
+  runtime/sdk/ETCoreMLOperationProfilingInfo.mm
+  runtime/sdk/ETCoreMLModelDebugger.mm
+  runtime/sdk/ETCoreMLModelProfiler.mm
+  runtime/sdk/ETCoreMLPair.mm
+  runtime/sdk/model_package_info.mm
+  runtime/sdk/model_event_logger_impl.mm
+  runtime/sdk/program_path.mm
+)
+
+# protobuf sources
+set(PROTOBUF_SOURCES
+  runtime/sdk/format/ArrayFeatureExtractor.pb.cc
+  runtime/sdk/format/AudioFeaturePrint.pb.cc
+  runtime/sdk/format/BayesianProbitRegressor.pb.cc
+  runtime/sdk/format/CategoricalMapping.pb.cc
+  runtime/sdk/format/ClassConfidenceThresholding.pb.cc
+  runtime/sdk/format/CustomModel.pb.cc
+  runtime/sdk/format/DataStructures.pb.cc
+  runtime/sdk/format/DictVectorizer.pb.cc
+  runtime/sdk/format/FeatureTypes.pb.cc
+  runtime/sdk/format/FeatureVectorizer.pb.cc
+  runtime/sdk/format/Gazetteer.pb.cc
+  runtime/sdk/format/GLMClassifier.pb.cc
+  runtime/sdk/format/GLMRegressor.pb.cc
+  runtime/sdk/format/Identity.pb.cc
+  runtime/sdk/format/Imputer.pb.cc
+  runtime/sdk/format/ItemSimilarityRecommender.pb.cc
+  runtime/sdk/format/LinkedModel.pb.cc
+  runtime/sdk/format/MIL.pb.cc
+  runtime/sdk/format/Model.pb.cc
+  runtime/sdk/format/NearestNeighbors.pb.cc
+  runtime/sdk/format/NeuralNetwork.pb.cc
+  runtime/sdk/format/NonMaximumSuppression.pb.cc
+  runtime/sdk/format/Normalizer.pb.cc
+  runtime/sdk/format/NonMaximumSuppression.pb.cc
+  runtime/sdk/format/OneHotEncoder.pb.cc
+  runtime/sdk/format/Parameters.pb.cc
+  runtime/sdk/format/Scaler.pb.cc
+  runtime/sdk/format/SoundAnalysisPreprocessing.pb.cc
+  runtime/sdk/format/SVM.pb.cc
+  runtime/sdk/format/TextClassifier.pb.cc
+  runtime/sdk/format/TreeEnsemble.pb.cc
+  runtime/sdk/format/VisionFeaturePrint.pb.cc
+  runtime/sdk/format/WordEmbedding.pb.cc
+  runtime/sdk/format/WordTagger.pb.cc
+)
+
 # Define the delegate library
 add_library(coremldelegate)
 target_sources(
@@ -63,6 +117,7 @@ target_sources(
   ${DELEGATE_SOURCES}
   ${UTIL_SOURCES}
 )
+
 target_include_directories(
   coremldelegate PRIVATE
   ${CMAKE_CURRENT_SOURCE_DIR}/runtime/include
@@ -87,7 +142,30 @@ target_include_directories(
   coremldelegate PRIVATE
   ${EXECUTORCH_ROOT}/..
 )
-target_compile_options(executorch PUBLIC -DET_EVENT_TRACER_ENABLED)
+target_link_libraries(
+  coremldelegate PRIVATE
+  executorch
+)
+
+if(EXECUTORCH_BUILD_SDK)
+target_sources(
+  coremldelegate PRIVATE
+  ${SDK_SOURCES}
+  ${PROTOBUF_SOURCES}
+)
+target_include_directories(
+  coremldelegate PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/runtime/sdk
+  ${CMAKE_CURRENT_SOURCE_DIR}/third-party/coremltools/deps/protobuf/src
+)
+add_subdirectory(
+  ${CMAKE_CURRENT_SOURCE_DIR}/third-party/coremltools/deps/protobuf/cmake
+)
+target_link_libraries(
+  coremldelegate PRIVATE
+  libprotobuf-lite
+)
+endif()
 
 find_library(ACCELERATE_FRAMEWORK Accelerate)
 find_library(COREML_FRAMEWORK CoreML)
@@ -105,7 +183,14 @@ target_link_libraries(coremldelegate
 
 target_compile_options(coremldelegate PRIVATE "-fobjc-arc")
 target_compile_options(coremldelegate PRIVATE "-fno-exceptions")
+
+if(EXECUTORCH_BUILD_SDK)
+target_compile_options(executorch PUBLIC -DET_EVENT_TRACER_ENABLED)
+target_compile_options(coremldelegate PRIVATE "-frtti")
+target_compile_options(libprotobuf-lite PRIVATE "-frtti")
+else()
 target_compile_options(coremldelegate PRIVATE "-fno-rtti")
+endif()
 
 set(
   TARGET coremldelegate

backends/apple/coreml/runtime/delegate/ETCoreMLAsset.h

@@ -1,7 +1,7 @@
 //
 // ETCoreMLAsset.h
 //
-// Copyright © 2023 Apple Inc. All rights reserved.
+// Copyright © 2024 Apple Inc. All rights reserved.
 //
 // Please refer to the license found in the LICENSE file in the root directory of the source tree.
 
@@ -11,7 +11,7 @@
 
 NS_ASSUME_NONNULL_BEGIN
 
-/// Represents an asset on the filesystem. An asset is merely a directory with a unique identifier.
+/// Represents an asset on the filesystem.
 @interface ETCoreMLAsset : NSObject
 
 - (instancetype)init NS_UNAVAILABLE;
@@ -21,7 +21,7 @@ NS_ASSUME_NONNULL_BEGIN
 /// Constructs an `ETCoreMLAsset` instance from `ModelAsset`
 ///
 /// @param backingAsset The cpp asset.
-- (instancetype)initWithBackingAsset:(const executorchcoreml::Asset&)backingAsset NS_DESIGNATED_INITIALIZER;
+- (instancetype)initWithBackingAsset:(executorchcoreml::Asset)backingAsset NS_DESIGNATED_INITIALIZER;
 
 /// The unique identifier.
 @property (copy, readonly, nonatomic) NSString* identifier;
@@ -51,7 +51,7 @@ NS_ASSUME_NONNULL_BEGIN
 /// @retval `YES` is the advisory read to all the files succeeded otherwise`NO`.
 - (BOOL)prewarmAndReturnError:(NSError* __autoreleasing*)error;
 
-/// Closes all the opened file handles by the `keepAliveAndReturnError` call.
+/// Closes all the file handles opened by the `keepAliveAndReturnError` call.
 - (void)close;
 
 @end

backends/apple/coreml/runtime/delegate/ETCoreMLAsset.mm

@@ -1,7 +1,7 @@
 //
 // ETCoreMLAsset.mm
 //
-// Copyright © 2023 Apple Inc. All rights reserved.
+// Copyright © 2024 Apple Inc. All rights reserved.
 //
 // Please refer to the license found in the LICENSE file in the root directory of the source tree.
 
@@ -66,14 +66,14 @@ @implementation ETCoreMLAsset {
     os_unfair_lock _lock;
 }
 
-- (instancetype)initWithBackingAsset:(const executorchcoreml::Asset&)backingAsset {
+- (instancetype)initWithBackingAsset:(executorchcoreml::Asset)backingAsset {
     self = [super init];
     if (self) {
-        _backingAsset = backingAsset;
         _isValid = static_cast<BOOL>(is_asset_valid(backingAsset));
         _identifier = @(backingAsset.identifier.c_str());
         _contentURL = [NSURL fileURLWithPath:@(backingAsset.path.c_str())];
         _totalSizeInBytes = backingAsset.total_size_in_bytes();
+        _backingAsset = std::move(backingAsset);
     }
 
     return self;

backends/apple/coreml/runtime/delegate/ETCoreMLAssetManager.h

@@ -1,7 +1,7 @@
 //
 // ETCoreMLAssetManager.h
 //
-// Copyright © 2023 Apple Inc. All rights reserved.
+// Copyright © 2024 Apple Inc. All rights reserved.
 //
 // Please refer to the license found in the LICENSE file in the root directory of the source tree.
 
@@ -117,6 +117,9 @@ NS_ASSUME_NONNULL_BEGIN
 /// contents are deleted asynchronously.
 @property (copy, readonly, nonatomic) NSURL* trashDirectoryURL;
 
+/// The file manager.
+@property (strong, readonly, nonatomic) NSFileManager* fileManager;
+
 @end
 
 NS_ASSUME_NONNULL_END

backends/apple/coreml/runtime/delegate/ETCoreMLAssetManager.mm

@@ -1,21 +1,18 @@
 //
 // ETCoreMLAssetManager.mm
 //
-// Copyright © 2023 Apple Inc. All rights reserved.
+// Copyright © 2024 Apple Inc. All rights reserved.
 //
 // Please refer to the license found in the LICENSE file in the root directory of the source tree.
 
 #import "ETCoreMLAssetManager.h"
-
-#import <iostream>
-#import <sstream>
-
+#import <ETCoreMLAsset.h>
+#import <ETCoreMLLogging.h>
 #import <database.hpp>
+#import <iostream>
 #import <json_key_value_store.hpp>
 #import <serde_json.h>
-
-#import <ETCoreMLAsset.h>
-#import <ETCoreMLLogging.h>
+#import <sstream>
 
 namespace  {
 
@@ -264,7 +261,6 @@ @interface ETCoreMLAssetManager () <NSFileManagerDelegate> {
 
 @property (assign, readwrite, atomic) NSInteger estimatedSizeInBytes;
 @property (copy, readonly, nonatomic) NSURL *assetsDirectoryURL;
-@property (strong, readonly, nonatomic) NSFileManager *fileManager;
 @property (strong, readonly, nonatomic) dispatch_queue_t syncQueue;
 @property (strong, readonly, nonatomic) dispatch_queue_t trashQueue;
 @property (strong, readonly, nonatomic) NSMapTable<NSString *, ETCoreMLAsset *> *assetsInUseMap;

backends/apple/coreml/runtime/delegate/ETCoreMLComputeUnits.h

@@ -0,0 +1,14 @@
+//
+// ETCoreMLComputeUnits.h
+//
+// Copyright © 2024 Apple Inc. All rights reserved.
+//
+// Please refer to the license found in the LICENSE file in the root directory of the source tree.
+
+/// An enum representing the compute options on the system.
+typedef NS_OPTIONS(NSUInteger, ETCoreMLComputeUnits) {
+    ETCoreMLComputeUnitUnknown = 0, // Unknown compute unit.
+    ETCoreMLComputeUnitCPU = 1 << 0, // Represents the CPU compute unit.
+    ETCoreMLComputeUnitGPU = 1 << 1, // Represents the GPU compute unit.
+    ETCoreMLComputeUnitNeuralEngine = 1 << 2 // Represents the NeuralEngine compute unit.
+};

backends/apple/coreml/runtime/delegate/ETCoreMLDefaultModelExecutor.h

@@ -0,0 +1,32 @@
+//
+//  ETCoreMLDefaultModelExecutor.h
+//  executorchcoreml_tests
+//
+//  Created by Gyan Sinha on 2/25/24.
+//
+
+#import <CoreML/CoreML.h>
+
+#import <ETCoreMLModelExecutor.h>
+
+@class ETCoreMLModel;
+
+NS_ASSUME_NONNULL_BEGIN
+/// The default model executor, the executor ignores logging options.
+__attribute__((objc_subclassing_restricted)) @interface ETCoreMLDefaultModelExecutor : NSObject<ETCoreMLModelExecutor>
+
++ (instancetype)new NS_UNAVAILABLE;
+
+- (instancetype)init NS_UNAVAILABLE;
+
+/// Constructs an `ETCoreMLDefaultModelExecutor` from the given model.
+///
+/// @param model The model.
+- (instancetype)initWithModel:(ETCoreMLModel*)model NS_DESIGNATED_INITIALIZER;
+
+/// The model.
+@property (readonly, strong, nonatomic) ETCoreMLModel* model;
+
+@end
+
+NS_ASSUME_NONNULL_END

backends/apple/coreml/runtime/delegate/ETCoreMLDefaultModelExecutor.mm

@@ -0,0 +1,55 @@
+//
+//  ETCoreMLDefaultModelExecutor.m
+//  executorchcoreml_tests
+//
+//  Created by Gyan Sinha on 2/25/24.
+//
+
+#import <ETCoreMLAsset.h>
+#import <ETCoreMLDefaultModelExecutor.h>
+#import <ETCoreMLLogging.h>
+#import <ETCoreMLModel.h>
+
+@implementation ETCoreMLDefaultModelExecutor
+
+- (instancetype)initWithModel:(ETCoreMLModel *)model {
+    self = [super init];
+    if (self) {
+        _model = model;
+    }
+    
+    return self;
+}
+
+- (nullable NSArray<MLMultiArray *> *)executeModelWithInputs:(id<MLFeatureProvider>)inputs
+                                           predictionOptions:(MLPredictionOptions *)predictionOptions
+                                              loggingOptions:(const executorchcoreml::ModelLoggingOptions& __unused)loggingOptions
+                                                 eventLogger:(const executorchcoreml::ModelEventLogger* _Nullable __unused)eventLogger
+                                                       error:(NSError * __autoreleasing *)error {
+    id<MLFeatureProvider> outputs = [self.model.mlModel predictionFromFeatures:inputs
+                                                                       options:predictionOptions
+                                                                         error:error];
+    if (!outputs) {
+        return nil;
+    }
+    
+    NSOrderedSet<NSString*>* orderedOutputNames = self.model.orderedOutputNames;
+    NSMutableArray<MLMultiArray *> *result = [NSMutableArray arrayWithCapacity:orderedOutputNames.count];
+    for (NSString *outputName in orderedOutputNames) {
+        MLFeatureValue *featureValue = [outputs featureValueForName:outputName];
+        if (!featureValue.multiArrayValue) {
+            ETCoreMLLogErrorAndSetNSError(error,
+                                          ETCoreMLErrorBrokenModel,
+                                          "%@: Model is broken, expected multiarray for output=%@.",
+                                          NSStringFromClass(self.class),
+                                          outputName);
+            return nil;
+        }
+        
+        [result addObject:featureValue.multiArrayValue];
+    }
+    
+    return result;
+}
+
+@end

backends/apple/coreml/runtime/delegate/ETCoreMLLogging.h

@@ -1,7 +1,7 @@
 //
 // ETCoreMLLogging.h
 //
-// Copyright © 2023 Apple Inc. All rights reserved.
+// Copyright © 2024 Apple Inc. All rights reserved.
 //
 // Please refer to the license found in the LICENSE file in the root directory of the source tree.
 
@@ -22,7 +22,8 @@ typedef NS_ERROR_ENUM(ETCoreMLErrorDomain, ETCoreMLError) {
     ETCoreMLErrorBrokenModel, // CoreML model doesn't match the input and output specification.
     ETCoreMLErrorCompilationFailed, // CoreML model failed to compile.
     ETCoreMLErrorModelSaveFailed, // Failed to save CoreML model to disk.
-    ETCoreMLErrorModelCacheCreationFailed // Failed to create model cache.
+    ETCoreMLErrorModelCacheCreationFailed, // Failed to create model cache.
+    ETCoreMLErrorInternalError, // Internal error.
 };
 
 @interface ETCoreMLErrorUtils : NSObject

backends/apple/coreml/runtime/delegate/ETCoreMLLogging.mm

@@ -1,7 +1,7 @@
 //
 // ETCoreMLLogging.mm
 //
-// Copyright © 2023 Apple Inc. All rights reserved.
+// Copyright © 2024 Apple Inc. All rights reserved.
 //
 // Please refer to the license found in the LICENSE file in the root directory of the source tree.
 

backends/apple/coreml/runtime/delegate/ETCoreMLModel.h

@@ -1,7 +1,7 @@
 //
 // ETCoreMLModel.h
 //
-// Copyright © 2023 Apple Inc. All rights reserved.
+// Copyright © 2024 Apple Inc. All rights reserved.
 //
 // Please refer to the license found in the LICENSE file in the root directory of the source tree.