CoreML delegate scripts cleanup (#1025)

Summary:
- Removed `install_requirements_internal script`. The script was cloning a private coremltools repo, executorch changes are now in the public coremltools repo.
- Xcode projects were referring to the parent directory of executorch, the `include` path becomes invalid if the directory name is not executorch. Updated the scripts to copy the required executorch headers.
- Updated coreml-delegate-test to use the build_all script. The script builds and runs the tests and the executor.

Pull Request resolved: #1025

Reviewed By: cccclai

Differential Revision: D50479135

Pulled By: shoumikhin

fbshipit-source-id: f9908e58bcd9ae8882c051afb8adabca61642fef

Author: cymbalrush

.github/workflows/pull.yml

@@ -117,7 +117,6 @@ jobs:
   test-coreml-delegate:
     name: test-coreml-delegate
     uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
-    secrets: inherit
     strategy:
       matrix:
         include:
@@ -128,7 +127,6 @@ jobs:
       submodules: 'true'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 60
-      secrets-env: TOKEN_COREML_PRIVATE_REPO
       script: |
         WORKSPACE=$(pwd)
         pushd "${WORKSPACE}/pytorch/executorch"
@@ -137,9 +135,7 @@ jobs:
         PYTHON_EXECUTABLE=python bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}"
 
         # Build and test coreml delegate
-        PYTHON_EXECUTABLE=python sh backends/apple/coreml/scripts/install_requirements_internal.sh
-        PYTHON_EXECUTABLE=python sh backends/apple/coreml/scripts/build_tests.sh
-        PYTHON_EXECUTABLE=python sh backends/apple/coreml/scripts/run_tests.sh
+        PYTHON_EXECUTABLE=python bash backends/apple/coreml/scripts/build_all.sh
         popd
 
   unittest:

backends/apple/coreml/.gitignore

@@ -18,6 +18,7 @@ runtime/runner/build/
 runtime/libraries/
 runtime/inmemoryfs/build/
 runtime/test/models/
+runtime/include/executorch/
 third-party/
 
 xcode-build/

backends/apple/coreml/runtime/workspace/executorchcoreml.xcodeproj/project.pbxproj

@@ -529,7 +529,6 @@
 				DEVELOPMENT_TEAM = "";
 				GENERATE_INFOPLIST_FILE = YES;
 				HEADER_SEARCH_PATHS = (
-					"$(SRCROOT)/../../../../../../",
 					"$(SRCROOT)/../delegate",
 					"$(SRCROOT)/../kvstore",
 					"$(SRCROOT)/../inmemoryfs",
@@ -558,7 +557,6 @@
 				DEVELOPMENT_TEAM = "";
 				GENERATE_INFOPLIST_FILE = YES;
 				HEADER_SEARCH_PATHS = (
-					"$(SRCROOT)/../../../../../../",
 					"$(SRCROOT)/../delegate",
 					"$(SRCROOT)/../kvstore",
 					"$(SRCROOT)/../inmemoryfs",

backends/apple/coreml/runtime/workspace/executorchcoreml.xcodeproj/xcshareddata/xcschemes/executorchcoreml_runner.xcscheme

@@ -12,53 +12,18 @@ SCRIPT_DIR_PATH="$(
     pwd -P
 )"
 
-EXECUTORCH_ROOT_PATH="$SCRIPT_DIR_PATH/../../../../"
+EXECUTORCH_ROOT_PATH=$(realpath "$SCRIPT_DIR_PATH/../../../../")
 COREML_DIR_PATH="$EXECUTORCH_ROOT_PATH/backends/apple/coreml"
 COREML_EXAMPLES_DIR_PATH="$EXECUTORCH_ROOT_PATH/examples/apple/coreml"
-TEST_ENV_NAME=".coreml-test-env"
 TEST_MODEL_PATH="$COREML_DIR_PATH/runtime/test/models/mv3_coreml_all.pte"
 
 red=`tput setaf 1`
 green=`tput setaf 2`
 
 cd "$EXECUTORCH_ROOT_PATH"
 
-echo "${green}ExecuTorch: Updating git submmodules"
-git submodule sync
-git submodule update --init
-
-STATUS=$?
-if [ $STATUS -ne 0 ]; then
-    echo "${red}ExecuTorch: Failed to update git submodules."
-    exit 1
-fi
-
-echo "${green}ExecuTorch: Creating virtual environment."
-
-function cleanup {
-    echo "${green}ExecuTorch: Deactivating virtual environment"
-    deactivate
-    rm -rf "$TEST_ENV_NAME"
-}
-
-python3 -m venv "$TEST_ENV_NAME"
-source "$TEST_ENV_NAME/bin/activate"
-trap cleanup EXIT
-
-STATUS=$?
-if [ ${STATUS} -ne 0 ]; then
-    echo "${red}ExecuTorch: Failed to create virtual environment."
-    exit 1
-fi
-
-FLATBUFFERS_PATH="$EXECUTORCH_ROOT_PATH/third-party/flatbuffers/cmake-out"
-export PATH="${FLATBUFFERS_PATH}:${PATH}"
-
-echo "${green}ExecuTorch: Installing ExecuTorch requirements"
-source "$EXECUTORCH_ROOT_PATH/install_requirements.sh"
-
 echo "${green}ExecuTorch: Installing CoreML requirements"
-source "$COREML_DIR_PATH/scripts/install_requirements_internal.sh"
+source "$COREML_DIR_PATH/scripts/install_requirements.sh"
 STATUS=$?
 if [ ${STATUS} -ne 0 ]; then
     echo "${red}ExecuTorch: Failed to install dependencies required by CoreML backend."

backends/apple/coreml/scripts/build_all.sh

@@ -10,11 +10,14 @@ SCRIPT_DIR_PATH="$(
     pwd -P
 )"
 
-EXECUTORCH_ROOT_PATH="$SCRIPT_DIR_PATH/../../../../"
+EXECUTORCH_ROOT_PATH=$(realpath "$SCRIPT_DIR_PATH/../../../../")
 COREML_DIR_PATH="$EXECUTORCH_ROOT_PATH/backends/apple/coreml"
 IOS_TOOLCHAIN_PATH="$COREML_DIR_PATH/third-party/ios-cmake/ios.toolchain.cmake"
 CMAKE_BUILD_DIR_PATH="$COREML_DIR_PATH/cmake-out"
 LIBRARIES_DIR_PATH="$COREML_DIR_PATH/runtime/libraries"
+EXECUTORCH_INCLUDE_DIR_PATH="$COREML_DIR_PATH/runtime/include/executorch"
+
+cd "$EXECUTORCH_ROOT_PATH"
 
 echo "ExecuTorch: Building Tests"
 
@@ -39,4 +42,12 @@ echo "ExecuTorch: Copying libraries"
 mkdir "$LIBRARIES_DIR_PATH"
 cp -f "$CMAKE_BUILD_DIR_PATH/libexecutorch.a" "$LIBRARIES_DIR_PATH"
 
+#Copy ExecuTorch headers
+echo "ExecuTorch: Copying headers"
+rm -rf "$EXECUTORCH_INCLUDE_DIR_PATH"
+mkdir -p "$EXECUTORCH_INCLUDE_DIR_PATH"
+find extension \( -name "*.h" -o -name "*.hpp" \) -exec rsync -R '{}' "$EXECUTORCH_INCLUDE_DIR_PATH" \;
+find runtime \( -name "*.h" -o -name "*.hpp" \) -exec rsync -R '{}' "$EXECUTORCH_INCLUDE_DIR_PATH" \;
+find util \( -name "*.h" -o -name "*.hpp" \) -exec rsync -R '{}' "$EXECUTORCH_INCLUDE_DIR_PATH" \;
+
 source "$SCRIPT_DIR_PATH/generate_test_models.sh"

backends/apple/coreml/scripts/build_tests.sh

@@ -10,9 +10,11 @@ SCRIPT_DIR_PATH="$(
     pwd -P
 )"
 
-EXECUTORCH_ROOT_PATH="$SCRIPT_DIR_PATH/../../../../"
+EXECUTORCH_ROOT_PATH=$(realpath "$SCRIPT_DIR_PATH/../../../../")
 COREML_DIR_PATH="$EXECUTORCH_ROOT_PATH/backends/apple/coreml"
 
+cd "$EXECUTORCH_ROOT_PATH"
+
 mkdir "$COREML_DIR_PATH/runtime/test/models/"
 #Generate models
 echo "Executorch: Generating test models"

backends/apple/coreml/scripts/generate_test_models.sh

@@ -13,9 +13,11 @@ SCRIPT_DIR_PATH="$(
 red=`tput setaf 1`
 green=`tput setaf 2`
 
-EXECUTORCH_ROOT_PATH="$SCRIPT_DIR_PATH/../../../../"
+EXECUTORCH_ROOT_PATH=$(realpath "$SCRIPT_DIR_PATH/../../../../")
 COREML_DIR_PATH="$EXECUTORCH_ROOT_PATH/backends/apple/coreml"
 
+cd "$EXECUTORCH_ROOT_PATH"
+
 # clone and install coremltools
 if [ -d "/tmp/coremltools" ]; then
     rm -rf "/tmp/coremltools"
@@ -85,4 +87,4 @@ STATUS=$?
 if [ $STATUS -ne 0 ]; then
     echo "${red}ExecuTorch: Failed to install inmemoryfs extension."
     exit 1
-fi
+fi

backends/apple/coreml/scripts/install_requirements.sh

@@ -10,9 +10,11 @@ SCRIPT_DIR_PATH="$(
     pwd -P
 )"
 
-EXECUTORCH_ROOT_PATH="$SCRIPT_DIR_PATH/../../../../"
+EXECUTORCH_ROOT_PATH=$(realpath "$SCRIPT_DIR_PATH/../../../../")
 COREML_DIR_PATH="$EXECUTORCH_ROOT_PATH/backends/apple/coreml"
 
+cd "$EXECUTORCH_ROOT_PATH"
+
 # Run the test
 echo "ExecuTorch: Running executorchcoreml_tests"
 XCODE_WORKSPACE_DIR_PATH="$COREML_DIR_PATH/runtime/workspace"