Update on "Reduce memory requirement on export_llama tests with no params"

For some reason, after the previous PR in the stack, test_export_llama_lib was ooming on gh actions CI and I couldn't really figure out why, since I profiled the running memory on the test both before and after they were the same. This addresses fixes the ci oom, and I've been meaning to do this anyway since if we are loading a transformer without params specified, we likely just want to test some basic functionality, so a 1 layer makes more sense than a 8 layer default. Have made sure code elsewhere is not relying on this 8 layer default atm.

Differential Revision: [D75498713](https://our.internmc.facebook.com/intern/diff/D75498713)

[ghstack-poisoned]

Author: jackzhxng

.ci/scripts/wheel/test_linux.py

@@ -14,6 +14,10 @@
             test_base.ModelTest(
                 model=Model.Mv3,
                 backend=Backend.XnnpackQuantizationDelegation,
-            )
+            ),
+            test_base.ModelTest(
+                model=Model.Mv3,
+                backend=Backend.CoreMlExportOnly,
+            ),
         ]
     )

.ci/scripts/wheel/test_macos.py

@@ -17,7 +17,7 @@
             ),
             test_base.ModelTest(
                 model=Model.Mv3,
-                backend=Backend.CoreMlTest,
+                backend=Backend.CoreMlExportAndTest,
             ),
         ]
     )

.github/workflows/apple-perf.yml

@@ -386,8 +386,7 @@ jobs:
         echo "::endgroup::"
 
         echo "::group::Build ExecuTorch iOS frameworks"
-        PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \
-          scripts/build_apple_frameworks.sh --Release --Debug --coreml --custom --mps --optimized --portable --quantized --xnnpack
+        PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output scripts/build_apple_frameworks.sh
         echo "::endgroup::"
 
         # NB: Although exported models can be copied to this directory and bundled together with the

.github/workflows/apple.yml

@@ -156,6 +156,7 @@ jobs:
           "kernels_optimized"
           "kernels_portable"
           "kernels_quantized"
+          "threadpool"
         )
 
         .ci/scripts/setup-conda.sh
@@ -173,8 +174,7 @@ jobs:
         backends/apple/mps/install_requirements.sh
 
         # Build iOS Frameworks
-        PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \
-        scripts/build_apple_frameworks.sh --Release --Debug --coreml --custom --mps --optimized --portable --quantized --xnnpack
+        PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output scripts/build_apple_frameworks.sh
 
         # Bundle iOS Frameworks
         for FRAMEWORK in "${FRAMEWORKS[@]}"; do (
@@ -314,8 +314,7 @@ jobs:
         echo "::endgroup::"
 
         echo "::group::Build ExecuTorch iOS frameworks"
-        PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \
-          scripts/build_apple_frameworks.sh --Release --Debug --coreml --custom --mps --optimized --portable --quantized --xnnpack
+        PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output scripts/build_apple_frameworks.sh
         echo "::endgroup::"
 
         echo "::group::Build ExecuTorch benchmark app"

.github/workflows/build-presets.yml

@@ -65,6 +65,7 @@ jobs:
 
         ./install_requirements.sh > /dev/null
         cmake --preset ${{ matrix.preset }}
+        cmake --build cmake-out -j$(( $(nproc) - 1 ))
 
   windows:
     uses: pytorch/test-infra/.github/workflows/windows_job.yml@main

.lintrunner.toml

@@ -386,19 +386,14 @@ exclude_patterns = [
     "third-party/**",
     # TODO: remove exceptions as we migrate
     # backends
-    "backends/arm/quantizer/**",
-    "backends/arm/test/ops/**",
     "backends/vulkan/quantizer/**",
     "backends/vulkan/test/**",
-    "backends/qualcomm/quantizer/**",
-    "examples/qualcomm/**",
     "backends/xnnpack/quantizer/**",
     "backends/xnnpack/test/**",
     "exir/tests/test_passes.py",
     "extension/llm/export/builder.py",
     "extension/llm/export/quantizer_lib.py",
     "exir/tests/test_memory_planning.py",
-    "backends/transforms/duplicate_dynamic_quant_chain.py",
     "exir/backend/test/demos/test_xnnpack_qnnpack.py",
 ]
 

CMakeLists.txt

@@ -143,6 +143,11 @@ else()
   set(CMAKE_CXX_FLAGS_RELEASE "-O2 ${CMAKE_CXX_FLAGS_RELEASE}")
 endif()
 
+if(EXECUTORCH_BUILD_TESTS)
+  set(EXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR ON)
+  include(CTest)
+endif()
+
 add_subdirectory(third-party)
 
 if(EXECUTORCH_BUILD_EXTENSION_TRAINING)
@@ -254,11 +259,6 @@ if(EXECUTORCH_BUILD_PTHREADPOOL)
   )
 endif()
 
-if(EXECUTORCH_BUILD_TESTS)
-  set(EXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR ON)
-  include(CTest)
-endif()
-
 # TODO(dbort): Fix these warnings and remove this flag.
 set(_common_compile_options -Wno-deprecated-declarations -fPIC)
 
@@ -472,11 +472,6 @@ install(
 )
 install(FILES tools/cmake/executorch-config.cmake DESTINATION lib/cmake/ExecuTorch)
 
-# Add googletest if any test targets should be built
-if(BUILD_TESTING)
-  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/third-party/googletest)
-endif()
-
 if(EXECUTORCH_BUILD_ARM_BAREMETAL)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/backends/arm)
 endif()
@@ -565,8 +560,6 @@ if(EXECUTORCH_BUILD_PTHREADPOOL AND EXECUTORCH_BUILD_CPUINFO)
 endif()
 
 if(EXECUTORCH_BUILD_PYBIND)
-  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/third-party/pybind11)
-
   if(NOT EXECUTORCH_BUILD_EXTENSION_DATA_LOADER)
     add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/data_loader)
   endif()
@@ -602,7 +595,7 @@ if(EXECUTORCH_BUILD_PYBIND)
     list(APPEND _dep_libs portable_ops_lib)
   endif()
 
-  if(EXECUTORCH_BUILD_COREML)
+  if(EXECUTORCH_BUILD_COREML AND APPLE)
     list(APPEND _dep_libs coremldelegate)
   endif()
 
@@ -701,7 +694,7 @@ if(EXECUTORCH_BUILD_EXECUTOR_RUNNER)
     list(APPEND _executor_runner_libs etdump flatccrt)
   endif()
 
-  if(EXECUTORCH_BUILD_COREML)
+  if(EXECUTORCH_BUILD_COREML AND APPLE)
     list(APPEND _executor_runner_libs coremldelegate)
   endif()
 

Package.swift

@@ -19,8 +19,30 @@
 
 import PackageDescription
 
-let debug = "_debug"
-let deliverables = [
+let debug_suffix = "_debug"
+let dependencies_suffix = "_with_dependencies"
+
+func deliverables(_ dict: [String: [String: Any]]) -> [String: [String: Any]] {
+  dict
+    .reduce(into: [String: [String: Any]]()) { result, pair in
+      let (key, value) = pair
+      result[key] = value
+      result[key + debug_suffix] = value
+    }
+    .reduce(into: [String: [String: Any]]()) { result, pair in
+      let (key, value) = pair
+      var newValue = value
+      if key.hasSuffix(debug_suffix) {
+        for (k, v) in value where k.hasSuffix(debug_suffix) {
+          let trimmed = String(k.dropLast(debug_suffix.count))
+          newValue[trimmed] = v
+        }
+      }
+      result[key] = newValue.filter { !$0.key.hasSuffix(debug_suffix) }
+    }
+}
+
+let products = deliverables([
   "backend_coreml": [
     "frameworks": [
       "Accelerate",
@@ -37,25 +59,62 @@ let deliverables = [
       "MetalPerformanceShadersGraph",
     ],
   ],
-  "backend_xnnpack": [:],
-  "executorch": [:],
+  "backend_xnnpack": [
+    "targets": [
+      "threadpool",
+    ],
+  ],
+  "executorch": [
+    "libraries": [
+      "c++",
+    ],
+  ],
   "kernels_custom": [:],
-  "kernels_optimized": [:],
+  "kernels_optimized": [
+    "frameworks": [
+      "Accelerate",
+    ],
+    "targets": [
+      "threadpool",
+    ],
+  ],
   "kernels_portable": [:],
   "kernels_quantized": [:],
-].reduce(into: [String: [String: Any]]()) {
-  $0[$1.key] = $1.value
-  $0[$1.key + debug] = $1.value
-}.reduce(into: [String: [String: Any]]()) {
-  var newValue = $1.value
-  if $1.key.hasSuffix(debug) {
-    $1.value.forEach { key, value in
-      if key.hasSuffix(debug) {
-        newValue[String(key.dropLast(debug.count))] = value
-      }
-    }
-  }
-  $0[$1.key] = newValue.filter { key, _ in !key.hasSuffix(debug) }
+])
+
+let targets = deliverables([
+  "threadpool": [:],
+])
+
+let packageProducts: [Product] = products.keys.map { key -> Product in
+  .library(name: key, targets: ["\(key)\(dependencies_suffix)"])
+}.sorted { $0.name < $1.name }
+
+var packageTargets: [Target] = []
+
+for (key, value) in targets {
+  packageTargets.append(.binaryTarget(
+    name: key,
+    path: "cmake-out/\(key).xcframework"
+  ))
+}
+
+for (key, value) in products {
+  packageTargets.append(.binaryTarget(
+    name: key,
+    path: "cmake-out/\(key).xcframework"
+  ))
+  let target: Target = .target(
+    name: "\(key)\(dependencies_suffix)",
+    dependencies: ([key] + (value["targets"] as? [String] ?? []).map {
+      key.hasSuffix(debug_suffix) ? $0 + debug_suffix : $0
+    }).map { .target(name: $0) },
+    path: ".Package.swift/\(key)",
+    linkerSettings:
+      (value["frameworks"] as? [String] ?? []).map { .linkedFramework($0) } +
+      (value["libraries"] as? [String] ?? []).map { .linkedLibrary($0) }
+  )
+  packageTargets.append(target)
 }
 
 let package = Package(
@@ -64,42 +123,22 @@ let package = Package(
     .iOS(.v17),
     .macOS(.v10_15),
   ],
-  products: deliverables.keys.map { key in
-    .library(name: key, targets: ["\(key)_dependencies"])
-  }.sorted { $0.name < $1.name },
-  targets: deliverables.flatMap { key, value -> [Target] in
-    [
-      .binaryTarget(
-        name: key,
-        path: "cmake-out/\(key).xcframework"
-      ),
-      .target(
-        name: "\(key)_dependencies",
-        dependencies: [.target(name: key)],
-        path: ".Package.swift/\(key)",
-        linkerSettings: [
-          .linkedLibrary("c++")
-        ] +
-          (value["frameworks"] as? [String] ?? []).map { .linkedFramework($0) } +
-          (value["libraries"] as? [String] ?? []).map { .linkedLibrary($0) }
-      ),
-    ]
-  } + [
+  products: packageProducts,
+  targets: packageTargets + [
     .testTarget(
       name: "tests",
       dependencies: [
-        .target(name: "executorch_debug"),
-        .target(name: "kernels_portable"),
+        .target(name: "executorch\(debug_suffix)"),
+        .target(name: "kernels_optimized\(dependencies_suffix)"),
       ],
       path: "extension/apple/ExecuTorch/__tests__",
       resources: [
-        .copy("resources/add.pte")
+        .copy("resources/add.pte"),
       ],
       linkerSettings: [
-        .linkedLibrary("c++"),
         .unsafeFlags([
           "-Xlinker", "-force_load",
-          "-Xlinker", "cmake-out/kernels_portable.xcframework/macos-arm64/libkernels_portable_macos.a",
+          "-Xlinker", "cmake-out/kernels_optimized.xcframework/macos-arm64/libkernels_optimized_macos.a",
         ])
       ]
     )