Merge branch 'main' of github.com:pytorch/executorch into hardsigmoid_test

Author: chuntl

.ci/docker/ci_commit_pins/pytorch.txt

@@ -1 +1 @@
-a52b4e22571507abc35c2d47de138497190d2e0a
+0a038cf0cff2d071b7359ac0491fd2ba7798a438

.ci/scripts/test_llama.sh

@@ -118,7 +118,7 @@ EXPORTED_MODEL_NAME="${EXPORTED_MODEL_NAME}.pte"
 echo "Exporting ${EXPORTED_MODEL_NAME}"
 EXPORT_ARGS="-c stories110M.pt -p ${PARAMS} -d ${DTYPE} -n ${EXPORTED_MODEL_NAME}"
 if [[ "${MODE}" == "xnnpack" ]]; then
-  EXPORT_ARGS="${EXPORT_ARGS} --pt2e_quantize xnnpack_dynamic"
+  EXPORT_ARGS="${EXPORT_ARGS} -kv --use_sdpa_with_kv_cache -X -qmode 8da4w -G 128"
 fi
 $PYTHON_EXECUTABLE -m examples.models.llama2.export_llama ${EXPORT_ARGS}
 

.github/workflows/_unittest.yml

@@ -37,6 +37,7 @@ jobs:
         # Setup MacOS dependencies as there is no Docker support on MacOS atm
         PYTHON_EXECUTABLE=python \
         EXECUTORCH_BUILD_PYBIND=ON \
+        EXECUTORCH_BUILD_XNNPACK=ON \
         .ci/scripts/setup-linux.sh "${BUILD_TOOL}"
 
         # Run pytest with coverage

.github/workflows/apple.yml

@@ -100,6 +100,17 @@ jobs:
           zip -r "${RUNNER_TEMP}/artifacts/${FRAMEWORK}-${VERSION}.zip" "${FRAMEWORK}.xcframework"
         ) done
 
+        # Build Debug iOS Frameworks
+        PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \
+        build/build_apple_frameworks.sh --coreml --custom --mps --optimized --portable --quantized --xnnpack --Debug
+
+        # Bundle Debug iOS Frameworks
+        for FRAMEWORK in "${FRAMEWORKS[@]}"; do (
+          cd cmake-out && \
+          mv "${FRAMEWORK}.xcframework" "${FRAMEWORK}_debug.xcframework" && \
+          zip -r "${RUNNER_TEMP}/artifacts/${FRAMEWORK}_debug-${VERSION}.zip" "${FRAMEWORK}_debug.xcframework"
+        ) done
+
         popd
 
   upload-frameworks-ios:
@@ -146,5 +157,6 @@ jobs:
 
           for FILENAME in "${RUNNER_TEMP}"/frameworks-ios/*.zip; do
             [ -e "${FILENAME}" ] || continue
+            shasum -a 256 "${FILENAME}"
             ${AWS_CMD} "${FILENAME}" s3://ossci-ios/executorch/ --acl public-read
           done

.github/workflows/pull.yml

@@ -232,6 +232,7 @@ jobs:
         # build module for executorch.extension.pybindings.portable_lib
         BUILD_TOOL=${{ matrix.build-tool }}
         PYTHON_EXECUTABLE=python \
+        EXECUTORCH_BUILD_XNNPACK=ON \
         EXECUTORCH_BUILD_PYBIND=ON \
         bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}"
 

.github/workflows/trunk.yml

@@ -159,6 +159,10 @@ jobs:
         git config --global user.name "Github Executorch"
         bash examples/arm/setup.sh --i-agree-to-the-contained-eula
 
+        # Increase number of files user can monitor to bypass buck failures.
+        # Hopefully this is high enough for this setup.
+        sudo sysctl fs.inotify.max_user_watches=1048576 # 1024 * 1024
+
         # Test ethos-u delegate examples with run.sh
         source examples/arm/ethos-u-scratch/setup_path.sh
         PYTHON_EXECUTABLE=python bash examples/arm/run.sh examples/arm/ethos-u-scratch/ buck2

.gitmodules

@@ -37,9 +37,6 @@
 [submodule "third-party/flatcc"]
 	path = third-party/flatcc
 	url = https://github.com/dvidelabs/flatcc.git
-[submodule "examples/third-party/llama"]
-	path = examples/third-party/llama
-	url = https://github.com/facebookresearch/llama.git
 [submodule "examples/third-party/fbjni"]
 	path = examples/third-party/fbjni
 	url = https://github.com/facebookincubator/fbjni.git

.swift/coreml_backend_debug/dummy.swift

@@ -56,45 +56,6 @@ if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE Debug)
 endif()
 
-# --- cpuinfo
-set(CPUINFO_SOURCE_DIR "backends/xnnpack/third-party/cpuinfo")
-set(CPUINFO_BUILD_TOOLS
-    OFF
-    CACHE BOOL "")
-set(CPUINFO_BUILD_UNIT_TESTS
-    OFF
-    CACHE BOOL "")
-set(CPUINFO_BUILD_MOCK_TESTS
-    OFF
-    CACHE BOOL "")
-set(CPUINFO_BUILD_BENCHMARKS
-    OFF
-    CACHE BOOL "")
-set(CPUINFO_LIBRARY_TYPE
-    "static"
-    CACHE STRING "")
-set(CPUINFO_LOG_LEVEL
-    "error"
-    CACHE STRING "")
-set(CLOG_SOURCE_DIR "${CPUINFO_SOURCE_DIR}/deps/clog")
-add_subdirectory("${CPUINFO_SOURCE_DIR}")
-
-# --- pthreadpool
-set(PTHREADPOOL_SOURCE_DIR "backends/xnnpack/third-party/pthreadpool")
-set(PTHREADPOOL_BUILD_TESTS
-    OFF
-    CACHE BOOL "")
-set(PTHREADPOOL_BUILD_BENCHMARKS
-    OFF
-    CACHE BOOL "")
-set(PTHREADPOOL_LIBRARY_TYPE
-    "static"
-    CACHE STRING "")
-set(PTHREADPOOL_ALLOW_DEPRECATED_API
-    ON
-    CACHE BOOL "")
-add_subdirectory("${PTHREADPOOL_SOURCE_DIR}")
-
 # ------------------------------ OPTIONS -------------------------------------
 # WARNING: Please don't add example specific options in this CMakeLists.txt.
 # Instead please use `find_package(executorch REQUIRED)` in the example
@@ -211,6 +172,66 @@ option(EXECUTORCH_BUILD_XNNPACK "Build the XNNPACK backend" OFF)
 
 option(EXECUTORCH_BUILD_VULKAN "Build the Vulkan backend" OFF)
 
+#
+# pthreadpool: build pthreadpool library. Disable on unsupported platforms
+#
+cmake_dependent_option(EXECUTORCH_BUILD_PTHREADPOOL "Build pthreadpool library."
+                       ON "NOT EXECUTORCH_BUILD_ARM_BAREMETAL" OFF)
+
+#
+# cpuinfo: build cpuinfo library. Disable on unsupported platforms
+#
+cmake_dependent_option(EXECUTORCH_BUILD_CPUINFO "Build cpuinfo library." ON
+                       "NOT EXECUTORCH_BUILD_ARM_BAREMETAL" OFF)
+
+if(EXECUTORCH_BUILD_CPUINFO)
+  # --- cpuinfo
+  set(CPUINFO_SOURCE_DIR "backends/xnnpack/third-party/cpuinfo")
+  set(CPUINFO_BUILD_TOOLS
+      OFF
+      CACHE BOOL "")
+  set(CPUINFO_BUILD_UNIT_TESTS
+      OFF
+      CACHE BOOL "")
+  set(CPUINFO_BUILD_MOCK_TESTS
+      OFF
+      CACHE BOOL "")
+  set(CPUINFO_BUILD_BENCHMARKS
+      OFF
+      CACHE BOOL "")
+  set(CPUINFO_LIBRARY_TYPE
+      "static"
+      CACHE STRING "")
+  set(CPUINFO_LOG_LEVEL
+      "error"
+      CACHE STRING "")
+  set(CLOG_SOURCE_DIR "${CPUINFO_SOURCE_DIR}/deps/clog")
+  add_subdirectory("${CPUINFO_SOURCE_DIR}")
+endif()
+
+if(EXECUTORCH_BUILD_PTHREADPOOL)
+  # --- pthreadpool
+  set(PTHREADPOOL_SOURCE_DIR "backends/xnnpack/third-party/pthreadpool")
+  set(PTHREADPOOL_BUILD_TESTS
+      OFF
+      CACHE BOOL "")
+  set(PTHREADPOOL_BUILD_BENCHMARKS
+      OFF
+      CACHE BOOL "")
+  set(PTHREADPOOL_LIBRARY_TYPE
+      "static"
+      CACHE STRING "")
+  set(PTHREADPOOL_ALLOW_DEPRECATED_API
+      ON
+      CACHE BOOL "")
+  if(APPLE)
+    set(PTHREADPOOL_SYNC_PRIMITIVE
+       "condvar"
+       CACHE STRING "")
+  endif()
+  add_subdirectory("${PTHREADPOOL_SOURCE_DIR}")
+endif()
+
 if(NOT PYTHON_EXECUTABLE)
   resolve_python_executable()
 endif()
@@ -492,7 +513,9 @@ if(EXECUTORCH_BUILD_PYBIND)
   endif()
 
   if(EXECUTORCH_BUILD_XNNPACK)
-    set(PYBIND_LINK_XNNPACK "xnnpack_backend")
+    # need to explicitly specify XNNPACK here
+    # otherwise uses XNNPACK symbols from libtorch_cpu
+    set(PYBIND_LINK_XNNPACK xnnpack_backend XNNPACK)
   endif()
 
   # find pytorch lib, to allow pybind to take at::Tensor as input/output

.swift/custom_backend_debug/dummy.swift

@@ -9,95 +9,93 @@
 
 import PackageDescription
 
-let url = "https://ossci-ios.s3.amazonaws.com/executorch"
 let version = "0.1.0"
-let coreml_sha256 = "78d853d87be478696e56e658aa4ff17d47ae185a9a6a36316c821fa8b2d3aacd"
-let custom_sha256 = "f059f6716298403dff89a952a70e323c54911be140d05f2467bd5cc61aaefae3"
-let executorch_sha256 = "ba9a0c2b061afaedbc3c5454040a598b1371170bd9d9a30b7163c20e23339841"
-let mps_sha256 = "39542a8671cca1aa627102aa47785d0f6e2dfe9a40e2c22288a755057b00fbfa"
-let optimized_sha256 = "1d84fa16197bb6f0dec01aaa29d2a140c0e14d8e5e92630a7b4dd6f48012506d"
-let portable_sha256 = "4993904f89ecb4476677ff3c072ed1a314a608170f10d364cfd23947851ccbf3"
-let quantized_sha256 = "8d35ee0e7ca77c19782eaea07a1888f576cda679f8a4a5edb03d80ebe858047e"
-let xnnpack_sha256 = "380e5185c4c48ede7cc0d0f0657ffb26df83cd9f55813d78593aea8a93942caf"
-
-struct Framework {
-  let name: String
-  let checksum: String
-  var frameworks: [String] = []
-  var libraries: [String] = []
-
-  func target() -> Target {
-    .binaryTarget(
-      name: name,
-      url: "\(url)/\(name)-\(version).zip",
-      checksum: checksum
-    )
-  }
-
-  func dependencies() -> Target {
-    .target(
-      name: "\(name)_dependencies",
-      dependencies: [.target(name: name)],
-      path: ".swift/\(name)",
-      linkerSettings:
-          frameworks.map { .linkedFramework($0) } +
-          libraries.map { .linkedLibrary($0) }
-    )
-  }
-}
-
-let frameworks = [
-  Framework(
-    name: "coreml_backend",
-    checksum: coreml_sha256,
-    frameworks: [
+let url = "https://ossci-ios.s3.amazonaws.com/executorch/"
+let debug = "_debug"
+let deliverables = [
+  "coreml_backend": [
+    "sha256": "0e5973bbc547e3a39f988f9a7a68b47bda0a6a17b04516fff6957fd527f8cd48",
+    "sha256" + debug: "c63773f0098625f884fecb11b4a5f6318b97d566329fef8b013444829cd7c421",
+    "frameworks": [
       "Accelerate",
       "CoreML",
     ],
-    libraries: [
+    "libraries": [
       "sqlite3",
-    ]
-  ),
-  Framework(
-    name: "custom_backend",
-    checksum: custom_sha256
-  ),
-  Framework(
-    name: "executorch",
-    checksum: executorch_sha256
-  ),
-  Framework(
-    name: "mps_backend",
-    checksum: mps_sha256,
-    frameworks: [
+    ],
+  ],
+  "custom_backend": [
+    "sha256": "c8405e21324262cd6590046096ddeb3ac33a598f88afc817a2f2fdee821da150",
+    "sha256" + debug: "a08a6aa15ddce61a76cd1bf2206d017cc4ac7dcb9ca312ad7750a36814448eaa",
+  ],
+  "executorch": [
+    "sha256": "57269f9b81d56a3d96ece2012e2ece3af24174846abd98de9a3bee07f3b9583d",
+    "sha256" + debug: "66975caf3d9c1238d29945288f23ddb6e07e16386d4dedf429c0f2d81cfbe0cc",
+  ],
+  "mps_backend": [
+    "sha256": "bb7531172252b6535429fbde429de208665f933d0f509982872eada86839e734",
+    "sha256" + debug: "6d41437e40cb794b4b7a0d971931773de263370463b38a014f38e99bd1c5d52b",
+    "frameworks": [
       "Metal",
       "MetalPerformanceShaders",
       "MetalPerformanceShadersGraph",
-    ]
-  ),
-  Framework(
-    name: "optimized_backend",
-    checksum: optimized_sha256
-  ),
-  Framework(
-    name: "portable_backend",
-    checksum: portable_sha256
-  ),
-  Framework(
-    name: "quantized_backend",
-    checksum: quantized_sha256
-  ),
-  Framework(
-    name: "xnnpack_backend",
-    checksum: xnnpack_sha256
-  )
-]
+    ],
+  ],
+  "optimized_backend": [
+    "sha256": "bdab593fb49c9000291dbf691ad578d771883745ed2851f00492e828d089d1ea",
+    "sha256" + debug: "8316ad259d6aafecf2e9abc91a04fc1fa3e0398597e043119b4c29c21e9f2029",
+  ],
+  "portable_backend": [
+    "sha256": "38ebdad7d5cd24ca44cd950d561dcf9a9b883dff626c167bc6f5f28f041b8406",
+    "sha256" + debug: "9e68b3e92e5c920875845f59821ee984b87486d05c1bf8a461b011530e02dd55",
+  ],
+  "quantized_backend": [
+    "sha256": "245a3acbf06c6afe9cfb6b03eddfa015390e582ffdfb76efd23b7c810f080f10",
+    "sha256" + debug: "134d759fe708a4ffbf7efbd25c6020186e1a13abc0dac0a897e2fe13aac3e76a",
+  ],
+  "xnnpack_backend": [
+    "sha256": "a1c9cf8347c17f3e50e45d7f37f64ee040f0a1b0a40fa4748d90b45c4150e3b2",
+    "sha256" + debug: "e92a15c2982630951e5ae5e927d548049db25d89e8b639e8901c5f4650f3a7d0",
+  ],
+].reduce(into: [String: [String: Any]]()) {
+  $0[$1.key] = $1.value
+  $0[$1.key + debug] = $1.value
+}
+.reduce(into: [String: [String: Any]]()) {
+  var newValue = $1.value
+  if $1.key.hasSuffix(debug) {
+    $1.value.forEach { key, value in
+      if key.hasSuffix(debug) {
+        newValue[String(key.dropLast(debug.count))] = value
+      }
+    }
+  }
+  $0[$1.key] = newValue.filter { key, _ in !key.hasSuffix(debug) }
+}
 
 let package = Package(
   name: "executorch",
   platforms: [
     .iOS(.v15),
   ],
-  products: frameworks.map { .library(name: $0.name, targets: ["\($0.name)_dependencies"]) },
-  targets: frameworks.flatMap { [$0.target(), $0.dependencies()] }
+  products: deliverables.keys.map { key in
+    .library(name: key, targets: ["\(key)_dependencies"])
+  }.sorted { $0.name < $1.name },
+  targets: deliverables.flatMap { key, value -> [Target] in
+    [
+      .binaryTarget(
+        name: key,
+        url: "\(url)\(key)-\(version).zip",
+        checksum: value["sha256"] as? String ?? ""
+      ),
+      .target(
+        name: "\(key)_dependencies",
+        dependencies: [.target(name: key)],
+        path: ".swift/\(key)",
+        linkerSettings:
+          (value["frameworks"] as? [String] ?? []).map { .linkedFramework($0) } +
+          (value["libraries"] as? [String] ?? []).map { .linkedLibrary($0) }
+      ),
+    ]
+  }
 )

.swift/executorch_debug/dummy.swift

@@ -55,7 +55,12 @@ def define_node(
         elif node.target == exir_ops.edge.aten.empty.memory_format:
             fill_value = 0
         elif node.target == exir_ops.edge.aten.scalar_tensor.default:
-            fill_value = float(node.args[0])
+            fill_value = cast(float, node.args[0])
+
+        if fill_value == float("-inf"):
+            fill_value = "-inf"
+        elif fill_value == float("inf"):
+            fill_value = "inf"
 
         dtype = MPSDataType.mps_data_type_float32
         if node.kwargs and "dtype" in node.kwargs and node.kwargs["dtype"] is not None:

.swift/mps_backend_debug/dummy.swift

@@ -157,10 +157,9 @@ def define_scalar(
         """
         assert isinstance(val, int) or isinstance(val, float)
 
-        if val in self.tensor_to_id:
-            return self.tensor_to_id[val]
+        id = len(mps_graph.mps_values)
+        self.tensor_to_id[val] = id
 
-        id = self.get_serialized_id(val, mps_graph)
         tensor = torch.tensor(val)
         constant_buffer_size, constant_buffer, mps_data_type = self.get_serialized_data(
             tensor, mps_graph, mps_data_type, id