Update base for Update on "[ET-VK][Ez] Fix Validation Layer warnings about wrong image layout"

## Context

Currently, when executing a `ComputeGraph` with prepacked tensors with [Vulkan Validation Layers](https://github.com/KhronosGroup/Vulkan-ValidationLayers) turned on, the following Validation Errors can be observed. Note that Validation Layers can be turned on by running Vulkan binaries on Mac with the `vkconfig` app opened.

```
UNASSIGNED-CoreValidation-DrawState-InvalidImageLayout(ERROR / SPEC): msgNum: 1303270965 - Validation Error: [ UNASSIGNED-CoreValidation-DrawState-InvalidImageLayout ] Object 0: handle = 0x7fb76dbbf988, type = VK_OBJECT_TYPE_COMMAND_BUFFER; | MessageID = 0x4dae5635 | vkQueueSubmit(): pSubmits[0].pCommandBuffers[0] command buffer VkCommandBuffer 0x7fb76dbbf988[] expects VkImage 0xd79c8a0000000f09[] (subresource: aspectMask 0x1 array layer 0, mip level 0) to be in layout VK_IMAGE_LAYOUT_GENERAL--instead, current layout is VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL.
    Objects: 1
        [0] 0x7fb76dbbf988, type: 6, name: NULL
```

The reason for this is that prepacked textures are written to with `WRITE` memory access during packing, which means they will be in the `VK_IMAGE_LAYOUT_GENERAL` layout. However, they will subsequently be read from during `graph.execute()`, meaning the texture will have transitioned to `VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL`, but will be bound using the `VK_IMAGE_LAYOUT_GENERAL` layout. Subsequent calls to `execute()` will therefore see that the prepacked texture has been bound with the wrong layout, since after the first graph execution the texture will have the `VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL` layout.

The solution is to submit a no-op shader dispatch during prepacking to trigger a transition to the `READ_ONLY_OPTIMAL` layout.

Differential Revision: [D55772003](https://our.internmc.facebook.com/intern/diff/D55772003/)

[ghstack-poisoned]

Author: SS-JIA

.ci/scripts/test_llama.sh

@@ -118,7 +118,7 @@ EXPORTED_MODEL_NAME="${EXPORTED_MODEL_NAME}.pte"
 echo "Exporting ${EXPORTED_MODEL_NAME}"
 EXPORT_ARGS="-c stories110M.pt -p ${PARAMS} -d ${DTYPE} -n ${EXPORTED_MODEL_NAME}"
 if [[ "${MODE}" == "xnnpack" ]]; then
-  EXPORT_ARGS="${EXPORT_ARGS} --pt2e_quantize xnnpack_dynamic"
+  EXPORT_ARGS="${EXPORT_ARGS} -kv --use_sdpa_with_kv_cache -X -qmode 8da4w -G 128"
 fi
 $PYTHON_EXECUTABLE -m examples.models.llama2.export_llama ${EXPORT_ARGS}
 

.github/workflows/apple.yml

@@ -157,5 +157,6 @@ jobs:
 
           for FILENAME in "${RUNNER_TEMP}"/frameworks-ios/*.zip; do
             [ -e "${FILENAME}" ] || continue
+            shasum -a 256 "${FILENAME}"
             ${AWS_CMD} "${FILENAME}" s3://ossci-ios/executorch/ --acl public-read
           done

.gitignore

@@ -2,6 +2,7 @@
 buck-out/
 cmake-out/
 cmake-android-out/
+cmake-out-android/
 cmake-ios-out/
 ethos-u-scratch/
 executorch.egg-info

Package.swift

@@ -9,95 +9,93 @@
 
 import PackageDescription
 
-let url = "https://ossci-ios.s3.amazonaws.com/executorch"
 let version = "0.1.0"
-let coreml_sha256 = "78d853d87be478696e56e658aa4ff17d47ae185a9a6a36316c821fa8b2d3aacd"
-let custom_sha256 = "f059f6716298403dff89a952a70e323c54911be140d05f2467bd5cc61aaefae3"
-let executorch_sha256 = "ba9a0c2b061afaedbc3c5454040a598b1371170bd9d9a30b7163c20e23339841"
-let mps_sha256 = "39542a8671cca1aa627102aa47785d0f6e2dfe9a40e2c22288a755057b00fbfa"
-let optimized_sha256 = "1d84fa16197bb6f0dec01aaa29d2a140c0e14d8e5e92630a7b4dd6f48012506d"
-let portable_sha256 = "4993904f89ecb4476677ff3c072ed1a314a608170f10d364cfd23947851ccbf3"
-let quantized_sha256 = "8d35ee0e7ca77c19782eaea07a1888f576cda679f8a4a5edb03d80ebe858047e"
-let xnnpack_sha256 = "380e5185c4c48ede7cc0d0f0657ffb26df83cd9f55813d78593aea8a93942caf"
-
-struct Framework {
-  let name: String
-  let checksum: String
-  var frameworks: [String] = []
-  var libraries: [String] = []
-
-  func target() -> Target {
-    .binaryTarget(
-      name: name,
-      url: "\(url)/\(name)-\(version).zip",
-      checksum: checksum
-    )
-  }
-
-  func dependencies() -> Target {
-    .target(
-      name: "\(name)_dependencies",
-      dependencies: [.target(name: name)],
-      path: ".swift/\(name)",
-      linkerSettings:
-          frameworks.map { .linkedFramework($0) } +
-          libraries.map { .linkedLibrary($0) }
-    )
-  }
-}
-
-let frameworks = [
-  Framework(
-    name: "coreml_backend",
-    checksum: coreml_sha256,
-    frameworks: [
+let url = "https://ossci-ios.s3.amazonaws.com/executorch/"
+let debug = "_debug"
+let deliverables = [
+  "coreml_backend": [
+    "sha256": "5bfa35cb5143b4af6840e0e5dd2d40bce93dff331b8eb5798a46274239391a5d",
+    "sha256" + debug: "1422019da9000f8ff7be597de9e0e3b2482f99cdaa75c2d179835778647be1a6",
+    "frameworks": [
       "Accelerate",
       "CoreML",
     ],
-    libraries: [
+    "libraries": [
       "sqlite3",
-    ]
-  ),
-  Framework(
-    name: "custom_backend",
-    checksum: custom_sha256
-  ),
-  Framework(
-    name: "executorch",
-    checksum: executorch_sha256
-  ),
-  Framework(
-    name: "mps_backend",
-    checksum: mps_sha256,
-    frameworks: [
+    ],
+  ],
+  "custom_backend": [
+    "sha256": "2201a61eaf7e06e1937cb73a469fb36cabc219496ba004b85feb2cc7c10f300d",
+    "sha256" + debug: "3eb6eb97bf0641d2305b0f50ff05a8862d7d65e2491cf4aa05ef1d108649f07c",
+  ],
+  "executorch": [
+    "sha256": "2b55cbcff845ab9eaf16a21e520546b2975ef8c55b9e3fbbcc0c375334e40c6f",
+    "sha256" + debug: "12933cedff6cf21c9d21668779f8d8af8049646fe7d290787b12227ff7abe4a7",
+  ],
+  "mps_backend": [
+    "sha256": "510d708361b6ea0692ce5aeb638725d6275824b37bbe744aa876fda24cc2bbbf",
+    "sha256" + debug: "6a67ba0bf8033f17bd66acb222446df51cd1304e24a4fb2c6d97e15a30fb24f0",
+    "frameworks": [
       "Metal",
       "MetalPerformanceShaders",
       "MetalPerformanceShadersGraph",
-    ]
-  ),
-  Framework(
-    name: "optimized_backend",
-    checksum: optimized_sha256
-  ),
-  Framework(
-    name: "portable_backend",
-    checksum: portable_sha256
-  ),
-  Framework(
-    name: "quantized_backend",
-    checksum: quantized_sha256
-  ),
-  Framework(
-    name: "xnnpack_backend",
-    checksum: xnnpack_sha256
-  )
-]
+    ],
+  ],
+  "optimized_backend": [
+    "sha256": "50aaa54901a7cee1059e71cc623f054610406d65ba8fd6edb10b45861be67237",
+    "sha256" + debug: "3f43f465727c8705432f4bb69260cc9501c519e5da006fc19ee2ab2ea260d1f0",
+  ],
+  "portable_backend": [
+    "sha256": "964238e92828665aa598c05b2264faab91fb13ce0f42633cc7d5653300af3e9b",
+    "sha256" + debug: "d6d85304a4b40f13c9b893e8c264ebdb15307cacf8997494b3818a52e4914b28",
+  ],
+  "quantized_backend": [
+    "sha256": "37d31a319f92e26bab2b7ec5e783a8b14457dee0a4638dcdca1d9e17539ee3fb",
+    "sha256" + debug: "6b45f66f60f6106a41e191418c970bf7b0605df73b9815a06441a5f0809b54e6",
+  ],
+  "xnnpack_backend": [
+    "sha256": "03d506243c392e872519ae1335a025ef202319c1db339a753f9d7d74cba226f0",
+    "sha256" + debug: "3341e89abc99552a6a5bad360003baed194a83e865338bc07afe9e4f171ea169",
+  ],
+].reduce(into: [String: [String: Any]]()) {
+  $0[$1.key] = $1.value
+  $0[$1.key + debug] = $1.value
+}
+.reduce(into: [String: [String: Any]]()) {
+  var newValue = $1.value
+  if $1.key.hasSuffix(debug) {
+    $1.value.forEach { key, value in
+      if key.hasSuffix(debug) {
+        newValue[String(key.dropLast(debug.count))] = value
+      }
+    }
+  }
+  $0[$1.key] = newValue.filter { key, _ in !key.hasSuffix(debug) }
+}
 
 let package = Package(
   name: "executorch",
   platforms: [
     .iOS(.v15),
   ],
-  products: frameworks.map { .library(name: $0.name, targets: ["\($0.name)_dependencies"]) },
-  targets: frameworks.flatMap { [$0.target(), $0.dependencies()] }
+  products: deliverables.keys.map { key in
+    .library(name: key, targets: ["\(key)_dependencies"])
+  }.sorted { $0.name < $1.name },
+  targets: deliverables.flatMap { key, value -> [Target] in
+    [
+      .binaryTarget(
+        name: key,
+        url: "\(url)\(key)-\(version).zip",
+        checksum: value["sha256"] as? String ?? ""
+      ),
+      .target(
+        name: "\(key)_dependencies",
+        dependencies: [.target(name: key)],
+        path: ".swift/\(key)",
+        linkerSettings:
+          (value["frameworks"] as? [String] ?? []).map { .linkedFramework($0) } +
+          (value["libraries"] as? [String] ?? []).map { .linkedLibrary($0) }
+      ),
+    ]
+  }
 )

backends/qualcomm/builders/node_visitor.py

@@ -150,19 +150,19 @@ def make_qnn_per_tensor_config(self, quant_attrs: Dict):
             quant_config,
         )
 
-    def get_quant_encoding_conf(self, node: torch.fx.Node) -> Tuple[Any, Dict]:
+    def get_quant_encoding_conf(
+        self, node: torch.fx.Node, is_input_tensor: bool = False
+    ) -> Tuple[Any, Dict]:
         if not node.meta.get("quant_attrs", None):
             return (
                 PyQnnWrapper.Qnn_QuantizationEncoding_t.QNN_QUANTIZATION_ENCODING_UNDEFINED,
                 {},
             )
-
         quant_attrs = (
-            node.meta["requantize"]["dq_attrs"]
-            if "requantize" in node.meta
+            node.meta["requantize"]
+            if "requantize" in node.meta and is_input_tensor
             else node.meta["quant_attrs"]
         )
-
         if quant_attrs["encoding"] in PER_CHANNEL_ENCODING:
             return self.make_qnn_per_channel_config(node, quant_attrs)
 
@@ -275,15 +275,27 @@ def define_custom_tensor_wrapper(
         nodes_to_wrappers[node_name] = tensor_wrapper
         return tensor_wrapper
 
-    def define_value(
+    def define_tensor(
         self,
         node: torch.fx.Node,
         tensor: torch.Tensor,
         tensor_type: PyQnnWrapper.Qnn_TensorType_t,
         nodes_to_wrappers: Dict[str, PyQnnWrapper.TensorWrapper],
-        is_tensor: bool,
+        is_input_tensor: bool,
         node_name: str = None,
+        is_tensor: bool = True,
     ) -> PyQnnWrapper.TensorWrapper:
+        """
+        Covert torch.Tensor to TensorWrapper
+
+        Args:
+            node: EdgeIR Node
+            tensor: EdgeIR Tensor
+            tensor_type: QNN tensor type
+            nodes_to_wrappers: Set contains edge_graph values(node targets)
+            is_input_tensor: Whether tensor is a fake input tensor relatively to
+                             the op builder that is calling this function
+        """
         if node_name is None:
             node_name = node.name
 
@@ -294,7 +306,9 @@ def define_value(
             tensor_name = "output_" + tensor_name
         dims = [1] if len(tensor.size()) == 0 else tensor.size()
         tensor_type = self.get_tensor_type(node, tensor_type)
-        quant_encoding, quant_configs = self.get_quant_encoding_conf(node)
+        quant_encoding, quant_configs = self.get_quant_encoding_conf(
+            node, is_input_tensor
+        )
         dtype = self.get_data_type(tensor, quant_configs, is_tensor)
         if isinstance(tensor, torch._subclasses.fake_tensor.FakeTensor):
             tensor_wrapper = PyQnnWrapper.TensorWrapper(
@@ -330,52 +344,6 @@ def define_value(
         nodes_to_wrappers[node_name] = tensor_wrapper
         return tensor_wrapper
 
-    def define_scalar(
-        self,
-        node: torch.fx.Node,
-        tensor: torch.Tensor,
-        tensor_type: PyQnnWrapper.Qnn_TensorType_t,
-        nodes_to_wrappers: Dict[torch.fx.Node, PyQnnWrapper.TensorWrapper],
-    ) -> PyQnnWrapper.TensorWrapper:
-        """
-        Covert constant scalar to TensorWrapper
-
-        Args:
-            tensor: EdgeIR Tensor
-            nodes_to_wrappers: Set contains edge_graph values(node targets)
-        """
-        return self.define_value(
-            node,
-            tensor,
-            tensor_type,
-            nodes_to_wrappers,
-            is_tensor=False,
-        )
-
-    def define_tensor(
-        self,
-        node: torch.fx.Node,
-        tensor: torch.Tensor,
-        tensor_type: PyQnnWrapper.Qnn_TensorType_t,
-        nodes_to_wrappers: Dict[str, PyQnnWrapper.TensorWrapper],
-        node_name: str = None,
-    ) -> PyQnnWrapper.TensorWrapper:
-        """
-        Covert torch.Tensor to TensorWrapper
-
-        Args:
-            tensor: EdgeIR Tensor
-            nodes_to_wrappers: Set contains edge_graph values(node targets)
-        """
-        return self.define_value(
-            node,
-            tensor,
-            tensor_type,
-            nodes_to_wrappers,
-            is_tensor=True,
-            node_name=node_name,
-        )
-
     def define_node(
         self,
         node: torch.fx.Node,

backends/qualcomm/builders/op_add.py

@@ -31,6 +31,7 @@ def define_node(
             out_tensor,
             PyQnnWrapper.Qnn_TensorType_t.QNN_TENSOR_TYPE_NATIVE,
             nodes_to_wrappers,
+            is_input_tensor=False,
         )
         add_output_tensors = [output_tensor_wrapper]
 
@@ -45,6 +46,7 @@ def define_node(
                 input_tensor,
                 tensor_type,
                 nodes_to_wrappers,
+                is_input_tensor=True,
             )
             add_input_tensors.append(input_tensor_wrapper)
 

backends/qualcomm/builders/op_avg_pool2d.py

@@ -33,6 +33,7 @@ def define_node(
             input_tensor,
             PyQnnWrapper.Qnn_TensorType_t.QNN_TENSOR_TYPE_NATIVE,
             nodes_to_wrappers,
+            is_input_tensor=True,
         )
 
         output_tensor = self.get_tensor(node, node)
@@ -41,6 +42,7 @@ def define_node(
             output_tensor,
             PyQnnWrapper.Qnn_TensorType_t.QNN_TENSOR_TYPE_NATIVE,
             nodes_to_wrappers,
+            is_input_tensor=False,
         )
         # kernel info
         filter_size = cast(List[int], node.args[1])

backends/qualcomm/builders/op_batch_norm.py

@@ -38,6 +38,7 @@ def define_node(
             input_tensor,
             PyQnnWrapper.Qnn_TensorType_t.QNN_TENSOR_TYPE_NATIVE,
             nodes_to_wrappers,
+            is_input_tensor=True,
         )
 
         bias_node = node.args[2]
@@ -52,6 +53,7 @@ def define_node(
             bias_tensor,
             PyQnnWrapper.Qnn_TensorType_t.QNN_TENSOR_TYPE_STATIC,
             nodes_to_wrappers,
+            is_input_tensor=False,
         )
 
         filter_tensor = filter_tensor / torch.sqrt(var_tensor + eps)
@@ -60,6 +62,7 @@ def define_node(
             filter_tensor,
             PyQnnWrapper.Qnn_TensorType_t.QNN_TENSOR_TYPE_STATIC,
             nodes_to_wrappers,
+            is_input_tensor=False,
         )
 
         batch_norm_input_tensors = [
@@ -74,6 +77,7 @@ def define_node(
             output_tensor,
             PyQnnWrapper.Qnn_TensorType_t.QNN_TENSOR_TYPE_NATIVE,
             nodes_to_wrappers,
+            is_input_tensor=False,
         )
         batch_norm_output_tensors = [output_tensor_wrapper]
 

backends/qualcomm/builders/op_bmm.py

@@ -35,6 +35,7 @@ def define_node(
                 input_tensor,
                 PyQnnWrapper.Qnn_TensorType_t.QNN_TENSOR_TYPE_NATIVE,
                 nodes_to_wrappers,
+                is_input_tensor=True,
             )
             bmm_input_tensors.append(input_tensor_wrapper)
 
@@ -44,6 +45,7 @@ def define_node(
             output_tensor,
             PyQnnWrapper.Qnn_TensorType_t.QNN_TENSOR_TYPE_NATIVE,
             nodes_to_wrappers,
+            is_input_tensor=False,
         )
         bmm_output_tensors = [output_tensor_wrapper]