Decouple model e2e tests (#487)

Summary:

We should not couple e2e tests with different recipes *(e.g. run with portable kernels, run with xnnpack options)* in a single job. Addressing timeout issue and improving CI stability is one of the reasons, however, even w/o timeout we still want to do it for:
  1. Maximal parallelism to reduce waiting time
  2. Flexibility to enable/disable an e2e test for a given recipe, e.g. disable slow running e2e test with portable kernels but keep the xnnpack e2e tests enabled, or if there are issue with a flavor
  3. Cleaner CI job names to describe what it does

Of course this is not real fix of the timeout issue we're facing, the complete fix will include 1) introducing release build for both cmake and buck2 and 2) fixing the slow portable kernels from the root. And both are far away and not possible before MVP.

For MVP as we have more and more models enabled with XNNPACK as a part of the out-of-box experience, our CI is facing extra stress. See the [ci dashboard](https://hud.pytorch.org/hud/pytorch/executorch/main). The major motivation in this diff is to ensure the healthiness of enabled models as well as our CI.

Reviewed By: huydhn

Differential Revision: D49622665

Author: guangy10

.ci/scripts/gather_test_models.py

@@ -49,23 +49,27 @@ def export_models_for_ci() -> None:
     # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
     models = {"include": []}
     for name in MODEL_NAME_TO_MODEL.keys():
-        quantization = (
-            name in MODEL_NAME_TO_OPTIONS and MODEL_NAME_TO_OPTIONS[name].quantization
-        )
-        xnnpack_delegation = (
+        quantization_configs = {
+            False,
+            name in MODEL_NAME_TO_OPTIONS and MODEL_NAME_TO_OPTIONS[name].quantization,
+        }
+        delegation_configs = {
+            False,
             name in MODEL_NAME_TO_OPTIONS
-            and MODEL_NAME_TO_OPTIONS[name].xnnpack_delegation
-        )
+            and MODEL_NAME_TO_OPTIONS[name].xnnpack_delegation,
+        }
         for build_tool in BUILD_TOOLS:
-            models["include"].append(
-                {
-                    "build-tool": build_tool,
-                    "model": name,
-                    "quantization": quantization,
-                    "xnnpack_delegation": xnnpack_delegation,
-                    "runner": RUNNERS.get(name, DEFAULT_RUNNER),
-                }
-            )
+            for q_config in quantization_configs:
+                for d_config in delegation_configs:
+                    models["include"].append(
+                        {
+                            "build-tool": build_tool,
+                            "model": name,
+                            "xnnpack_quantization": q_config,
+                            "xnnpack_delegation": d_config,
+                            "runner": RUNNERS.get(name, DEFAULT_RUNNER),
+                        }
+                    )
     set_output("models", json.dumps(models))
 
 

.ci/scripts/test.sh

@@ -22,9 +22,9 @@ if [[ -z "${BUILD_TOOL:-}" ]]; then
   exit 1
 fi
 
-QUANTIZATION=$3
-if [[ -z "${QUANTIZATION:-}" ]]; then
-  QUANTIZATION=false
+XNNPACK_QUANTIZATION=$3
+if [[ -z "${XNNPACK_QUANTIZATION:-}" ]]; then
+  XNNPACK_QUANTIZATION=false
 fi
 
 XNNPACK_DELEGATION=$4
@@ -83,6 +83,15 @@ build_cmake_xnn_executor_runner() {
 
 test_model_with_xnnpack() {
   WITH_QUANTIZATION=$1
+  WITH_DELEGATION=$2
+
+  # Quantization-only
+  if [[ ${WITH_QUANTIZATION} == true ]] && [[ ${WITH_DELEGATION} == false ]]; then
+    bash examples/quantization/test_quantize.sh "${BUILD_TOOL}" "${MODEL_NAME}"
+    exit 0
+  fi
+
+  # Delegation
   if [[ ${WITH_QUANTIZATION} == true ]]; then
     SUFFIX="q8"
     "${PYTHON_EXECUTABLE}" -m examples.backend.xnnpack_examples --model_name="${MODEL_NAME}" --delegate --quantize
@@ -92,6 +101,7 @@ test_model_with_xnnpack() {
   fi
 
   OUTPUT_MODEL_PATH="${MODEL_NAME}_xnnpack_${SUFFIX}.pte"
+
   # Run test model
   if [[ "${BUILD_TOOL}" == "buck2" ]]; then
     buck2 run //examples/backend:xnn_executor_runner -- --model_path "${OUTPUT_MODEL_PATH}"
@@ -106,25 +116,10 @@ test_model_with_xnnpack() {
   fi
 }
 
-echo "Testing ${MODEL_NAME} (fp32, quantized, xnnpack) with ${BUILD_TOOL}..."
-# Test the select model without XNNPACK or quantization
-test_model
-
-# Test quantization
-if [[ "${QUANTIZATION}" == true ]]; then
-  bash examples/quantization/test_quantize.sh "${BUILD_TOOL}" "${MODEL_NAME}"
+if [[ "${XNNPACK_DELEGATION}" == false ]] && [[ "${XNNPACK_QUANTIZATION}" == false ]]; then
+  echo "Testing ${MODEL_NAME} with portable kernels..."
+  test_model
 else
-  echo "The model ${MODEL_NAME} doesn't support quantization yet"
-fi
-
-# Test XNNPACK without quantization
-if [[ "${XNNPACK_DELEGATION}" == true ]]; then
-  test_model_with_xnnpack false
-else
-  echo "The model ${MODEL_NAME} doesn't support XNNPACK yet"
-fi
-
-# Test XNNPACK with quantization
-if [[ "${XNNPACK_DELEGATION}" == true ]] && [[ "${QUANTIZATION}" == true ]]; then
-  test_model_with_xnnpack true
+  echo "Testing ${MODEL_NAME} with XNNPACK quantization=${XNNPACK_QUANTIZATION} delegation=${XNNPACK_DELEGATION}..."
+  test_model_with_xnnpack "${XNNPACK_QUANTIZATION}" "${XNNPACK_DELEGATION}"
 fi

.github/workflows/pull.yml

@@ -57,12 +57,12 @@ jobs:
 
         MODEL_NAME=${{ matrix.model }}
         BUILD_TOOL=${{ matrix.build-tool }}
-        QUANTIZATION=${{ matrix.quantization }}
+        XNNPACK_QUANTIZATION=${{ matrix.xnnpack_quantization }}
         XNNPACK_DELEGATION=${{ matrix.xnnpack_delegation }}
 
         PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}"
         # Build and test Executorch
-        PYTHON_EXECUTABLE=python bash .ci/scripts/test.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${QUANTIZATION}" "${XNNPACK_DELEGATION}"
+        PYTHON_EXECUTABLE=python bash .ci/scripts/test.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${XNNPACK_QUANTIZATION}" "${XNNPACK_DELEGATION}"
 
   test-models-macos:
     name: test-models-macos
@@ -82,13 +82,13 @@ jobs:
 
         MODEL_NAME=${{ matrix.model }}
         BUILD_TOOL=${{ matrix.build-tool }}
-        QUANTIZATION=${{ matrix.quantization }}
+        XNNPACK_QUANTIZATION=${{ matrix.xnnpack_quantization }}
         XNNPACK_DELEGATION=${{ matrix.xnnpack_delegation }}
 
         # Setup MacOS dependencies as there is no Docker support on MacOS atm
         PYTHON_EXECUTABLE=python bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}"
         # Build and test Executorch
-        PYTHON_EXECUTABLE=python bash .ci/scripts/test.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${QUANTIZATION}" "${XNNPACK_DELEGATION}"
+        PYTHON_EXECUTABLE=python bash .ci/scripts/test.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${XNNPACK_QUANTIZATION}" "${XNNPACK_DELEGATION}"
         popd
 
   test-custom-ops-linux: