Decouple model e2e tests

Differential Revision: D49622665

Author: guangy10

.ci/scripts/gather_test_models.py

@@ -49,23 +49,27 @@ def export_models_for_ci() -> None:
     # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
     models = {"include": []}
     for name in MODEL_NAME_TO_MODEL.keys():
-        quantization = (
-            name in MODEL_NAME_TO_OPTIONS and MODEL_NAME_TO_OPTIONS[name].quantization
-        )
-        xnnpack_delegation = (
+        quantization_configs = {
+            False,
+            name in MODEL_NAME_TO_OPTIONS and MODEL_NAME_TO_OPTIONS[name].quantization,
+        }
+        delegation_configs = {
+            False,
             name in MODEL_NAME_TO_OPTIONS
-            and MODEL_NAME_TO_OPTIONS[name].xnnpack_delegation
-        )
+            and MODEL_NAME_TO_OPTIONS[name].xnnpack_delegation,
+        }
         for build_tool in BUILD_TOOLS:
-            models["include"].append(
-                {
-                    "build-tool": build_tool,
-                    "model": name,
-                    "quantization": quantization,
-                    "xnnpack_delegation": xnnpack_delegation,
-                    "runner": RUNNERS.get(name, DEFAULT_RUNNER),
-                }
-            )
+            for q_config in quantization_configs:
+                for d_config in delegation_configs:
+                    models["include"].append(
+                        {
+                            "build-tool": build_tool,
+                            "model": name,
+                            "xnnpack_quantization": q_config,
+                            "xnnpack_delegation": d_config,
+                            "runner": RUNNERS.get(name, DEFAULT_RUNNER),
+                        }
+                    )
     set_output("models", json.dumps(models))
 
 

.ci/scripts/test.sh

@@ -22,9 +22,9 @@ if [[ -z "${BUILD_TOOL:-}" ]]; then
   exit 1
 fi
 
-QUANTIZATION=$3
-if [[ -z "${QUANTIZATION:-}" ]]; then
-  QUANTIZATION=false
+XNNPACK_QUANTIZATION=$3
+if [[ -z "${XNNPACK_QUANTIZATION:-}" ]]; then
+  XNNPACK_QUANTIZATION=false
 fi
 
 XNNPACK_DELEGATION=$4
@@ -83,6 +83,15 @@ build_cmake_xnn_executor_runner() {
 
 test_model_with_xnnpack() {
   WITH_QUANTIZATION=$1
+  WITH_DELEGATION=$2
+
+  # Quantization-only
+  if [[ ${WITH_QUANTIZATION} == true ]] && [[ ${WITH_DELEGATION=$2} == false ]]; then
+    bash examples/quantization/test_quantize.sh "${BUILD_TOOL}" "${MODEL_NAME}"
+    exit 0
+  fi
+
+  # Delegation
   if [[ ${WITH_QUANTIZATION} == true ]]; then
     SUFFIX="q8"
     "${PYTHON_EXECUTABLE}" -m examples.backend.xnnpack_examples --model_name="${MODEL_NAME}" --delegate --quantize
@@ -92,6 +101,7 @@ test_model_with_xnnpack() {
   fi
 
   OUTPUT_MODEL_PATH="${MODEL_NAME}_xnnpack_${SUFFIX}.pte"
+
   # Run test model
   if [[ "${BUILD_TOOL}" == "buck2" ]]; then
     buck2 run //examples/backend:xnn_executor_runner -- --model_path "${OUTPUT_MODEL_PATH}"
@@ -106,25 +116,10 @@ test_model_with_xnnpack() {
   fi
 }
 
-echo "Testing ${MODEL_NAME} (fp32, quantized, xnnpack) with ${BUILD_TOOL}..."
-# Test the select model without XNNPACK or quantization
-test_model
-
-# Test quantization
-if [[ "${QUANTIZATION}" == true ]]; then
-  bash examples/quantization/test_quantize.sh "${BUILD_TOOL}" "${MODEL_NAME}"
+if [[ "${XNNPACK_DELEGATION}" == false ]] && [[ "${XNNPACK_QUANTIZATION}" == false ]]; then
+echo "Testing ${MODEL_NAME} with portable kernels..."
+  test_model
 else
-  echo "The model ${MODEL_NAME} doesn't support quantization yet"
-fi
-
-# Test XNNPACK without quantization
-if [[ "${XNNPACK_DELEGATION}" == true ]]; then
-  test_model_with_xnnpack false
-else
-  echo "The model ${MODEL_NAME} doesn't support XNNPACK yet"
-fi
-
-# Test XNNPACK with quantization
-if [[ "${XNNPACK_DELEGATION}" == true ]] && [[ "${QUANTIZATION}" == true ]]; then
-  test_model_with_xnnpack true
+  echo "Testing ${MODEL_NAME} with XNNPACK quantization=${XNNPACK_QUANTIZATION} delegation=${XNNPACK_DELEGATION}..."
+  test_model_with_xnnpack "${XNNPACK_QUANTIZATION}" "${XNNPACK_DELEGATION}"
 fi

.github/workflows/pull.yml

@@ -57,12 +57,12 @@ jobs:
 
         MODEL_NAME=${{ matrix.model }}
         BUILD_TOOL=${{ matrix.build-tool }}
-        QUANTIZATION=${{ matrix.quantization }}
+        XNNPACK_QUANTIZATION=${{ matrix.xnnpack_quantization }}
         XNNPACK_DELEGATION=${{ matrix.xnnpack_delegation }}
 
         PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}"
         # Build and test Executorch
-        PYTHON_EXECUTABLE=python bash .ci/scripts/test.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${QUANTIZATION}" "${XNNPACK_DELEGATION}"
+        PYTHON_EXECUTABLE=python bash .ci/scripts/test.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${XNNPACK_QUANTIZATION}" "${XNNPACK_DELEGATION}"
 
   test-models-macos:
     name: test-models-macos
@@ -82,13 +82,13 @@ jobs:
 
         MODEL_NAME=${{ matrix.model }}
         BUILD_TOOL=${{ matrix.build-tool }}
-        QUANTIZATION=${{ matrix.quantization }}
+        XNNPACK_QUANTIZATION=${{ matrix.xnnpack_quantization }}
         XNNPACK_DELEGATION=${{ matrix.xnnpack_delegation }}
 
         # Setup MacOS dependencies as there is no Docker support on MacOS atm
         PYTHON_EXECUTABLE=python bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}"
         # Build and test Executorch
-        PYTHON_EXECUTABLE=python bash .ci/scripts/test.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${QUANTIZATION}" "${XNNPACK_DELEGATION}"
+        PYTHON_EXECUTABLE=python bash .ci/scripts/test.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${XNNPACK_QUANTIZATION}" "${XNNPACK_DELEGATION}"
         popd
 
   test-custom-ops-linux: