Run XNNPACK on single workflow (#1071)

Summary:
Run XNNPACK (delegation-only, quantization-only, delegation+quantization) on a single workflow to save the number of jobs.

Pull Request resolved: #1071

Reviewed By: huydhn

Differential Revision: D50580091

Pulled By: kirklandsign

fbshipit-source-id: 6b750a818497cb42739b91492fa137b7d1518e64

Author: kirklandsign

.ci/scripts/gather_test_models.py

@@ -93,32 +93,30 @@ def export_models_for_ci() -> dict[str, dict]:
     # This is the JSON syntax for configuration matrix used by GitHub
     # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
     models = {"include": []}
-    for (name, build_tool, q_config, d_config) in itertools.product(
-        MODEL_NAME_TO_MODEL.keys(), BUILD_TOOLS.keys(), [False, True], [False, True]
+    backends = ["portable", "xnnpack"]
+    for (name, build_tool, backend) in itertools.product(
+        MODEL_NAME_TO_MODEL.keys(), BUILD_TOOLS.keys(), backends
     ):
         if not model_should_run_on_event(name, event):
             continue
 
-        if q_config and (
-            (name not in MODEL_NAME_TO_OPTIONS)
-            or (not MODEL_NAME_TO_OPTIONS[name].quantization)
-        ):
-            continue
+        if backend == "xnnpack":
+            if (
+                name in MODEL_NAME_TO_OPTIONS
+                and MODEL_NAME_TO_OPTIONS[name].quantization
+            ):
+                backend += "-quantization"
 
-        if d_config and (
-            (name not in MODEL_NAME_TO_OPTIONS)
-            or (not MODEL_NAME_TO_OPTIONS[name].delegation)
-        ):
-            continue
+            if name in MODEL_NAME_TO_OPTIONS and MODEL_NAME_TO_OPTIONS[name].delegation:
+                backend += "-delegation"
 
         if target_os not in BUILD_TOOLS[build_tool]:
             continue
 
         record = {
             "build-tool": build_tool,
             "model": name,
-            "xnnpack_quantization": q_config,
-            "xnnpack_delegation": d_config,
+            "backend": backend,
             "runner": DEFAULT_RUNNERS.get(target_os, "linux.2xlarge"),
             # demo_backend_delegation test only supports add_mul model
             "demo_backend_delegation": name == "add_mul",

.ci/scripts/test.sh

@@ -22,17 +22,13 @@ if [[ -z "${BUILD_TOOL:-}" ]]; then
   exit 1
 fi
 
-XNNPACK_QUANTIZATION=$3
-if [[ -z "${XNNPACK_QUANTIZATION:-}" ]]; then
-  XNNPACK_QUANTIZATION=false
-fi
-
-XNNPACK_DELEGATION=$4
-if [[ -z "${XNNPACK_DELEGATION:-}" ]]; then
-  XNNPACK_DELEGATION=false
+BACKEND=$3
+if [[ -z "${BACKEND:-}" ]]; then
+  echo "Missing backend (require portable or xnnpack), exiting..."
+  exit 1
 fi
 
-DEMO_BACKEND_DELEGATION=$5
+DEMO_BACKEND_DELEGATION=$4
 if [[ -z "${DEMO_BACKEND_DELEGATION:-}" ]]; then
   DEMO_BACKEND_DELEGATION=false
 fi
@@ -99,7 +95,7 @@ test_model_with_xnnpack() {
   # Quantization-only
   if [[ ${WITH_QUANTIZATION} == true ]] && [[ ${WITH_DELEGATION} == false ]]; then
     bash examples/xnnpack/quantization/test_quantize.sh "${BUILD_TOOL}" "${MODEL_NAME}"
-    exit 0
+    return 0
   fi
 
   # Delegation
@@ -151,12 +147,25 @@ test_demo_backend_delegation() {
   fi
 }
 
-if [[ "${XNNPACK_DELEGATION}" == false ]] && [[ "${XNNPACK_QUANTIZATION}" == false ]]; then
+if [[ "${BACKEND}" == "portable" ]]; then
   echo "Testing ${MODEL_NAME} with portable kernels..."
   test_model
 else
-  echo "Testing ${MODEL_NAME} with XNNPACK quantization=${XNNPACK_QUANTIZATION} delegation=${XNNPACK_DELEGATION}..."
-  test_model_with_xnnpack "${XNNPACK_QUANTIZATION}" "${XNNPACK_DELEGATION}"
+  if [[ "${BACKEND}" == *"quantization"* ]]; then
+    echo "::group::Testing ${MODEL_NAME} with XNNPACK quantization only..."
+    test_model_with_xnnpack true false
+    echo "::endgroup::"
+  fi
+  if [[ "${BACKEND}" == *"delegation"* ]]; then
+    echo "::group::Testing ${MODEL_NAME} with XNNPACK delegation only..."
+    test_model_with_xnnpack false true
+    echo "::endgroup::"
+  fi
+  if [[ "${BACKEND}" == *"quantization"* ]] && [[ "${BACKEND}" == *"delegation"* ]]; then
+    echo "::group::Testing ${MODEL_NAME} with XNNPACK quantization and delegation..."
+    test_model_with_xnnpack true true
+    echo "::endgroup::"
+  fi
 fi
 
 # Test demo backend delegation

.github/workflows/pull.yml

@@ -51,13 +51,12 @@ jobs:
 
         MODEL_NAME=${{ matrix.model }}
         BUILD_TOOL=${{ matrix.build-tool }}
-        XNNPACK_QUANTIZATION=${{ matrix.xnnpack_quantization }}
-        XNNPACK_DELEGATION=${{ matrix.delegation }}
+        BACKEND=${{ matrix.backend }}
         DEMO_BACKEND_DELEGATION=${{ matrix.demo_backend_delegation }}
 
         PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}"
         # Build and test ExecuTorch
-        PYTHON_EXECUTABLE=python bash .ci/scripts/test.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${XNNPACK_QUANTIZATION}" "${XNNPACK_DELEGATION}" "${DEMO_BACKEND_DELEGATION}"
+        PYTHON_EXECUTABLE=python bash .ci/scripts/test.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" "${DEMO_BACKEND_DELEGATION}"
 
   test-custom-ops-linux:
     name: test-custom-ops-linux

.github/workflows/trunk.yml

@@ -21,22 +21,15 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          submodules: 'true'
+          submodules: 'false'
       - uses: actions/setup-python@v4
         with:
           python-version: '3.10'
-          cache: pip
       - name: Extract the list of models to test
         id: gather-models
         run: |
           set -eux
 
-          source .ci/scripts/utils.sh
-          # This is a simple Python script but as it tries to import executorch.examples.models,
-          # it requires a whole bunch of ExecuTorch dependencies on the Docker image
-          install_pip_dependencies
-          install_executorch
-
           PYTHONPATH="${PWD}" python .ci/scripts/gather_test_models.py --target-os macos --event "${GITHUB_EVENT_NAME}"
 
   test-models-macos:
@@ -57,14 +50,13 @@ jobs:
 
         MODEL_NAME=${{ matrix.model }}
         BUILD_TOOL=${{ matrix.build-tool }}
-        XNNPACK_QUANTIZATION=${{ matrix.xnnpack_quantization }}
-        XNNPACK_DELEGATION=${{ matrix.delegation }}
+        BACKEND=${{ matrix.backend }}
         DEMO_BACKEND_DELEGATION=${{ matrix.demo_backend_delegation }}
 
         # Setup MacOS dependencies as there is no Docker support on MacOS atm
         PYTHON_EXECUTABLE=python bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}"
         # Build and test xecutorch
-        PYTHON_EXECUTABLE=python bash .ci/scripts/test.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${XNNPACK_QUANTIZATION}" "${XNNPACK_DELEGATION}" "${DEMO_BACKEND_DELEGATION}"
+        PYTHON_EXECUTABLE=python bash .ci/scripts/test.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" "${DEMO_BACKEND_DELEGATION}"
         popd
 
   test-custom-ops-macos: