Use only cmake for MacOS CI to save capacity (#613)

Summary:
This change updates the gather script to:

1. Use only cmake for MacOS CI to save capacity. This cuts the number of MacOS job in half
2. Fix the wrong and confusing reference of `linux.2xlarge` on MacOS jobs

Pull Request resolved: #613

Reviewed By: seemethere

Differential Revision: D49919115

Pulled By: huydhn

fbshipit-source-id: c6b8619153959259786e8c2adb3e721934c3841e

Author: huydhn

.ci/scripts/gather_test_models.py

@@ -12,20 +12,42 @@
 from examples.models import MODEL_NAME_TO_MODEL
 from examples.recipes.xnnpack_optimization import MODEL_NAME_TO_OPTIONS
 
-BUILD_TOOLS = [
-    "buck2",
-    "cmake",
-]
-DEFAULT_RUNNER = "linux.2xlarge"
-RUNNERS = {
-    # This one runs OOM on smaller runner, the root cause is unclear (T163016365)
-    "w2l": "linux.12xlarge",
-    "ic4": "linux.12xlarge",
-    "resnet50": "linux.12xlarge",
-    # This one causes timeout on smaller runner, the root cause is unclear (T161064121)
-    "dl3": "linux.12xlarge",
-    "emformer_join": "linux.12xlarge",
+# NB: Skip buck2 on MacOS to cut down the number of combinations we
+# need to run there as the number of MacOS runner is limited. Buck2
+# build and test has already been covered on Linux
+BUILD_TOOLS = {
+    "buck2": {"linux"},
+    "cmake": {"linux", "macos"},
 }
+DEFAULT_RUNNERS = {
+    "linux": "linux.2xlarge",
+    "macos": "macos-m1-12",
+}
+CUSTOM_RUNNERS = {
+    "linux": {
+        # This one runs OOM on smaller runner, the root cause is unclear (T163016365)
+        "w2l": "linux.12xlarge",
+        "ic4": "linux.12xlarge",
+        "resnet50": "linux.12xlarge",
+        # This one causes timeout on smaller runner, the root cause is unclear (T161064121)
+        "dl3": "linux.12xlarge",
+        "emformer_join": "linux.12xlarge",
+    }
+}
+
+
+def parse_args() -> Any:
+    from argparse import ArgumentParser
+
+    parser = ArgumentParser("Gather all models to test on CI for the target OS")
+    parser.add_argument(
+        "--target-os",
+        type=str,
+        choices=["linux", "macos"],
+        default="linux",
+        help="the target OS",
+    )
+    return parser.parse_args()
 
 
 def set_output(name: str, val: Any) -> None:
@@ -45,6 +67,9 @@ def export_models_for_ci() -> None:
     """
     This gathers all the example models that we want to test on GitHub OSS CI
     """
+    args = parse_args()
+    target_os = args.target_os
+
     # This is the JSON syntax for configuration matrix used by GitHub
     # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
     models = {"include": []}
@@ -58,20 +83,31 @@ def export_models_for_ci() -> None:
             name in MODEL_NAME_TO_OPTIONS
             and MODEL_NAME_TO_OPTIONS[name].xnnpack_delegation,
         }
-        for build_tool in BUILD_TOOLS:
+        for build_tool in BUILD_TOOLS.keys():
+            if target_os not in BUILD_TOOLS[build_tool]:
+                continue
+
             for q_config in quantization_configs:
                 for d_config in delegation_configs:
-                    models["include"].append(
-                        {
-                            "build-tool": build_tool,
-                            "model": name,
-                            "xnnpack_quantization": q_config,
-                            "xnnpack_delegation": d_config,
-                            "runner": RUNNERS.get(name, DEFAULT_RUNNER),
-                            # demo_backend_delegation test only supports add_mul model
-                            "demo_backend_delegation": name == "add_mul",
-                        }
-                    )
+                    record = {
+                        "build-tool": build_tool,
+                        "model": name,
+                        "xnnpack_quantization": q_config,
+                        "xnnpack_delegation": d_config,
+                        "runner": DEFAULT_RUNNERS.get(target_os, "linux.2xlarge"),
+                        # demo_backend_delegation test only supports add_mul model
+                        "demo_backend_delegation": name == "add_mul",
+                    }
+
+                    # NB: Some model requires much bigger Linux runner to avoid
+                    # running OOM. The team is investigating the root cause
+                    if target_os in CUSTOM_RUNNERS and name in CUSTOM_RUNNERS.get(
+                        target_os, {}
+                    ):
+                        record["runner"] = CUSTOM_RUNNERS[target_os][name]
+
+                    models["include"].append(record)
+
     set_output("models", json.dumps(models))
 
 

.github/workflows/trunk.yml

@@ -34,7 +34,7 @@ jobs:
           install_pip_dependencies
           install_executorch
 
-          PYTHONPATH="${PWD}" python .ci/scripts/gather_test_models.py
+          PYTHONPATH="${PWD}" python .ci/scripts/gather_test_models.py --target-os macos
 
   test-models-macos:
     name: test-models-macos
@@ -44,7 +44,7 @@ jobs:
       matrix: ${{ fromJSON(needs.gather-models.outputs.models) }}
       fail-fast: false
     with:
-      runner: macos-m1-12
+      runner: ${{ matrix.runner }}
       submodules: 'true'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
@@ -70,7 +70,6 @@ jobs:
     strategy:
       matrix:
         include:
-          - build-tool: buck2
           - build-tool: cmake
       fail-fast: false
     with:
@@ -95,7 +94,6 @@ jobs:
     strategy:
       matrix:
         include:
-          - build-tool: buck2
           - build-tool: cmake
       fail-fast: false
     with: