Migrate test-demo-backend-delegation to a separate job (#1078)

Summary:
Instead of using test-models and test.sh, we use a separate job, because this job is quite different, and we want to get rid of the hack from the additional arg.

See https://github.com/pytorch/executorch/actions/runs/6631141293/job/18014013866?pr=1078 for logs

Pull Request resolved: #1078

Reviewed By: huydhn

Differential Revision: D50611594

Pulled By: kirklandsign

fbshipit-source-id: abcc03dec57484a7c6fb8629214d978d7a6b9067

Author: kirklandsign

.ci/scripts/gather_test_models.py

@@ -118,8 +118,6 @@ def export_models_for_ci() -> dict[str, dict]:
             "model": name,
             "backend": backend,
             "runner": DEFAULT_RUNNERS.get(target_os, "linux.2xlarge"),
-            # demo_backend_delegation test only supports add_mul model
-            "demo_backend_delegation": name == "add_mul",
         }
 
         # NB: Some model requires much bigger Linux runner to avoid

.ci/scripts/test.sh

@@ -28,11 +28,6 @@ if [[ -z "${BACKEND:-}" ]]; then
   exit 1
 fi
 
-DEMO_BACKEND_DELEGATION=$4
-if [[ -z "${DEMO_BACKEND_DELEGATION:-}" ]]; then
-  DEMO_BACKEND_DELEGATION=false
-fi
-
 which "${PYTHON_EXECUTABLE}"
 # Just set this variable here, it's cheap even if we use buck2
 CMAKE_OUTPUT_DIR=cmake-out
@@ -123,30 +118,6 @@ test_model_with_xnnpack() {
   fi
 }
 
-test_demo_backend_delegation() {
-  echo "Testing demo backend delegation on AddMul"
-  "${PYTHON_EXECUTABLE}" -m examples.portable.scripts.export_and_delegate  --option "composite"
-  "${PYTHON_EXECUTABLE}" -m examples.portable.scripts.export_and_delegate  --option "partition"
-  "${PYTHON_EXECUTABLE}" -m examples.portable.scripts.export_and_delegate  --option "whole"
-
-  # Run test model
-  if [[ "${BUILD_TOOL}" == "buck2" ]]; then
-    buck2 run //examples/portable/executor_runner:executor_runner -- --model_path "./composite_model.pte"
-    buck2 run //examples/portable/executor_runner:executor_runner -- --model_path "./partition_lowered_model.pte"
-    buck2 run //examples/portable/executor_runner:executor_runner -- --model_path "./whole.pte"
-  elif [[ "${BUILD_TOOL}" == "cmake" ]]; then
-    if [[ ! -f ${CMAKE_OUTPUT_DIR}/executor_runner ]]; then
-      build_cmake_executor_runner
-    fi
-    ./${CMAKE_OUTPUT_DIR}/executor_runner --model_path "./composite_model.pte"
-    ./${CMAKE_OUTPUT_DIR}/executor_runner --model_path "./partition_lowered_model.pte"
-    ./${CMAKE_OUTPUT_DIR}/executor_runner --model_path "./whole.pte"
-  else
-    echo "Invalid build tool ${BUILD_TOOL}. Only buck2 and cmake are supported atm"
-    exit 1
-  fi
-}
-
 if [[ "${BACKEND}" == "portable" ]]; then
   echo "Testing ${MODEL_NAME} with portable kernels..."
   test_model
@@ -173,8 +144,3 @@ else
     exit 1
   fi
 fi
-
-# Test demo backend delegation
-if [[ "${DEMO_BACKEND_DELEGATION}" == true ]]; then
-  test_demo_backend_delegation
-fi

.github/workflows/trunk.yml

@@ -107,6 +107,30 @@ jobs:
         PYTHON_EXECUTABLE=python bash examples/selective_build/test_selective_build.sh "${BUILD_TOOL}"
         popd
 
+  test-demo-backend-delegation:
+    name: test-demo-backend-delegation
+    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
+    strategy:
+      matrix:
+        include:
+          - build-tool: buck2
+          - build-tool: cmake
+      fail-fast: false
+    with:
+      runner: linux.2xlarge
+      docker-image: executorch-ubuntu-22.04-clang12
+      submodules: 'true'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      script: |
+        # The generic Linux job chooses to use base env, not the one setup by the image
+        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+        conda activate "${CONDA_ENV}"
+
+        BUILD_TOOL=${{ matrix.build-tool }}
+        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}"
+        # Test selective build
+        PYTHON_EXECUTABLE=python bash examples/portable/scripts/test_demo_backend_delegation.sh "${BUILD_TOOL}"
+
   test-coreml-delegate:
     name: test-coreml-delegate
     uses: pytorch/test-infra/.github/workflows/macos_job.yml@main

examples/portable/scripts/test_demo_backend_delegation.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -exu
+
+which "${PYTHON_EXECUTABLE}"
+
+BUILD_TOOL=$1
+if [[ -z "${BUILD_TOOL:-}" ]]; then
+  echo "Missing build tool (require buck2 or cmake), exiting..."
+  exit 1
+fi
+
+CMAKE_OUTPUT_DIR=cmake-out
+
+build_cmake_executor_runner() {
+  echo "Building executor_runner"
+  (rm -rf ${CMAKE_OUTPUT_DIR} \
+    && mkdir ${CMAKE_OUTPUT_DIR} \
+    && cd ${CMAKE_OUTPUT_DIR} \
+    && retry cmake -DBUCK2=buck2 \
+      -DPYTHON_EXECUTABLE="$PYTHON_EXECUTABLE" ..)
+
+  cmake --build ${CMAKE_OUTPUT_DIR} -j4
+}
+
+test_demo_backend_delegation() {
+  echo "Testing demo backend delegation on AddMul"
+  "${PYTHON_EXECUTABLE}" -m examples.portable.scripts.export_and_delegate  --option "composite"
+  "${PYTHON_EXECUTABLE}" -m examples.portable.scripts.export_and_delegate  --option "partition"
+  "${PYTHON_EXECUTABLE}" -m examples.portable.scripts.export_and_delegate  --option "whole"
+
+  # Run test model
+  if [[ "${BUILD_TOOL}" == "buck2" ]]; then
+    buck2 run //examples/portable/executor_runner:executor_runner -- --model_path "./composite_model.pte"
+    buck2 run //examples/portable/executor_runner:executor_runner -- --model_path "./partition_lowered_model.pte"
+    buck2 run //examples/portable/executor_runner:executor_runner -- --model_path "./whole.pte"
+  elif [[ "${BUILD_TOOL}" == "cmake" ]]; then
+    if [[ ! -f ${CMAKE_OUTPUT_DIR}/executor_runner ]]; then
+      build_cmake_executor_runner
+    fi
+    ./${CMAKE_OUTPUT_DIR}/executor_runner --model_path "./composite_model.pte"
+    ./${CMAKE_OUTPUT_DIR}/executor_runner --model_path "./partition_lowered_model.pte"
+    ./${CMAKE_OUTPUT_DIR}/executor_runner --model_path "./whole.pte"
+  else
+    echo "Invalid build tool ${BUILD_TOOL}. Only buck2 and cmake are supported atm"
+    exit 1
+  fi
+}
+
+test_demo_backend_delegation