Arm backend: Test TOSA, Ethos-U55 and Ethos-U85 on github

This will run some more models in the github test flow and
also enable some unit tests to use Corston3x0 FVP.

This structure up the Arm backend testing into separate runable
scripts in the same structure as other backends.

Signed-off-by: Zingo Andersen <[email protected]>
Change-Id: I5e11b1aca19460845e330b84d0696513c400c0f0

Author: zingo

.ci/scripts/setup-arm_baremetal.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# Copyright 2024 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# NB: This function could be used to install Arm dependencies
+# Setup arm example environment (including TOSA tools)
+git config --global user.email "[email protected]"
+git config --global user.name "Github Executorch"
+bash examples/arm/setup.sh --i-agree-to-the-contained-eula
+

.ci/scripts/test_arm_baremetal.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+# Copyright 2024 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+TEST_SUITE=$1
+
+if [[ -z "${TEST_SUITE:-}" ]]; then
+  echo "Missing test suite name, exiting..."
+  exit 1
+else
+  echo "Run Arm baremetal test suite ${TEST_SUITE}"
+fi
+
+TEST_SUITE_NAME="$(basename "$0") ${TEST_SUITE}"
+
+test_pytest() {
+    echo "${TEST_SUITE_NAME}: Run pytest"
+    # Run arm baremetal pytest tests without FVP
+    pytest -c /dev/null -v -n auto backends/arm/test
+}
+
+test_pytest_ethosu_fvp() {
+    echo "${TEST_SUITE_NAME}: Run pytest with fvp"
+    # Run arm baremetal pytest tests with FVP
+    pytest -c /dev/null -v -n auto backends/arm/test -p executorch.backends.arm.test.common --arm_quantize_io --arm_run_corstone300
+}
+
+test_run_ethosu_fvp() {
+    echo "${TEST_SUITE_NAME}: Test ethos-u delegate examples with run.sh"
+
+    # TOSA quantized
+    echo "${TEST_SUITE_NAME}: Test ethos-u target TOSA"
+    PYTHON_EXECUTABLE=python bash examples/arm/run.sh --target=TOSA --model_name=mv2
+    PYTHON_EXECUTABLE=python bash examples/arm/run.sh --target=TOSA --model_name=esdr
+    PYTHON_EXECUTABLE=python bash examples/arm/run.sh --target=TOSA --model_name=emformer_join
+    PYTHON_EXECUTABLE=python bash examples/arm/run.sh --target=TOSA --model_name=w2l
+
+    # Ethos-U55
+    echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U55"
+    PYTHON_EXECUTABLE=python bash examples/arm/run.sh --target=ethos-u55-128 --model_name=mv2
+
+    # Ethos-U85
+    echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U85"
+    PYTHON_EXECUTABLE=python bash examples/arm/run.sh --target=ethos-u85-128 --model_name=mv2
+    }
+
+${TEST_SUITE}

.ci/scripts/utils.sh

@@ -55,17 +55,6 @@ install_flatc_from_source() {
   popd || return
 }
 
-install_arm() {
-  # NB: This function could be used to install Arm dependencies
-  # Setup arm example environment (including TOSA tools)
-  git config --global user.email "[email protected]"
-  git config --global user.name "Github Executorch"
-  bash examples/arm/setup.sh --i-agree-to-the-contained-eula
-
-  # Test tosa_reference flow
-  source examples/arm/ethos-u-scratch/setup_path.sh
-}
-
 build_executorch_runner_buck2() {
   # Build executorch runtime with retry as this step is flaky on macos CI
   retry buck2 build //examples/portable/executor_runner:executor_runner

.github/workflows/pull.yml

@@ -354,13 +354,14 @@ jobs:
         EXECUTORCH_BUILD_ARM_BAREMETAL=ON \
         .ci/scripts/setup-linux.sh "${BUILD_TOOL}"
 
-        source .ci/scripts/utils.sh
         # Install Arm dependencies
-        install_arm
-
-        # Run pytest with coverage
-        pytest -c /dev/null -v -n auto --cov=./ --cov-report=xml backends/arm/test
+        .ci/scripts/setup-arm_baremetal.sh
+        
+        # Test tosa_reference flow
+        source examples/arm/ethos-u-scratch/setup_path.sh
 
+        # Run pytest
+        .ci/scripts/test_arm_baremetal.sh test_pytest
 
   test-llama-runner-qnn-linux:
     name: test-llama-runner-qnn-linux

.github/workflows/trunk.yml

@@ -146,14 +146,17 @@ jobs:
         source .ci/scripts/utils.sh
         install_executorch
 
-        install_arm
+        .ci/scripts/setup-arm_baremetal.sh
+        # Test tosa_reference flow
+        source examples/arm/ethos-u-scratch/setup_path.sh
 
         # Increase number of files user can monitor to bypass buck failures.
         # Hopefully this is high enough for this setup.
         sudo sysctl fs.inotify.max_user_watches=1048576 # 1024 * 1024
 
         # Test ethos-u delegate examples with run.sh
-        PYTHON_EXECUTABLE=python bash examples/arm/run.sh examples/arm/ethos-u-scratch/
+        .ci/scripts/test_arm_baremetal.sh test_run_ethosu_fvp
+
 
   test-arm-reference-delegation:
     name: test-arm-reference-delegation
@@ -172,10 +175,12 @@ jobs:
         source .ci/scripts/utils.sh
         install_executorch
 
-        install_arm
+        .ci/scripts/setup-arm_baremetal.sh
+        # Test tosa_reference flow
+        source examples/arm/ethos-u-scratch/setup_path.sh
 
         # Run arm unit tests
-        pytest -c /dev/null -v -n auto --cov=./ --cov-report=xml backends/arm/test
+        .ci/scripts/test_arm_baremetal.sh test_pytest_ethosu_fvp
 
   test-coreml-delegate:
     name: test-coreml-delegate