Arm backend: Test TOSA, Ethos-U55 and Ethos-U85 on github

This will run some more models in the github test flow and
also enable some unit tests to use Corston3x0 FVP.

This structure up the Arm backend testing into separate runable
scripts in the same structure as other backends.

Signed-off-by: Zingo Andersen <[email protected]>
Change-Id: I5e11b1aca19460845e330b84d0696513c400c0f0

Author: zingo

.ci/scripts/setup-arm_baremetal.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+# Copyright 2024 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# NB: This function could be used to install Arm dependencies
+# Setup arm example environment (including TOSA tools)
+git config --global user.email "[email protected]"
+git config --global user.name "Github Executorch"
+bash examples/arm/setup.sh --i-agree-to-the-contained-eula

.ci/scripts/test_arm_baremetal.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+# Copyright 2024 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+TEST_SUITE=$1
+
+if [[ -z "${TEST_SUITE:-}" ]]; then
+  echo "Missing test suite name, exiting..."
+  exit 1
+else
+  echo "Run Arm baremetal test suite ${TEST_SUITE}"
+fi
+
+TEST_SUITE_NAME="$(basename "$0") ${TEST_SUITE}"
+
+test_pytest() {
+    echo "${TEST_SUITE_NAME}: Run pytest"
+
+    source examples/arm/ethos-u-scratch/setup_path.sh
+
+    # Run arm baremetal pytest tests without Corstone-300/Corstone-320 FVP
+    pytest -c /dev/null -v -n auto backends/arm/test
+}
+
+test_pytest_ethosu_fvp() {
+    echo "${TEST_SUITE_NAME}: Run pytest with fvp"
+
+    source examples/arm/ethos-u-scratch/setup_path.sh
+
+    # Prepare Corstone-3x0 FVP for pytest
+    examples/arm/run.sh --model_name=add --build_only
+    backends/arm/test/setup_testing.sh
+
+    # Run arm baremetal pytests with Corstone-300/Corstone-320 FVP
+    pytest -c /dev/null -v -n auto backends/arm/test -p executorch.backends.arm.test.common --arm_quantize_io --arm_run_corstone300
+}
+
+test_run_ethosu_fvp() {
+    echo "${TEST_SUITE_NAME}: Test ethos-u delegate examples with run.sh"
+
+    source examples/arm/ethos-u-scratch/setup_path.sh
+
+    # TOSA quantized
+    echo "${TEST_SUITE_NAME}: Test ethos-u target TOSA"
+    PYTHON_EXECUTABLE=python bash examples/arm/run.sh --target=TOSA --model_name=mv2
+    PYTHON_EXECUTABLE=python bash examples/arm/run.sh --target=TOSA --model_name=esdr
+    PYTHON_EXECUTABLE=python bash examples/arm/run.sh --target=TOSA --model_name=emformer_join
+    PYTHON_EXECUTABLE=python bash examples/arm/run.sh --target=TOSA --model_name=w2l
+
+    # Ethos-U55
+    echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U55"
+    PYTHON_EXECUTABLE=python bash examples/arm/run.sh --target=ethos-u55-128 --model_name=mv2
+
+    # Ethos-U85
+    echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U85"
+    PYTHON_EXECUTABLE=python bash examples/arm/run.sh --target=ethos-u85-128 --model_name=mv2
+    }
+
+${TEST_SUITE}

.ci/scripts/utils.sh

@@ -55,17 +55,6 @@ install_flatc_from_source() {
   popd || return
 }
 
-install_arm() {
-  # NB: This function could be used to install Arm dependencies
-  # Setup arm example environment (including TOSA tools)
-  git config --global user.email "[email protected]"
-  git config --global user.name "Github Executorch"
-  bash examples/arm/setup.sh --i-agree-to-the-contained-eula
-
-  # Test tosa_reference flow
-  source examples/arm/ethos-u-scratch/setup_path.sh
-}
-
 build_executorch_runner_buck2() {
   # Build executorch runtime with retry as this step is flaky on macos CI
   retry buck2 build //examples/portable/executor_runner:executor_runner

.github/workflows/pull.yml

@@ -354,13 +354,11 @@ jobs:
         EXECUTORCH_BUILD_ARM_BAREMETAL=ON \
         .ci/scripts/setup-linux.sh "${BUILD_TOOL}"
 
-        source .ci/scripts/utils.sh
         # Install Arm dependencies
-        install_arm
-
-        # Run pytest with coverage
-        pytest -c /dev/null -v -n auto --cov=./ --cov-report=xml backends/arm/test
-
+        .ci/scripts/setup-arm_baremetal.sh
+        
+        # Run pytest
+        .ci/scripts/test_arm_baremetal.sh test_pytest
 
   test-llama-runner-qnn-linux:
     name: test-llama-runner-qnn-linux

.github/workflows/trunk.yml

@@ -146,14 +146,15 @@ jobs:
         source .ci/scripts/utils.sh
         install_executorch
 
-        install_arm
-
         # Increase number of files user can monitor to bypass buck failures.
         # Hopefully this is high enough for this setup.
         sudo sysctl fs.inotify.max_user_watches=1048576 # 1024 * 1024
 
+        .ci/scripts/setup-arm_baremetal.sh
+
         # Test ethos-u delegate examples with run.sh
-        PYTHON_EXECUTABLE=python bash examples/arm/run.sh examples/arm/ethos-u-scratch/
+        .ci/scripts/test_arm_baremetal.sh test_run_ethosu_fvp
+
 
   test-arm-reference-delegation:
     name: test-arm-reference-delegation
@@ -172,10 +173,10 @@ jobs:
         source .ci/scripts/utils.sh
         install_executorch
 
-        install_arm
+        .ci/scripts/setup-arm_baremetal.sh
 
-        # Run arm unit tests
-        pytest -c /dev/null -v -n auto --cov=./ --cov-report=xml backends/arm/test
+        # Run arm pytest tests with Corstone3x0 FVP testing
+        .ci/scripts/test_arm_baremetal.sh test_pytest_ethosu_fvp
 
   test-coreml-delegate:
     name: test-coreml-delegate