Arm backend: Test TOSA, Ethos-U55 and Ethos-U85 on github

.ci/scripts/setup-arm-baremetal-tools.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+# Copyright 2024 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# NB: This function could be used to install Arm dependencies
+# Setup arm example environment (including TOSA tools)
+git config --global user.email "[email protected]"
+git config --global user.name "Github Executorch"
+bash examples/arm/setup.sh --i-agree-to-the-contained-eula

.ci/scripts/utils.sh

@@ -59,17 +59,6 @@ install_flatc_from_source() {
   popd || return
 }
 
-install_arm() {
-  # NB: This function could be used to install Arm dependencies
-  # Setup arm example environment (including TOSA tools)
-  git config --global user.email "[email protected]"
-  git config --global user.name "Github Executorch"
-  bash examples/arm/setup.sh --i-agree-to-the-contained-eula
-
-  # Test tosa_reference flow
-  source examples/arm/ethos-u-scratch/setup_path.sh
-}
-
 build_executorch_runner_buck2() {
   # Build executorch runtime with retry as this step is flaky on macos CI
   retry buck2 build //examples/portable/executor_runner:executor_runner

.github/workflows/pull.yml

@@ -354,13 +354,11 @@ jobs:
         EXECUTORCH_BUILD_ARM_BAREMETAL=ON \
         .ci/scripts/setup-linux.sh "${BUILD_TOOL}"
 
-        source .ci/scripts/utils.sh
         # Install Arm dependencies
-        install_arm
-
-        # Run pytest with coverage
-        pytest -c /dev/null -v -n auto --cov=./ --cov-report=xml backends/arm/test
+        .ci/scripts/setup-arm-baremetal-tools.sh
 
+        # Run pytest without simulator
+        backends/arm/test/test_arm_baremetal.sh test_pytest
 
   test-llama-runner-qnn-linux:
     name: test-llama-runner-qnn-linux

.github/workflows/trunk.yml

@@ -146,14 +146,15 @@ jobs:
         source .ci/scripts/utils.sh
         install_executorch
 
-        install_arm
+        .ci/scripts/setup-arm-baremetal-tools.sh
 
         # Increase number of files user can monitor to bypass buck failures.
         # Hopefully this is high enough for this setup.
         sudo sysctl fs.inotify.max_user_watches=1048576 # 1024 * 1024
 
         # Test ethos-u delegate examples with run.sh
-        PYTHON_EXECUTABLE=python bash examples/arm/run.sh examples/arm/ethos-u-scratch/
+        backends/arm/test/test_arm_baremetal.sh test_run_ethosu_fvp
+
 
   test-arm-reference-delegation:
     name: test-arm-reference-delegation
@@ -172,10 +173,10 @@ jobs:
         source .ci/scripts/utils.sh
         install_executorch
 
-        install_arm
+        .ci/scripts/setup-arm-baremetal-tools.sh
 
-        # Run arm unit tests
-        pytest -c /dev/null -v -n auto --cov=./ --cov-report=xml backends/arm/test
+        # Run arm unit tests using the simulator
+        backends/arm/test/test_arm_baremetal.sh test_pytest_ethosu_fvp
 
   test-coreml-delegate:
     name: test-coreml-delegate

backends/arm/README.md

@@ -39,6 +39,28 @@ Other:
 - `third-party/` - Dependencies on other code - in particular the TOSA serialization_lib for compiling to TOSA and the ethos-u-core-driver for the bare-metal backend supporting Ethos-U
 - `test/` - Unit test and test support functions
 
+## Testing
+
+After a setup you can run unit tests with the test_arm_baremetal.sh script.
+
+To run the pytests suite run
+
+```
+backends/arm/test/test_arm_baremetal.sh test_pytest
+```
+
+To run the unit test suite with Corstone3x0 FVP simulator support use
+
+```
+backends/arm/test/test_arm_baremetal.sh test_pytest_ethosu_fvp
+```
+
+You can test to run some models with the run.sh flow
+
+```
+backends/arm/test/test_arm_baremetal.sh test_run_ethosu_fvp
+```
+
 ## Unit tests
 This is the structure of the test directory
 
@@ -51,6 +73,8 @@ test                            #  Root test folder
 ├── tester                      #  Arm Tester class
 ├── tosautil                    #  Utility functions for TOSA artifacts
 ├ common.py                     #  Common functions and definitions used by many tests
+├ setup_testing.sh              #  Script to prepare testing for using the Corstone 3x0 FVP
+├ test_arm_baremetal.sh         #  Help script to trigger testing
 ```
 
 Some example commands to run these tests follow. Run a single test:
@@ -59,6 +83,12 @@ Some example commands to run these tests follow. Run a single test:
 python -m unittest backends.arm.test.ops.test_add.TestSimpleAdd -k test_add2_tosa_BI
 ```
 
+or with pytest
+
+```
+pytest -c /dev/null -v -n auto backends/arm/test/ops/test_add.py -k test_add2_tosa_BI
+```
+
 Or all tests in "TestSimpleAdd":
 
 ```
@@ -71,6 +101,28 @@ Or discover and run many tests:
 python -m unittest discover -s backends/arm/test/ops/
 ```
 
+or with pytest
+
+```
+pytest -c /dev/null -v -n auto backends/arm/test/ops/
+```
+
+
+You can run tests using Corstone3x0 simulators to see how it would work on something more target like
+first you need to build and prepare some used target libs
+
+```
+examples/arm/run.sh --model_name=add --build_only
+backends/arm/test/setup_testing.sh
+```
+
+The you can run the tests with
+
+```
+pytest -c /dev/null -v -n auto backends/arm/test --arm_quantize_io --arm_run_corstoneFVP
+```
+
+
 ### A note on unit tests
 
 There are currently 3 ways we unit test our code.

backends/arm/test/runner_utils.py

@@ -178,7 +178,7 @@ def __init__(
         self.output_name: str = None
         self.qp_input: list[QuantizationParams] = None
         self.qp_output: QuantizationParams = None
-        self.timeout = 120
+        self.timeout = 480
         self.target_board: str = None
 
         self._has_init_run = False
@@ -316,7 +316,7 @@ def run_corstone(
         result = _run_cmd(command_args[self.target_board], check=False)
         if result.returncode != 0:
             raise RuntimeError(
-                f"Failed to run {command_args[self.target_board]}\nError: {result.stderr.decode()}"
+                f"Failed to run {command_args[self.target_board]}\nOutput:\n{result.stdout.decode()}\nError: {result.stderr.decode()}"
             )
         result_stdout = result.stdout.decode()
 

backends/arm/test/setup_testing.sh

@@ -14,7 +14,6 @@ ethos_u_root_dir=${et_root_dir}/examples/arm/ethos-u-scratch/ethos-u
 toolchain_cmake=${et_root_dir}/examples/arm/ethos-u-setup/arm-none-eabi-gcc.cmake
 et_build_dir=${et_root_dir}/cmake-out
 build_root_test_dir=${et_build_dir}/arm_semihosting_executor_runner
-fvp_model=FVP_Corstone_SSE-300_Ethos-U55
 
 # Build Arm Baremetal executor_runner in semihosting mode.
 # Put in backends/arm/test/res to be used by unit tests.

backends/arm/test/test_arm_baremetal.sh

@@ -0,0 +1,106 @@
+#!/bin/bash
+# Copyright 2024 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -e
+
+script_dir=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
+
+# Executorch root
+et_root_dir=$(cd ${script_dir}/../../.. && pwd)
+cd "${et_root_dir}"
+pwd
+
+
+TEST_SUITE=$1
+
+help() {
+  echo "Usage:"
+  echo " $0 <TESTNAME>"
+  echo " where <TESTNAME> can be any of:"
+  # This will list all lines in this file that is starting with test_ remove () { and print it as a list.
+  # e,g, "test_pytest() { # Test ops and other things" -> test_pytest # Test ops and other things
+  echo "all # run all tests"
+  grep "^test_" $0 | sed 's/([^)]*)[[:space:]]*{*//g'
+  exit
+}
+
+if [[ -z "${TEST_SUITE:-}" ]]; then
+  echo "Missing test suite name, exiting..."
+  help
+else
+  echo "Run Arm baremetal test suite ${TEST_SUITE}"
+fi
+
+TEST_SUITE_NAME="$(basename "$0") ${TEST_SUITE}"
+
+all() { # Run all tests
+  # This will list all lines in this file that is starting with test_ remove () { and add this script name in 
+  # front of it and execute it in a sub shell
+  # e.g. from this file:
+  #
+  # test_pytest() { # Test ops and other things
+  #  bla bla bla
+  # }
+  # test_pytest_ethosu_fvp() { # Same as test_pytest but ...
+  #  bla bla bla
+  # }
+  #...
+  # become a small script:
+  # ----
+  # backends/arm/test/test_arm_baremetal.sh test_pytest # Test ops and other things
+  # backends/arm/test/test_arm_baremetal.sh test_pytest_ethosu_fvp # Same as test_pytest but ...
+  # ...
+  # ----
+  # That is executed
+  echo "${TEST_SUITE_NAME}: Run all tests"
+  grep "^test_" backends/arm/test/test_arm_baremetal.sh | sed 's/([^)]*)[[:space:]]*{*//g' | sed "s|^|$0 |" | sh
+}
+
+test_pytest() { # Test ops and other things
+    echo "${TEST_SUITE_NAME}: Run pytest"
+    cd "${et_root_dir}"
+    source examples/arm/ethos-u-scratch/setup_path.sh
+
+    # Run arm baremetal pytest tests without FVP
+    pytest  --verbose --color=yes --numprocesses=auto backends/arm/test/
+}
+
+test_pytest_ethosu_fvp() { # Same as test_pytest but also sometime verify using Corstone FVP
+    echo "${TEST_SUITE_NAME}: Run pytest with fvp"
+
+    source examples/arm/ethos-u-scratch/setup_path.sh
+
+    # Prepare Corstone-3x0 FVP for pytest
+    examples/arm/run.sh --model_name=add --build_only
+    backends/arm/test/setup_testing.sh
+
+    # Run arm baremetal pytest tests with FVP
+    pytest  --verbose --color=yes --numprocesses=auto backends/arm/test/ --arm_quantize_io --arm_run_corstoneFVP
+}
+
+test_run_ethosu_fvp() { # End to End model tests
+    echo "${TEST_SUITE_NAME}: Test ethos-u delegate examples with run.sh"
+
+    source examples/arm/ethos-u-scratch/setup_path.sh
+
+    # TOSA quantized
+    echo "${TEST_SUITE_NAME}: Test ethos-u target TOSA"
+    examples/arm/run.sh --target=TOSA --model_name=mv2
+    examples/arm/run.sh --target=TOSA --model_name=lstm
+    examples/arm/run.sh --target=TOSA --model_name=edsr
+
+    # Ethos-U55
+    echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U55"
+    examples/arm/run.sh --target=ethos-u55-128 --model_name=mv2
+    examples/arm/run.sh --target=ethos-u55-128 --model_name=lstm --reorder_inputs=1,0,2
+
+    # Ethos-U85
+    echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U85"
+    examples/arm/run.sh --target=ethos-u85-128 --model_name=mv2
+    examples/arm/run.sh --target=ethos-u85-128 --model_name=lstm --reorder_inputs=1,0,2
+    }
+
+${TEST_SUITE}

backends/arm/test/tester/arm_tester.py

@@ -251,7 +251,7 @@ def to_executorch(self, to_executorch_stage: Optional[ToExecutorch] | None = Non
         return super().to_executorch(to_executorch_stage)
 
     def serialize(
-        self, serialize_stage: Optional[Serialize] = None, timeout: int = 120
+        self, serialize_stage: Optional[Serialize] = None, timeout: int = 480
     ):
         if serialize_stage is None:
             serialize_stage = Serialize(self.runner_util, timeout=timeout)