Arm Backend: Update unit tests for TOSA 1.0

backends/arm/scripts/parse_test_names.py

@@ -5,7 +5,15 @@
 from executorch.exir.dialects.edge.spec.utils import SAMPLE_INPUT
 
 # Add edge ops which we lower but which are not included in exir/dialects/edge/edge.yaml here.
-CUSTOM_EDGE_OPS = ["linspace.default", "eye.default"]
+CUSTOM_EDGE_OPS = [
+    "linspace.default",
+    "eye.default",
+    "hardsigmoid.default",
+    "hardswish.default",
+    "linear.default",
+    "maximum.default",
+    "adaptive_avg_pool2d.default",
+]
 ALL_EDGE_OPS = SAMPLE_INPUT.keys() | CUSTOM_EDGE_OPS
 
 # Add all targets and TOSA profiles we support here.

backends/arm/test/common.py

@@ -259,17 +259,15 @@ def decorator_func(func):
                     raise RuntimeError(
                         "xfail info needs to be str, or tuple[str, type[Exception]]"
                     )
-                pytest_param = pytest.param(
-                    test_parameters,
-                    id=id,
-                    marks=pytest.mark.xfail(
-                        reason=reason, raises=raises, strict=strict
-                    ),
+                # Set up our fail marker
+                marker = (
+                    pytest.mark.xfail(reason=reason, raises=raises, strict=strict),
                 )
             else:
-                pytest_param = pytest.param(test_parameters, id=id)
-            pytest_testsuite.append(pytest_param)
+                marker = ()
 
+            pytest_param = pytest.param(test_parameters, id=id, marks=marker)
+            pytest_testsuite.append(pytest_param)
         return pytest.mark.parametrize(arg_name, pytest_testsuite)(func)
 
     return decorator_func

backends/arm/test/conftest.py

@@ -12,12 +12,6 @@
 
 import pytest
 
-try:
-    import tosa_tools.v0_80.tosa_reference_model as tosa_reference_model
-except ImportError:
-    logging.warning("tosa_reference_model not found, can't run reference model tests")
-    tosa_reference_model = None
-
 """
 This file contains the pytest hooks, fixtures etc. for the Arm test suite.
 """
@@ -50,10 +44,11 @@ def pytest_configure(config):
     if getattr(config.option, "fast_fvp", False):
         pytest._test_options["fast_fvp"] = config.option.fast_fvp  # type: ignore[attr-defined]
 
-    # TODO: remove this flag once we have a way to run the reference model tests with Buck
-    pytest._test_options["tosa_ref_model"] = False  # type: ignore[attr-defined]
-    if tosa_reference_model is not None:
-        pytest._test_options["tosa_ref_model"] = True  # type: ignore[attr-defined]
+    if config.option.arm_run_tosa_version:
+        pytest._test_options["tosa_version"] = config.option.arm_run_tosa_version
+
+    pytest._test_options["tosa_ref_model"] = True  # type: ignore[attr-defined]
+
     logging.basicConfig(level=logging.INFO, stream=sys.stdout)
 
 
@@ -76,6 +71,7 @@ def try_addoption(*args, **kwargs):
         nargs="+",
         help="List of two files. Firstly .pt file. Secondly .json",
     )
+    try_addoption("--arm_run_tosa_version", action="store", default="0.80")
 
 
 def pytest_sessionstart(session):

backends/arm/test/ops/test_abs.py

@@ -1,125 +1,68 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates.
-# Copyright 2025 Arm Limited and/or its affiliates.
 # All rights reserved.
+# Copyright 2025 Arm Limited and/or its affiliates.
 #
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-import unittest
 
 from typing import Tuple
 
-import pytest
-
 import torch
-from executorch.backends.arm.test import common, conftest
-from executorch.backends.arm.test.tester.arm_tester import ArmTester
-from executorch.exir.backend.compile_spec_schema import CompileSpec
-from parameterized import parameterized
-
-
-class TestAbs(unittest.TestCase):
-    class Abs(torch.nn.Module):
-        test_parameters = [
-            (torch.zeros(5),),
-            (torch.full((5,), -1, dtype=torch.float32),),
-            (torch.ones(5) * -1,),
-            (torch.randn(8),),
-            (torch.randn(2, 3, 4),),
-            (torch.randn(1, 2, 3, 4),),
-            (torch.normal(mean=0, std=10, size=(2, 3, 4)),),
-        ]
-
-        def forward(self, x):
-            return torch.abs(x)
-
-    def _test_abs_tosa_MI_pipeline(
-        self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]
-    ):
-        (
-            ArmTester(
-                module,
-                example_inputs=test_data,
-                compile_spec=common.get_tosa_compile_spec("TOSA-0.80+MI"),
-            )
-            .export()
-            .check_count({"torch.ops.aten.abs.default": 1})
-            .check_not(["torch.ops.quantized_decomposed"])
-            .to_edge()
-            .partition()
-            .check_not(["torch.ops.aten.abs.default"])
-            .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
-            .to_executorch()
-            .run_method_and_compare_outputs(inputs=test_data)
-        )
-
-    def _test_abs_tosa_BI_pipeline(
-        self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]
-    ):
-        (
-            ArmTester(
-                module,
-                example_inputs=test_data,
-                compile_spec=common.get_tosa_compile_spec("TOSA-0.80+BI"),
-            )
-            .quantize()
-            .export()
-            .check_count({"torch.ops.aten.abs.default": 1})
-            .check(["torch.ops.quantized_decomposed"])
-            .to_edge()
-            .partition()
-            .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
-            .to_executorch()
-            .run_method_and_compare_outputs(inputs=test_data, qtol=1)
-        )
-
-    def _test_abs_ethosu_BI_pipeline(
-        self,
-        compile_spec: list[CompileSpec],
-        module: torch.nn.Module,
-        test_data: Tuple[torch.Tensor],
-    ):
-        tester = (
-            ArmTester(
-                module,
-                example_inputs=test_data,
-                compile_spec=compile_spec,
-            )
-            .quantize()
-            .export()
-            .check_count({"torch.ops.aten.abs.default": 1})
-            .check(["torch.ops.quantized_decomposed"])
-            .to_edge()
-            .partition()
-            .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
-            .to_executorch()
-            .serialize()
-        )
-        if conftest.is_option_enabled("corstone_fvp"):
-            tester.run_method_and_compare_outputs(qtol=1, inputs=test_data)
-
-    @parameterized.expand(Abs.test_parameters)
-    def test_abs_tosa_MI(self, test_data: torch.Tensor):
-        test_data = (test_data,)
-        self._test_abs_tosa_MI_pipeline(self.Abs(), test_data)
-
-    @parameterized.expand(Abs.test_parameters)
-    def test_abs_tosa_BI(self, test_data: torch.Tensor):
-        test_data = (test_data,)
-        self._test_abs_tosa_BI_pipeline(self.Abs(), test_data)
-
-    @parameterized.expand(Abs.test_parameters)
-    @pytest.mark.corstone_fvp
-    def test_abs_u55_BI(self, test_data: torch.Tensor):
-        test_data = (test_data,)
-        self._test_abs_ethosu_BI_pipeline(
-            common.get_u55_compile_spec(), self.Abs(), test_data
-        )
-
-    @parameterized.expand(Abs.test_parameters)
-    @pytest.mark.corstone_fvp
-    def test_abs_u85_BI(self, test_data: torch.Tensor):
-        test_data = (test_data,)
-        self._test_abs_ethosu_BI_pipeline(
-            common.get_u85_compile_spec(), self.Abs(), test_data
-        )
+from executorch.backends.arm.test import common
+from executorch.backends.arm.test.tester.test_pipeline import (
+    EthosU55PipelineBI,
+    EthosU85PipelineBI,
+    TosaPipelineBI,
+    TosaPipelineMI,
+)
+
+aten_op = "torch.ops.aten.abs.default"
+exir_op = "executorch_exir_dialects_edge__ops_aten_abs_default"
+
+input_t1 = Tuple[torch.Tensor]  # Input x
+
+
+class Abs(torch.nn.Module):
+    test_parameters = {
+        "zeros": lambda: (torch.zeros(5),),
+        "full": lambda: (torch.full((5,), -1, dtype=torch.float32),),
+        "ones": lambda: (torch.ones(5) * -1,),
+        "randn_1d": lambda: (torch.randn(8),),
+        "randn_3d": lambda: (torch.randn(2, 3, 4),),
+        "randn_4d": lambda: (torch.randn(1, 2, 3, 4),),
+        "torch_normal": lambda: (torch.normal(mean=0, std=10, size=(2, 3, 4)),),
+    }
+
+    def forward(self, x):
+        return torch.abs(x)
+
+
+@common.parametrize("test_data", Abs.test_parameters)
+def test_abs_tosa_MI(test_data: torch.Tensor):
+    pipeline = TosaPipelineMI[input_t1](Abs(), test_data(), aten_op, exir_op)
+    pipeline.run()
+
+
+@common.parametrize("test_data", Abs.test_parameters)
+def test_abs_tosa_BI(test_data: torch.Tensor):
+    pipeline = TosaPipelineBI[input_t1](Abs(), test_data(), aten_op, exir_op)
+    pipeline.run()
+
+
+@common.parametrize("test_data", Abs.test_parameters)
+@common.XfailIfNoCorstone300
+def test_abs_u55_BI(test_data: torch.Tensor):
+    pipeline = EthosU55PipelineBI[input_t1](
+        Abs(), test_data(), aten_op, exir_op, run_on_fvp=True
+    )
+    pipeline.run()
+
+
+@common.parametrize("test_data", Abs.test_parameters)
+@common.XfailIfNoCorstone320
+def test_abs_u85_BI(test_data: torch.Tensor):
+    pipeline = EthosU85PipelineBI[input_t1](
+        Abs(), test_data(), aten_op, exir_op, run_on_fvp=True
+    )
+    pipeline.run()