Skip to content

Commit f4728f4

Browse files
zingofacebook-github-bot
authored andcommitted
Add all relevant testcases for Arm Ethos-U85 (#5346)
Summary: Add separate tests for Ethos-U85 to all backend operator tests. Updated ethos-u-vela version to support more operators. Signed-off-by: Per Åstrand <[[email protected]](mailto:[email protected])> Signed-off-by: Tom Allsop <[[email protected]](mailto:[email protected])> Pull Request resolved: #5346 Reviewed By: manuelcandales Differential Revision: D62875027 Pulled By: digantdesai fbshipit-source-id: 3bf238d81957258ee93ae235d575beff8a575191
1 parent 5a984cc commit f4728f4

29 files changed

+637
-135
lines changed

backends/arm/arm_backend.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,8 @@ def __init__(self):
5252
def ethosu_compile_spec(
5353
self,
5454
config: str,
55-
system_config: Optional[str] = None,
56-
memory_mode: Optional[str] = None,
55+
system_config: str,
56+
memory_mode: str,
5757
extra_flags: Optional[str] = None,
5858
config_ini: Optional[str] = "Arm/vela.ini",
5959
) -> "ArmCompileSpecBuilder":

backends/arm/test/models/test_mobilenet_v2_arm.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,3 +102,18 @@ def test_mv2_u55_BI(self):
102102
tester.run_method_and_compare_outputs(
103103
atol=1.0, qtol=1, inputs=self.model_inputs
104104
)
105+
106+
def test_mv2_u85_BI(self):
107+
(
108+
ArmTester(
109+
self.mv2,
110+
example_inputs=self.model_inputs,
111+
compile_spec=common.get_u85_compile_spec(permute_memory_to_nhwc=True),
112+
)
113+
.quantize()
114+
.export()
115+
.to_edge(config=self._edge_compile_config)
116+
.check(list(self.operators_after_quantization))
117+
.partition()
118+
.to_executorch()
119+
)

backends/arm/test/ops/test_add.py

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
from executorch.backends.arm.test import common
1414
from executorch.backends.arm.test.tester.arm_tester import ArmTester
1515
from executorch.exir import EdgeCompileConfig
16+
from executorch.exir.backend.compile_spec_schema import CompileSpec
1617
from parameterized import parameterized
1718

1819

@@ -92,16 +93,17 @@ def _test_add_tosa_BI_pipeline(
9293
.run_method_and_compare_outputs(inputs=test_data, qtol=1)
9394
)
9495

95-
def _test_add_u55_BI_pipeline(
96+
def _test_add_ethos_BI_pipeline(
9697
self,
9798
module: torch.nn.Module,
99+
compile_spec: CompileSpec,
98100
test_data: Tuple[torch.Tensor],
99101
):
100102
tester = (
101103
ArmTester(
102104
module,
103105
example_inputs=test_data,
104-
compile_spec=common.get_u55_compile_spec(permute_memory_to_nhwc=True),
106+
compile_spec=compile_spec,
105107
)
106108
.quantize()
107109
.export()
@@ -114,8 +116,7 @@ def _test_add_u55_BI_pipeline(
114116
.serialize()
115117
)
116118

117-
if common.is_option_enabled("corstone300"):
118-
tester.run_method_and_compare_outputs(qtol=1, inputs=test_data)
119+
return tester
119120

120121
@parameterized.expand(Add.test_parameters)
121122
def test_add_tosa_MI(self, test_data: torch.Tensor):
@@ -130,7 +131,22 @@ def test_add_tosa_BI(self, test_data: torch.Tensor):
130131
@parameterized.expand(Add.test_parameters)
131132
def test_add_u55_BI(self, test_data: torch.Tensor):
132133
test_data = (test_data,)
133-
self._test_add_u55_BI_pipeline(self.Add(), test_data)
134+
tester = self._test_add_ethos_BI_pipeline(
135+
self.Add(),
136+
common.get_u55_compile_spec(permute_memory_to_nhwc=True),
137+
test_data,
138+
)
139+
if common.is_option_enabled("corstone300"):
140+
tester.run_method_and_compare_outputs(qtol=1, inputs=test_data)
141+
142+
@parameterized.expand(Add.test_parameters)
143+
def test_add_u85_BI(self, test_data: torch.Tensor):
144+
test_data = (test_data,)
145+
self._test_add_ethos_BI_pipeline(
146+
self.Add(),
147+
common.get_u85_compile_spec(permute_memory_to_nhwc=True),
148+
test_data,
149+
)
134150

135151
@parameterized.expand(Add2.test_parameters)
136152
def test_add2_tosa_MI(self, operand1: torch.Tensor, operand2: torch.Tensor):
@@ -145,4 +161,15 @@ def test_add2_tosa_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
145161
@parameterized.expand(Add2.test_parameters)
146162
def test_add2_u55_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
147163
test_data = (operand1, operand2)
148-
self._test_add_u55_BI_pipeline(self.Add2(), test_data)
164+
tester = self._test_add_ethos_BI_pipeline(
165+
self.Add2(), common.get_u55_compile_spec(), test_data
166+
)
167+
if common.is_option_enabled("corstone300"):
168+
tester.run_method_and_compare_outputs(qtol=1, inputs=test_data)
169+
170+
@parameterized.expand(Add2.test_parameters)
171+
def test_add2_u85_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
172+
test_data = (operand1, operand2)
173+
self._test_add_ethos_BI_pipeline(
174+
self.Add2(), common.get_u85_compile_spec(), test_data
175+
)

backends/arm/test/ops/test_avg_pool.py

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import torch
1414
from executorch.backends.arm.test import common
1515
from executorch.backends.arm.test.tester.arm_tester import ArmTester
16+
from executorch.exir.backend.backend_details import CompileSpec
1617
from parameterized import parameterized
1718

1819
logger = logging.getLogger(__name__)
@@ -86,14 +87,17 @@ def _test_avgpool2d_tosa_BI_pipeline(
8687
.run_method_and_compare_outputs(inputs=test_data, qtol=1)
8788
)
8889

89-
def _test_avgpool2d_tosa_u55_BI_pipeline(
90-
self, module: torch.nn.Module, test_data: Tuple[torch.tensor]
90+
def _test_avgpool2d_tosa_ethos_BI_pipeline(
91+
self,
92+
module: torch.nn.Module,
93+
compile_spec: CompileSpec,
94+
test_data: Tuple[torch.tensor],
9195
):
9296
(
9397
ArmTester(
9498
module,
9599
example_inputs=test_data,
96-
compile_spec=common.get_u55_compile_spec(permute_memory_to_nhwc=True),
100+
compile_spec=compile_spec,
97101
)
98102
.quantize()
99103
.export()
@@ -141,6 +145,22 @@ def test_avgpool2d_tosa_u55_BI(
141145
test_data: torch.Tensor,
142146
model_params: int | Tuple[int, int],
143147
):
144-
self._test_avgpool2d_tosa_u55_BI_pipeline(
145-
self.AvgPool2d(*model_params), (test_data,)
148+
self._test_avgpool2d_tosa_ethos_BI_pipeline(
149+
self.AvgPool2d(*model_params),
150+
common.get_u55_compile_spec(permute_memory_to_nhwc=True),
151+
(test_data,),
152+
)
153+
154+
@parameterized.expand(test_data_suite)
155+
@unittest.expectedFailure
156+
def test_avgpool2d_tosa_u85_BI(
157+
self,
158+
test_name: str,
159+
test_data: torch.Tensor,
160+
model_params: int | Tuple[int, int],
161+
):
162+
self._test_avgpool2d_tosa_ethos_BI_pipeline(
163+
self.AvgPool2d(*model_params),
164+
common.get_u85_compile_spec(permute_memory_to_nhwc=True),
165+
(test_data,),
146166
)

backends/arm/test/ops/test_bmm.py

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import torch
1212
from executorch.backends.arm.test import common
1313
from executorch.backends.arm.test.tester.arm_tester import ArmTester
14+
from executorch.exir.backend.compile_spec_schema import CompileSpec
1415
from parameterized import parameterized
1516

1617
torch.manual_seed(1)
@@ -83,14 +84,17 @@ def _test_bmm_tosa_BI_pipeline(
8384
.run_method_and_compare_outputs(inputs=test_data)
8485
)
8586

86-
def _test_bmm_u55_BI_pipeline(
87-
self, module: torch.nn.Module, test_data: Tuple[torch.Tensor, ...]
87+
def _test_bmm_ethosu_BI_pipeline(
88+
self,
89+
module: torch.nn.Module,
90+
compile_spec: CompileSpec,
91+
test_data: Tuple[torch.Tensor, ...],
8892
):
8993
(
9094
ArmTester(
9195
module,
9296
example_inputs=test_data,
93-
compile_spec=common.get_u55_compile_spec(),
97+
compile_spec=compile_spec,
9498
)
9599
.quantize()
96100
.export()
@@ -132,4 +136,13 @@ def test_bmm_u55_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
132136
@unittest.expectedFailure
133137
def test_bmm_single_input_u55_BI(self, operand1: torch.Tensor):
134138
test_data = (operand1,)
135-
self._test_bmm_u55_BI_pipeline(self.BMMSingleInput(), test_data)
139+
self._test_bmm_ethosu_BI_pipeline(
140+
self.BMMSingleInput(), common.get_u55_compile_spec(), test_data
141+
)
142+
143+
@parameterized.expand(BMMSingleInput.test_parameters)
144+
def test_bmm_single_input_u85_BI(self, operand1: torch.Tensor):
145+
test_data = (operand1,)
146+
self._test_bmm_ethosu_BI_pipeline(
147+
self.BMMSingleInput(), common.get_u85_compile_spec(), test_data
148+
)

backends/arm/test/ops/test_cat.py

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
from executorch.backends.arm.test import common
1414

1515
from executorch.backends.arm.test.tester.arm_tester import ArmTester
16+
from executorch.exir.backend.compile_spec_schema import CompileSpec
1617
from parameterized import parameterized
1718

1819

@@ -89,14 +90,17 @@ def _test_cat_tosa_BI_pipeline(
8990
.run_method_and_compare_outputs(inputs=test_data, qtol=1)
9091
)
9192

92-
def _test_cat_u55_BI_pipeline(
93-
self, module: torch.nn.Module, test_data: Tuple[tuple[torch.Tensor, ...], int]
93+
def _test_cat_ethosu_BI_pipeline(
94+
self,
95+
module: torch.nn.Module,
96+
compile_spec: CompileSpec,
97+
test_data: Tuple[tuple[torch.Tensor, ...], int],
9498
):
9599
(
96100
ArmTester(
97101
module,
98102
example_inputs=test_data,
99-
compile_spec=common.get_u55_compile_spec(),
103+
compile_spec=compile_spec,
100104
)
101105
.quantize()
102106
.export()
@@ -125,9 +129,16 @@ def test_cat_tosa_BI(self, operands: tuple[torch.Tensor, ...], dim: int):
125129
test_data = (operands, dim)
126130
self._test_cat_tosa_BI_pipeline(self.Cat(), test_data)
127131

128-
# TODO: Remove @unittest.expectedFailure when this issue is fixed in Regor
129132
@parameterized.expand(Cat.test_parameters)
130-
@unittest.expectedFailure
131133
def test_cat_u55_BI(self, operands: tuple[torch.Tensor, ...], dim: int):
132134
test_data = (operands, dim)
133-
self._test_cat_u55_BI_pipeline(self.Cat(), test_data)
135+
self._test_cat_ethosu_BI_pipeline(
136+
self.Cat(), common.get_u55_compile_spec(), test_data
137+
)
138+
139+
@parameterized.expand(Cat.test_parameters)
140+
def test_cat_u85_BI(self, operands: tuple[torch.Tensor, ...], dim: int):
141+
test_data = (operands, dim)
142+
self._test_cat_ethosu_BI_pipeline(
143+
self.Cat(), common.get_u85_compile_spec(), test_data
144+
)

backends/arm/test/ops/test_clone.py

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
from executorch.backends.arm.test.tester.arm_tester import ArmTester
2222

2323
from executorch.backends.xnnpack.test.tester.tester import Quantize
24+
25+
from executorch.exir.backend.compile_spec_schema import CompileSpec
2426
from parameterized import parameterized
2527

2628

@@ -76,16 +78,15 @@ def _test_clone_tosa_BI_pipeline(
7678
.run_method_and_compare_outputs(inputs=test_data, qtol=1)
7779
)
7880

79-
def _test_clone_tosa_u55_pipeline(
80-
self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]
81+
def _test_clone_tosa_ethos_pipeline(
82+
self,
83+
compile_spec: list[CompileSpec],
84+
module: torch.nn.Module,
85+
test_data: Tuple[torch.Tensor],
8186
):
8287
quantizer = ArmQuantizer().set_io(get_symmetric_quantization_config())
8388
(
84-
ArmTester(
85-
module,
86-
example_inputs=test_data,
87-
compile_spec=common.get_u55_compile_spec(),
88-
)
89+
ArmTester(module, example_inputs=test_data, compile_spec=compile_spec)
8990
.quantize(Quantize(quantizer, get_symmetric_quantization_config()))
9091
.export()
9192
.check_count({"torch.ops.aten.clone.default": 1})
@@ -95,6 +96,20 @@ def _test_clone_tosa_u55_pipeline(
9596
.to_executorch()
9697
)
9798

99+
def _test_clone_tosa_u55_pipeline(
100+
self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]
101+
):
102+
self._test_clone_tosa_ethos_pipeline(
103+
common.get_u55_compile_spec(), module, test_data
104+
)
105+
106+
def _test_clone_tosa_u85_pipeline(
107+
self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]
108+
):
109+
self._test_clone_tosa_ethos_pipeline(
110+
common.get_u85_compile_spec(), module, test_data
111+
)
112+
98113
@parameterized.expand(Clone.test_parameters)
99114
def test_clone_tosa_MI(self, test_tensor: torch.Tensor):
100115
self._test_clone_tosa_MI_pipeline(self.Clone(), (test_tensor,))
@@ -106,3 +121,7 @@ def test_clone_tosa_BI(self, test_tensor: torch.Tensor):
106121
@parameterized.expand(Clone.test_parameters)
107122
def test_clone_u55_BI(self, test_tensor: torch.Tensor):
108123
self._test_clone_tosa_u55_pipeline(self.Clone(), (test_tensor,))
124+
125+
@parameterized.expand(Clone.test_parameters)
126+
def test_clone_u85_BI(self, test_tensor: torch.Tensor):
127+
self._test_clone_tosa_u85_pipeline(self.Clone(), (test_tensor,))

backends/arm/test/ops/test_conv.py

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
from executorch.backends.arm.test import common
1313

1414
from executorch.backends.arm.test.tester.arm_tester import ArmTester
15+
from executorch.exir.backend.compile_spec_schema import CompileSpec
1516
from parameterized import parameterized
1617

1718

@@ -297,14 +298,17 @@ def _test_conv2d_tosa_BI_pipeline(
297298
.run_method_and_compare_outputs(inputs=test_data, qtol=1)
298299
)
299300

300-
def _test_conv2d_u55_BI_pipeline(
301-
self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]
301+
def _test_conv2d_ethosu_BI_pipeline(
302+
self,
303+
compile_spec: CompileSpec,
304+
module: torch.nn.Module,
305+
test_data: Tuple[torch.Tensor],
302306
):
303307
(
304308
ArmTester(
305309
module,
306310
example_inputs=test_data,
307-
compile_spec=common.get_u55_compile_spec(permute_memory_to_nhwc=True),
311+
compile_spec=compile_spec,
308312
)
309313
.quantize()
310314
.export()
@@ -325,4 +329,16 @@ def test_conv2d_tosa_BI(self, test_name, model):
325329

326330
@parameterized.expand(testsuite_u55)
327331
def test_conv2d_u55_BI(self, test_name, model):
328-
self._test_conv2d_u55_BI_pipeline(model, model.get_inputs())
332+
self._test_conv2d_ethosu_BI_pipeline(
333+
common.get_u55_compile_spec(permute_memory_to_nhwc=True),
334+
model,
335+
model.get_inputs(),
336+
)
337+
338+
@parameterized.expand(testsuite_u55)
339+
def test_conv2d_u85_BI(self, test_name, model):
340+
self._test_conv2d_ethosu_BI_pipeline(
341+
common.get_u85_compile_spec(permute_memory_to_nhwc=True),
342+
model,
343+
model.get_inputs(),
344+
)

0 commit comments

Comments
 (0)