Skip to content

Commit a8fa857

Browse files
Erik-Lundellfreddan80
Erik-Lundell
authored and
freddan80
committed
Add FVP testing to ops
Add expected fails accordingly Signed-off-by: Erik Lundell <[email protected]> Change-Id: Ic76626256ae4c53258536ffa747a7ee02832b168
1 parent d679ad7 commit a8fa857

19 files changed

+223
-60
lines changed

‎backends/arm/test/ops/test_avg_pool.py

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,10 @@
2323

2424
test_data_suite = [
2525
# (test_name, test_data, [kernel_size, stride, padding])
26-
("zeros", torch.zeros(20, 16, 50, 32), [4, 2, 0]),
27-
("ones", torch.zeros(20, 16, 50, 32), [4, 2, 0]),
28-
("rand", torch.rand(20, 16, 50, 32), [4, 2, 0]),
29-
("randn", torch.randn(20, 16, 50, 32), [4, 2, 0]),
26+
("zeros", torch.zeros(1, 16, 50, 32), [4, 2, 0]),
27+
("ones", torch.zeros(1, 16, 50, 32), [4, 2, 0]),
28+
("rand", torch.rand(1, 16, 50, 32), [4, 2, 0]),
29+
("randn", torch.randn(1, 16, 50, 32), [4, 2, 0]),
3030
]
3131

3232

@@ -101,7 +101,7 @@ def _test_avgpool2d_tosa_ethos_BI_pipeline(
101101
test_data: Tuple[torch.tensor],
102102
):
103103
quantizer = ArmQuantizer().set_io(get_symmetric_quantization_config())
104-
(
104+
tester = (
105105
ArmTester(
106106
module,
107107
example_inputs=test_data,
@@ -116,7 +116,10 @@ def _test_avgpool2d_tosa_ethos_BI_pipeline(
116116
.check_not(["executorch_exir_dialects_edge__ops_aten_avg_pool2d_default"])
117117
.check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
118118
.to_executorch()
119+
.serialize()
119120
)
121+
if common.is_option_enabled("corstone300"):
122+
tester.run_method_and_compare_outputs(qtol=1, inputs=test_data)
120123

121124
@parameterized.expand(test_data_suite)
122125
def test_avgpool2d_tosa_MI(

‎backends/arm/test/ops/test_bmm.py

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ def forward(self, x, y):
4141
class BMMSingleInput(torch.nn.Module):
4242
test_parameters = [
4343
(torch.rand(20, 3, 3),),
44-
(torch.ones(2, 128, 128),),
44+
(torch.rand(2, 128, 128),),
4545
(10000 * torch.randn(4, 25, 25),),
4646
(5 + 5 * torch.randn(3, 64, 64),),
4747
]
@@ -96,7 +96,7 @@ def _test_bmm_ethosu_BI_pipeline(
9696
compile_spec: CompileSpec,
9797
test_data: Tuple[torch.Tensor, ...],
9898
):
99-
(
99+
tester = (
100100
ArmTester(
101101
module,
102102
example_inputs=test_data,
@@ -110,7 +110,10 @@ def _test_bmm_ethosu_BI_pipeline(
110110
.partition()
111111
.check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
112112
.to_executorch()
113+
.serialize()
113114
)
115+
if common.is_option_enabled("corstone300"):
116+
tester.run_method_and_compare_outputs(inputs=test_data, qtol=1)
114117

115118
@parameterized.expand(BMM.test_parameters)
116119
def test_bmm_tosa_MI(self, operand1: torch.Tensor, operand2: torch.Tensor):
@@ -143,9 +146,20 @@ def test_bmm_single_input_tosa_BI(self, operand1: torch.Tensor):
143146
self._test_bmm_tosa_BI_pipeline(self.BMMSingleInput(), test_data)
144147

145148
@parameterized.expand(BMM.test_parameters)
149+
@unittest.expectedFailure
146150
def test_bmm_u55_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
147151
test_data = (operand1, operand2)
148-
self._test_bmm_tosa_BI_pipeline(self.BMM(), test_data)
152+
self._test_bmm_ethosu_BI_pipeline(
153+
self.BMM(), common.get_u55_compile_spec(), test_data
154+
)
155+
156+
@parameterized.expand(BMM.test_parameters)
157+
@common.expectedFailureOnFVP
158+
def test_bmm_u85_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
159+
test_data = (operand1, operand2)
160+
self._test_bmm_ethosu_BI_pipeline(
161+
self.BMM(), common.get_u85_compile_spec(), test_data
162+
)
149163

150164
# Expected to fail with error: Warning, unsupported fusing of TOSA Rescale previous operator is of type: Memcpy
151165
@parameterized.expand(BMMSingleInput.test_parameters)
@@ -156,7 +170,9 @@ def test_bmm_single_input_u55_BI(self, operand1: torch.Tensor):
156170
self.BMMSingleInput(), common.get_u55_compile_spec(), test_data
157171
)
158172

173+
# Numerical issues on FVP, MLETORCH 534
159174
@parameterized.expand(BMMSingleInput.test_parameters)
175+
@common.expectedFailureOnFVP
160176
def test_bmm_single_input_u85_BI(self, operand1: torch.Tensor):
161177
test_data = (operand1,)
162178
self._test_bmm_ethosu_BI_pipeline(

‎backends/arm/test/ops/test_cat.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ def _test_cat_ethosu_BI_pipeline(
9696
compile_spec: CompileSpec,
9797
test_data: Tuple[tuple[torch.Tensor, ...], int],
9898
):
99-
(
99+
tester = (
100100
ArmTester(
101101
module,
102102
example_inputs=test_data,
@@ -108,10 +108,14 @@ def _test_cat_ethosu_BI_pipeline(
108108
.check(["torch.ops.quantized_decomposed"])
109109
.to_edge()
110110
.partition()
111+
.dump_artifact()
111112
.check_not(["executorch_exir_dialects_edge__ops_aten_cat_default"])
112113
.check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
113114
.to_executorch()
115+
.serialize()
114116
)
117+
if common.is_option_enabled("corstone300"):
118+
tester.run_method_and_compare_outputs(inputs=test_data)
115119

116120
@parameterized.expand(Cat.test_parameters)
117121
def test_cat_tosa_MI(self, operands: tuple[torch.Tensor, ...], dim: int):
@@ -129,14 +133,18 @@ def test_cat_tosa_BI(self, operands: tuple[torch.Tensor, ...], dim: int):
129133
test_data = (operands, dim)
130134
self._test_cat_tosa_BI_pipeline(self.Cat(), test_data)
131135

136+
# Mismatch in provided number of inputs and model signature, MLETORCH 519
132137
@parameterized.expand(Cat.test_parameters)
138+
@common.expectedFailureOnFVP
133139
def test_cat_u55_BI(self, operands: tuple[torch.Tensor, ...], dim: int):
134140
test_data = (operands, dim)
135141
self._test_cat_ethosu_BI_pipeline(
136142
self.Cat(), common.get_u55_compile_spec(), test_data
137143
)
138144

145+
# Mismatch in provided number of inputs and model signature, MLETORCH 519
139146
@parameterized.expand(Cat.test_parameters)
147+
@common.expectedFailureOnFVP
140148
def test_cat_u85_BI(self, operands: tuple[torch.Tensor, ...], dim: int):
141149
test_data = (operands, dim)
142150
self._test_cat_ethosu_BI_pipeline(

‎backends/arm/test/ops/test_clone.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ def _test_clone_tosa_ethos_pipeline(
8585
test_data: Tuple[torch.Tensor],
8686
):
8787
quantizer = ArmQuantizer().set_io(get_symmetric_quantization_config())
88-
(
88+
tester = (
8989
ArmTester(module, example_inputs=test_data, compile_spec=compile_spec)
9090
.quantize(Quantize(quantizer, get_symmetric_quantization_config()))
9191
.export()
@@ -94,7 +94,10 @@ def _test_clone_tosa_ethos_pipeline(
9494
.partition()
9595
.check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
9696
.to_executorch()
97+
.serialize()
9798
)
99+
if common.is_option_enabled("corstone300"):
100+
tester.run_method_and_compare_outputs(qtol=1, inputs=test_data)
98101

99102
def _test_clone_tosa_u55_pipeline(
100103
self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]

‎backends/arm/test/ops/test_conv1d.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -268,7 +268,7 @@ def _test_conv1d_ethosu_BI_pipeline(
268268
compile_spec: CompileSpec,
269269
test_data: Tuple[torch.Tensor],
270270
):
271-
(
271+
tester = (
272272
ArmTester(module, example_inputs=test_data, compile_spec=compile_spec)
273273
.quantize()
274274
.export()
@@ -277,7 +277,10 @@ def _test_conv1d_ethosu_BI_pipeline(
277277
.check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
278278
.check_not(["executorch_exir_dialects_edge__ops_aten_convolution_default"])
279279
.to_executorch()
280+
.serialize()
280281
)
282+
if common.is_option_enabled("corstone300"):
283+
tester.run_method_and_compare_outputs(qtol=1, inputs=test_data)
281284

282285
@parameterized.expand(testsuite)
283286
def test_conv1d_tosa_MI(self, test_name, model):
@@ -295,6 +298,9 @@ def test_conv1d_u55_BI(self, test_name, model):
295298
model, common.get_u55_compile_spec(), model.get_inputs()
296299
)
297300

301+
# This specific test case has numerical errors on FVP, MLETORCH-520.
302+
testsuite.remove(("5_3x2x128_st1", conv1d_5_3x2x128_st1))
303+
298304
@parameterized.expand(testsuite)
299305
def test_conv1d_u85_BI(self, test_name, model):
300306
self._test_conv1d_ethosu_BI_pipeline(

‎backends/arm/test/ops/test_conv2d.py

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -295,7 +295,7 @@ def _test_conv2d_ethosu_BI_pipeline(
295295
module: torch.nn.Module,
296296
test_data: Tuple[torch.Tensor],
297297
):
298-
(
298+
tester = (
299299
ArmTester(
300300
module,
301301
example_inputs=test_data,
@@ -308,7 +308,10 @@ def _test_conv2d_ethosu_BI_pipeline(
308308
.check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
309309
.check_not(["executorch_exir_dialects_edge__ops_aten_convolution_default"])
310310
.to_executorch()
311+
.serialize()
311312
)
313+
if common.is_option_enabled("corstone300"):
314+
tester.run_method_and_compare_outputs(qtol=1, inputs=test_data)
312315

313316
@parameterized.expand(testsuite)
314317
def test_conv2d_tosa_MI(self, test_name, model):
@@ -318,6 +321,10 @@ def test_conv2d_tosa_MI(self, test_name, model):
318321
def test_conv2d_tosa_BI(self, test_name, model):
319322
self._test_conv2d_tosa_BI_pipeline(model, model.get_inputs())
320323

324+
# These cases have numerical issues on FVP, MLETORCH-520
325+
testsuite.remove(("2x2_3x2x40x40_nobias", conv2d_2x2_3x2x40x40_nobias))
326+
testsuite.remove(("5x5_3x2x128x128_st1", conv2d_5x5_3x2x128x128_st1))
327+
321328
@parameterized.expand(testsuite)
322329
def test_conv2d_u55_BI(self, test_name, model):
323330
self._test_conv2d_ethosu_BI_pipeline(

‎backends/arm/test/ops/test_conv_combos.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,7 @@ def _test_conv_combo_ethos_BI_pipeline(
238238
compile_spec: CompileSpec,
239239
test_data: Tuple[torch.Tensor],
240240
):
241-
(
241+
tester = (
242242
ArmTester(
243243
module,
244244
example_inputs=test_data,
@@ -251,7 +251,10 @@ def _test_conv_combo_ethos_BI_pipeline(
251251
.check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
252252
.check_not(list(module.edge_op_list))
253253
.to_executorch()
254+
.serialize()
254255
)
256+
if common.is_option_enabled("corstone300"):
257+
tester.run_method_and_compare_outputs(qtol=1, inputs=test_data)
255258

256259
####################
257260
## Conv + meandim ##
@@ -272,6 +275,8 @@ def test_conv_meandim_u55_BI(self):
272275
model.get_inputs(),
273276
)
274277

278+
# Numerical Issues on FVP, MLETORCH-520
279+
@common.expectedFailureOnFVP
275280
def test_conv_meandim_u85_BI(self):
276281
model = ComboConv2dMeandim()
277282
self._test_conv_combo_ethos_BI_pipeline(

‎backends/arm/test/ops/test_depthwise_conv.py

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@
88

99
from typing import Tuple
1010

11-
import pytest
12-
1311
import torch
1412
from executorch.backends.arm.test import common
1513
from executorch.backends.arm.test.ops.test_conv1d import Conv1d
@@ -160,8 +158,8 @@
160158

161159
testsuite_conv1d = [
162160
("2_1x6x4_gp6_st1", dw_conv1d_2_1x6x4_gp6_st1),
163-
("3_1x3x256_gp3_st1", dw_conv1d_3_1x3x256_gp3_st1),
164161
("two_dw_conv1d", two_dw_conv1d),
162+
("3_1x3x256_gp3_st1", dw_conv1d_3_1x3x256_gp3_st1),
165163
("3_1x3x14_gp3_st1", dw_conv1d_3_1x3x14_gp3_st1),
166164
]
167165

@@ -217,7 +215,7 @@ def _test_dw_conv_ethos_BI_pipeline(
217215
compile_spec: CompileSpec,
218216
test_data: Tuple[torch.Tensor],
219217
):
220-
(
218+
tester = (
221219
ArmTester(
222220
module,
223221
example_inputs=test_data,
@@ -230,19 +228,26 @@ def _test_dw_conv_ethos_BI_pipeline(
230228
.check_not(["executorch_exir_dialects_edge__ops_aten_convolution_default"])
231229
.check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
232230
.to_executorch()
231+
.serialize()
233232
)
233+
if common.is_option_enabled("corstone300"):
234+
tester.run_method_and_compare_outputs(qtol=1, inputs=test_data)
234235

235236
@parameterized.expand(testsuite_conv1d + testsuite_conv2d)
236237
def test_dw_conv_tosa_MI(self, test_name: str, model: torch.nn.Module):
237238
self._test_dw_conv_tosa_MI_pipeline(model, model.get_inputs())
238239

239240
# TODO: Investigate flakyness (MLTORCH-307)
240241
@parameterized.expand(testsuite_conv1d + testsuite_conv2d)
241-
@pytest.mark.flaky(reruns=3)
242242
def test_dw_conv_tosa_BI(self, test_name: str, model: torch.nn.Module):
243243
self._test_dw_conv_tosa_BI_pipeline(model, model.get_inputs())
244244

245+
testsuite_conv2d.remove(
246+
("3x3_1x3x256x256_gp3_st1", dw_conv2d_3x3_1x3x256x256_gp3_st1)
247+
) # Works
248+
245249
@parameterized.expand(testsuite_conv2d, skip_on_empty=True)
250+
@common.expectedFailureOnFVP
246251
def test_dw_conv2d_u55_BI(
247252
self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False
248253
):
@@ -269,7 +274,21 @@ def test_dw_conv1d_u55_BI(
269274
model.get_inputs(),
270275
)
271276

272-
@parameterized.expand(testsuite_conv1d + testsuite_conv2d)
277+
# All test cases except 3x3_1x3x256x256_gp3_st1 have numerical issues on FVP. MLETORCH-520
278+
@parameterized.expand(testsuite_conv1d[:-2] + testsuite_conv2d)
279+
@common.expectedFailureOnFVP
280+
def test_dw_conv_u85_BI_xfails(
281+
self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False
282+
):
283+
self._test_dw_conv_ethos_BI_pipeline(
284+
model,
285+
common.get_u85_compile_spec(
286+
permute_memory_to_nhwc=True, quantize_io=set_quantize_io
287+
),
288+
model.get_inputs(),
289+
)
290+
291+
@parameterized.expand(testsuite_conv1d[-2:])
273292
def test_dw_conv_u85_BI(
274293
self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False
275294
):

‎backends/arm/test/ops/test_div.py

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -136,10 +136,10 @@ def _test_div_tosa_BI_pipeline(
136136
.run_method_and_compare_outputs(inputs=test_data, atol=1, rtol=0.1)
137137
)
138138

139-
def _test_div_u55_BI_pipeline(
140-
self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]
139+
def _test_div_ethos_BI_pipeline(
140+
self, module: torch.nn.Module, compile_spec, test_data: Tuple[torch.Tensor]
141141
):
142-
(
142+
tester = (
143143
ArmTester(
144144
module,
145145
example_inputs=test_data,
@@ -155,7 +155,10 @@ def _test_div_u55_BI_pipeline(
155155
.partition()
156156
.check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
157157
.to_executorch()
158+
.serialize()
158159
)
160+
if common.is_option_enabled("corstone300"):
161+
tester.run_method_and_compare_outputs(qtol=1, inputs=test_data)
159162

160163
@parameterized.expand(test_data_suite)
161164
def test_div_tosa_MI(
@@ -180,7 +183,9 @@ def test_div_tosa_BI(
180183
test_data = (input_, other_)
181184
self._test_div_tosa_BI_pipeline(self.Div(), test_data)
182185

186+
# Numerical issues on FVP likely due to mul op, MLETORCH-521
183187
@parameterized.expand(test_data_suite)
188+
@common.expectedFailureOnFVP
184189
def test_div_u55_BI(
185190
self,
186191
test_name: str,
@@ -189,4 +194,21 @@ def test_div_u55_BI(
189194
rounding_mode: Optional[str] = None,
190195
):
191196
test_data = (input_, other_)
192-
self._test_div_u55_BI_pipeline(self.Div(), test_data)
197+
self._test_div_ethos_BI_pipeline(
198+
self.Div(), common.get_u55_compile_spec(), test_data
199+
)
200+
201+
# Numerical issues on FVP likely due to mul op, MLETORCH-521
202+
@parameterized.expand(test_data_suite)
203+
@common.expectedFailureOnFVP
204+
def test_div_u85_BI(
205+
self,
206+
test_name: str,
207+
input_: Union[torch.Tensor, torch.types.Number],
208+
other_: Union[torch.Tensor, torch.types.Number],
209+
rounding_mode: Optional[str] = None,
210+
):
211+
test_data = (input_, other_)
212+
self._test_div_ethos_BI_pipeline(
213+
self.Div(), common.get_u85_compile_spec(), test_data
214+
)

0 commit comments

Comments
 (0)