Arm backend: Fix sigmoid int16 and int32 flakyness

MLBEDSW-10642 has now been resolved. Step vela pin and remove all
related pytest.mark_flaky in sigmoid int16 and int32 tests.

Signed-off-by: Oscar Andersson <[email protected]>
Change-Id: I8c1aff31fb173fa6fdebc1d5b757623f9c2e1321

Author: oscarandersson8218

backends/arm/test/ops/test_sigmoid_16bit.py

@@ -4,7 +4,6 @@
 # LICENSE file in the root directory of this source tree.
 
 import pytest
-
 import torch
 from executorch.backends.arm.quantizer import (
     get_symmetric_quantization_config,
@@ -81,10 +80,13 @@ def forward(self, x):
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_tosa_BI(test_data):
     pipeline = TosaPipelineBI(
-        Sigmoid(), (test_data(),), Sigmoid.aten_op, Sigmoid.exir_op
+        Sigmoid(),
+        (test_data(),),
+        Sigmoid.aten_op,
+        Sigmoid.exir_op,
+        qtol=1,
     )
     pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()
@@ -97,10 +99,14 @@ def test_sigmoid_tosa_BI(test_data):
         "ramp": "AssertionError: Output 0 does not match reference output. MLETORCH-787"
     },
 )
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
+@pytest.mark.flaky(reruns=5)  # MLETORCH-787: Investigate int16-int8 rescaling precision
 def test_sigmoid_add_sigmoid_tosa_BI(test_data):
     pipeline = TosaPipelineBI(
-        SigmoidAddSigmoid(), (test_data(),), Sigmoid.aten_op, Sigmoid.exir_op
+        SigmoidAddSigmoid(),
+        (test_data(),),
+        Sigmoid.aten_op,
+        Sigmoid.exir_op,
+        qtol=1,
     )
     pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()
@@ -110,7 +116,6 @@ def test_sigmoid_add_sigmoid_tosa_BI(test_data):
     "test_data",
     test_data_suite,
 )
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_tosa_u55(test_data):
     pipeline = OpNotSupportedPipeline(
         Sigmoid(), (test_data(),), "TOSA-0.80+BI+u55", {Sigmoid.exir_op: 1}
@@ -123,7 +128,6 @@ def test_sigmoid_tosa_u55(test_data):
     "test_data",
     test_data_suite,
 )
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_add_sigmoid_tosa_u55(test_data):
     pipeline = OpNotSupportedPipeline(
         SigmoidAddSigmoid(),
@@ -137,7 +141,6 @@ def test_sigmoid_add_sigmoid_tosa_u55(test_data):
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 @common.XfailIfNoCorstone320
 def test_sigmoid_tosa_u85(test_data):
     pipeline = EthosU85PipelineBI(
@@ -151,10 +154,10 @@ def test_sigmoid_tosa_u85(test_data):
     "test_data",
     test_data_suite,
     xfails={
-        "ramp": "AssertionError: Output 0 does not match reference output.",
+        "ramp": "AssertionError: Output 0 does not match reference output. MLETORCH-787"
     },
 )
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
+@pytest.mark.flaky(reruns=5)  # MLETORCH-787: Investigate int16-int8 rescaling precision
 @common.XfailIfNoCorstone320
 def test_sigmoid_add_sigmoid_tosa_u85(test_data):
     pipeline = EthosU85PipelineBI(

backends/arm/test/ops/test_sigmoid_32bit.py

@@ -3,7 +3,6 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-import pytest
 import torch
 from executorch.backends.arm.quantizer import TOSAQuantizer
 from executorch.backends.arm.quantizer.quantization_config import QuantizationConfig
@@ -97,33 +96,32 @@ def forward(self, x):
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_tosa_BI(test_data):
     pipeline = TosaPipelineBI(
         Sigmoid(),
         (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
+        qtol=1,
     )
     pipeline.change_args("quantize", get_32bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_add_sigmoid_tosa_BI(test_data):
     pipeline = TosaPipelineBI(
         SigmoidAddSigmoid(),
         (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
+        qtol=1,
     )
     pipeline.change_args("quantize", get_32bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_tosa_u55(test_data):
     pipeline = OpNotSupportedPipeline(
         Sigmoid(), (test_data(),), "TOSA-0.80+BI+u55", {Sigmoid.exir_op: 1}
@@ -133,7 +131,6 @@ def test_sigmoid_tosa_u55(test_data):
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_add_sigmoid_tosa_u55(test_data):
     pipeline = OpNotSupportedPipeline(
         SigmoidAddSigmoid(),
@@ -147,7 +144,6 @@ def test_sigmoid_add_sigmoid_tosa_u55(test_data):
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 @common.XfailIfNoCorstone320
 def test_sigmoid_tosa_u85(test_data):
     pipeline = EthosU85PipelineBI(
@@ -160,11 +156,7 @@ def test_sigmoid_tosa_u85(test_data):
 @common.parametrize(
     "test_data",
     test_data_suite,
-    xfails={
-        "ramp": "AssertionError: Output 0 does not match reference output.",
-    },
 )
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 @common.XfailIfNoCorstone320
 def test_sigmoid_add_sigmoid_tosa_u85(test_data):
     pipeline = EthosU85PipelineBI(

examples/arm/setup.sh

@@ -60,7 +60,7 @@ fi
 
 # vela
 vela_repo_url="https://gitlab.arm.com/artificial-intelligence/ethos-u/ethos-u-vela"
-vela_rev="425541302c7e4b6fbeca7c0061286b131ee507c3"
+vela_rev="677651add50745478367da009936e4505d627399"
 
 ########
 ### Functions