Skip to content

Commit 3fc186c

Browse files
authored
Automatically retry flaky tests when running tests for PRs (#20637)
Note that when we run tests on the main branch we don't auto-retry so that way circleci can still take advantage of circleci's tracking of flaky tests. See #19795
1 parent 2b9d926 commit 3fc186c

File tree

3 files changed

+36
-1
lines changed

3 files changed

+36
-1
lines changed

.circleci/config.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,9 +238,17 @@ commands:
238238
type: string
239239
default: ""
240240
steps:
241+
- when:
242+
# We only set EMTEST_RETRY_FLAKY on pull requests. When we run
243+
# normal CI jobs on branches like main we still want to be able to
244+
# detect flakyness.
245+
condition: ${CIRCLE_PULL_REQUEST}
246+
steps:
247+
set-retry-flaky-tests
241248
- run:
242249
name: run tests (<< parameters.title >>)
243250
command: |
251+
env
244252
./test/runner << parameters.test_targets >>
245253
$EMSDK_PYTHON ./test/check_clean.py
246254
freeze-cache:
@@ -249,6 +257,12 @@ commands:
249257
- run:
250258
name: Add EM_FROZEN_CACHE to bash env
251259
command: echo "export EM_FROZEN_CACHE=1" >> $BASH_ENV
260+
set-retry-flaky-tests:
261+
description: "Set EMTEST_RETRY_FLAKY"
262+
steps:
263+
- run:
264+
name: Add EMTEST_RETRY_FLAKY to bash env
265+
command: echo "export EMTEST_RETRY_FLAKY=2" >> $BASH_ENV
252266
run-tests-linux:
253267
description: "Runs emscripten tests"
254268
parameters:

test/common.py

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@
5656
EMTEST_ALL_ENGINES = None
5757
EMTEST_SKIP_SLOW = None
5858
EMTEST_SKIP_FLAKY = None
59+
EMTEST_RETRY_FLAKY = None
5960
EMTEST_LACKS_NATIVE_CLANG = None
6061
EMTEST_VERBOSE = None
6162
EMTEST_REBASELINE = None
@@ -148,9 +149,27 @@ def decorated(self, *args, **kwargs):
148149

149150
def flaky(note=''):
150151
assert not callable(note)
152+
151153
if EMTEST_SKIP_FLAKY:
152154
return unittest.skip(note)
153-
return lambda f: f
155+
156+
if not EMTEST_RETRY_FLAKY:
157+
return lambda f: f
158+
159+
def decorated(f):
160+
@wraps(f)
161+
def modified(*args, **kwargs):
162+
for i in range(EMTEST_RETRY_FLAKY):
163+
try:
164+
return f(*args, **kwargs)
165+
except AssertionError as exc:
166+
preserved_exc = exc
167+
logging.info(f'Retrying flaky test (attempt {i}/{EMTEST_RETRY_FLAKY} failed): {exc}')
168+
raise AssertionError('Flaky test has failed too many times') from preserved_exc
169+
170+
return modified
171+
172+
return decorated
154173

155174

156175
def disabled(note=''):
@@ -288,6 +307,7 @@ def with_env_modify(updates):
288307
assert not callable(updates)
289308

290309
def decorated(f):
310+
@wraps(f)
291311
def modified(self, *args, **kwargs):
292312
with env_modify(updates):
293313
return f(self, *args, **kwargs)

test/runner.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -393,6 +393,7 @@ def configure():
393393
common.EMTEST_ALL_ENGINES = int(os.getenv('EMTEST_ALL_ENGINES', '0'))
394394
common.EMTEST_SKIP_SLOW = int(os.getenv('EMTEST_SKIP_SLOW', '0'))
395395
common.EMTEST_SKIP_FLAKY = int(os.getenv('EMTEST_SKIP_FLAKY', '0'))
396+
common.EMTEST_RETRY_FLAKY = int(os.getenv('EMTEST_RETRY_FLAKY', '0'))
396397
common.EMTEST_LACKS_NATIVE_CLANG = int(os.getenv('EMTEST_LACKS_NATIVE_CLANG', '0'))
397398
common.EMTEST_REBASELINE = int(os.getenv('EMTEST_REBASELINE', '0'))
398399
common.EMTEST_VERBOSE = int(os.getenv('EMTEST_VERBOSE', '0')) or shared.DEBUG

0 commit comments

Comments
 (0)