Skip to content

Commit 33d89a7

Browse files
author
Ace Nassri
authored
chore(run/jobs): add retries for flaky Cloud Builds (#8491)
* chore(run/jobs): add retries to failing test * Fix lint * Lint * Switch to backoff (part 1 of 2) * Switch to backoff (part 2 of 2) * Move retried code into its own submethod(s) * Lint
1 parent fbc738e commit 33d89a7

File tree

2 files changed

+38
-29
lines changed

2 files changed

+38
-29
lines changed

run/jobs/e2e_test.py

Lines changed: 36 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import time
2323
import uuid
2424

25+
import backoff
2526
from google.cloud.logging_v2.services.logging_service_v2 import LoggingServiceV2Client
2627

2728
import pytest
@@ -36,36 +37,43 @@
3637
@pytest.fixture
3738
def setup_job():
3839
# Build container image and run the job
39-
subprocess.check_call(
40-
[
41-
"gcloud",
42-
"builds",
43-
"submit",
44-
"--config",
45-
"e2e_test_setup.yaml",
46-
"--project",
47-
PROJECT,
48-
"--substitutions",
49-
"_SERVICE=" + SERVICE + ",_VERSION=" + SUFFIX + ",_REGION=" + REGION,
50-
]
51-
)
52-
53-
yield SERVICE
40+
@backoff.on_exception(backoff.expo, subprocess.CalledProcessError)
41+
def setup():
42+
subprocess.check_call(
43+
[
44+
"gcloud",
45+
"builds",
46+
"submit",
47+
"--config",
48+
"e2e_test_setup.yaml",
49+
"--project",
50+
PROJECT,
51+
"--substitutions",
52+
"_SERVICE=" + SERVICE + ",_VERSION=" + SUFFIX + ",_REGION=" + REGION,
53+
]
54+
)
5455

5556
# Clean up the test resource
56-
subprocess.check_call(
57-
[
58-
"gcloud",
59-
"builds",
60-
"submit",
61-
"--config",
62-
"e2e_test_cleanup.yaml",
63-
"--project",
64-
PROJECT,
65-
"--substitutions",
66-
"_SERVICE=" + SERVICE + ",_VERSION=" + SUFFIX + ",_REGION=" + REGION,
67-
]
68-
)
57+
@backoff.on_exception(backoff.expo, subprocess.CalledProcessError)
58+
def teardown():
59+
subprocess.check_call(
60+
[
61+
"gcloud",
62+
"builds",
63+
"submit",
64+
"--config",
65+
"e2e_test_cleanup.yaml",
66+
"--project",
67+
PROJECT,
68+
"--substitutions",
69+
"_SERVICE=" + SERVICE + ",_VERSION=" + SUFFIX + ",_REGION=" + REGION,
70+
]
71+
)
72+
73+
# Run the fixture
74+
setup()
75+
yield SERVICE
76+
teardown()
6977

7078

7179
def test_end_to_end(setup_job):

run/jobs/requirements-test.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
1+
backoff==2.2.1
12
pytest==7.0.1
2-
google-cloud-logging==3.1.2
3+
google-cloud-logging==3.1.2

0 commit comments

Comments
 (0)