Skip to content

Commit 3d212ac

Browse files
Revert "fix: Add retry in session.py to check if training is finished (#3285)"
This reverts commit 5bc3ccf.
1 parent 7963385 commit 3d212ac

File tree

1 file changed

+12
-22
lines changed

1 file changed

+12
-22
lines changed

src/sagemaker/session.py

Lines changed: 12 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@
4242
secondary_training_status_changed,
4343
secondary_training_status_message,
4444
sts_regional_endpoint,
45-
retries,
4645
)
4746
from sagemaker import exceptions
4847
from sagemaker.session_settings import SessionSettings
@@ -5459,30 +5458,21 @@ def _train_done(sagemaker_client, job_name, last_desc):
54595458
"""Placeholder docstring"""
54605459
in_progress_statuses = ["InProgress", "Created"]
54615460

5462-
for _ in retries(
5463-
max_retry_count=10, # 10*30 = 5min
5464-
exception_message_prefix="Waiting for schedule to leave 'Pending' status",
5465-
seconds_to_sleep=30,
5466-
):
5467-
try:
5468-
desc = sagemaker_client.describe_training_job(TrainingJobName=job_name)
5469-
status = desc["TrainingJobStatus"]
5461+
desc = sagemaker_client.describe_training_job(TrainingJobName=job_name)
5462+
status = desc["TrainingJobStatus"]
54705463

5471-
if secondary_training_status_changed(desc, last_desc):
5472-
print()
5473-
print(secondary_training_status_message(desc, last_desc), end="")
5474-
else:
5475-
print(".", end="")
5476-
sys.stdout.flush()
5464+
if secondary_training_status_changed(desc, last_desc):
5465+
print()
5466+
print(secondary_training_status_message(desc, last_desc), end="")
5467+
else:
5468+
print(".", end="")
5469+
sys.stdout.flush()
54775470

5478-
if status in in_progress_statuses:
5479-
return desc, False
5471+
if status in in_progress_statuses:
5472+
return desc, False
54805473

5481-
print()
5482-
return desc, True
5483-
except botocore.exceptions.ClientError as err:
5484-
if err.response["Error"]["Code"] == "AccessDeniedException":
5485-
pass
5474+
print()
5475+
return desc, True
54865476

54875477

54885478
def _processing_job_status(sagemaker_client, job_name):

0 commit comments

Comments
 (0)