refactor + add log + update test

Author: NihalHarish

src/sagemaker/estimator.py

@@ -2148,10 +2148,6 @@ def __init__(
         self.checkpoint_s3_uri = checkpoint_s3_uri
         self.checkpoint_local_path = checkpoint_local_path
 
-        # Disable debugger if checkpointing is enabled by the customer
-        self.debugger_hook_config = \
-            self.debugger_hook_config if checkpoint_s3_uri is None \
-                                         and checkpoint_local_path is None else False
         self.enable_sagemaker_metrics = enable_sagemaker_metrics
 
     def _prepare_for_training(self, job_name=None):
@@ -2204,7 +2200,6 @@ def _prepare_for_training(self, job_name=None):
         self._hyperparameters[CONTAINER_LOG_LEVEL_PARAM_NAME] = self.container_log_level
         self._hyperparameters[JOB_NAME_PARAM_NAME] = self._current_job_name
         self._hyperparameters[SAGEMAKER_REGION_PARAM_NAME] = self.sagemaker_session.boto_region_name
-
         self._validate_and_set_debugger_configs()
 
     def _validate_and_set_debugger_configs(self):
@@ -2214,7 +2209,14 @@ def _validate_and_set_debugger_configs(self):
         ):
             self.debugger_hook_config = DebuggerHookConfig(s3_output_path=self.output_path)
         elif not self.debugger_hook_config:
-            self.debugger_hook_config = None
+            self.debugger_hook_config = False
+
+        # Disable debugger if checkpointing is enabled by the customer
+        if self.checkpoint_s3_uri and self.checkpoint_local_path and self.debugger_hook_config:
+            logger.info(
+                "SM Debug Does Not Currently Support Training Jobs With Checkpointing Enabled"
+            )
+            self.debugger_hook_config = False
 
     def _stage_user_code_in_s3(self):
         """Upload the user training script to s3 and return the location.

tests/integ/test_debugger.py

@@ -358,11 +358,11 @@ def test_debug_hook_disabled_with_checkpointing(
     cpu_instance_type,
 ):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
-        s3_output_path = os.path.join("s3://", sagemaker_session.default_bucket(), str(uuid.uuid4()))
+        s3_output_path = os.path.join(
+            "s3://", sagemaker_session.default_bucket(), str(uuid.uuid4())
+        )
         debugger_hook_config = DebuggerHookConfig(
-            s3_output_path=os.path.join(
-                s3_output_path, "tensors"
-            )
+            s3_output_path=os.path.join(s3_output_path, "tensors")
         )
 
         script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist_gluon.py")
@@ -378,21 +378,10 @@ def test_debug_hook_disabled_with_checkpointing(
             sagemaker_session=sagemaker_session,
             debugger_hook_config=debugger_hook_config,
             checkpoint_local_path="/opt/ml/checkpoints",
-            checkpoint_s3_uri=os.path.join(s3_output_path, "checkpoints")
-
-        )
-
-        train_input = mx.sagemaker_session.upload_data(
-            path=os.path.join(data_path, "train"), key_prefix="integ-test-data/mxnet_mnist/train"
+            checkpoint_s3_uri=os.path.join(s3_output_path, "checkpoints"),
         )
-        test_input = mx.sagemaker_session.upload_data(
-            path=os.path.join(data_path, "test"), key_prefix="integ-test-data/mxnet_mnist/test"
-        )
-
-        mx.fit({"train": train_input, "test": test_input})
-
-        job_description = mx.latest_training_job.describe()
-        assert "DebugHookConfig" not in job_description
+        mx._prepare_for_training()
+        assert mx.debugger_hook_config is False
 
 
 def test_mxnet_with_rules_and_debugger_hook_config(