Bug fix for getting dataframes in TrainingJobAnalytics.

Author: Piyush Adlakha

src/sagemaker/analytics.py

@@ -199,7 +199,7 @@ class TrainingJobAnalytics(AnalyticsMetricsBase):
     """Fetch training curve data from CloudWatch Metrics for a specific training job.
     """
 
-    CLOUDWATCH_NAMESPACE = '/aws/sagemaker/HyperParameterTuningJobs'
+    CLOUDWATCH_NAMESPACE = '/aws/sagemaker/TrainingJobs'
 
     def __init__(self, training_job_name, metric_names=None, sagemaker_session=None):
         """Initialize a ``TrainingJobAnalytics`` instance.
@@ -246,7 +246,12 @@ def _determine_timeinterval(self):
         """
         description = self._sage_client.describe_training_job(TrainingJobName=self.name)
         start_time = description[u'TrainingStartTime']  # datetime object
-        end_time = description.get(u'TrainingEndTime', datetime.datetime.utcnow())
+        # Incrementing end time by 1 min since cloud watch drops seconds before finding the logs.
+        # This results in logs being searched in the time range in which the correct log line was not present.
+        # Example - Log time - 2018-10-22 08:25:55
+        #           Here calculated end time would also be  2018-10-22 08:25:55 (without 1 min addition)
+        #           CW will consider end time as 2018-10-22 08:25 and will not be able to search the correct log.
+        end_time = description.get(u'TrainingEndTime', datetime.datetime.utcnow()) + datetime.timedelta(minutes=1)
         return {
             'start_time': start_time,
             'end_time': end_time,