Merge branch 'master' into fix-updater

Author: ajaykarpur

src/sagemaker/feature_store/feature_group.py

@@ -179,10 +179,10 @@ def _ingest_single_batch(
             end_index (int): ending position to ingest in this batch.
         """
         logger.info("Started ingesting index %d to %d", start_index, end_index)
-        for row in data_frame[start_index:end_index].itertuples(index=False):
+        for _, row in data_frame[start_index:end_index].iterrows():
             record = [
                 FeatureValue(feature_name=name, value_as_string=str(value))
-                for name, value in row._asdict().items()
+                for name, value in row.items()
             ]
             sagemaker_session.put_record(
                 feature_group_name=feature_group_name, record=[value.to_dict() for value in record]

src/sagemaker/tensorflow/estimator.py

@@ -113,6 +113,7 @@ def __init__(
             :class:`~sagemaker.estimator.Framework` and
             :class:`~sagemaker.estimator.EstimatorBase`.
         """
+        distribution = renamed_kwargs("distributions", "distribution", distribution, kwargs)
         instance_type = renamed_kwargs(
             "train_instance_type", "instance_type", kwargs.get("instance_type"), kwargs
         )

tests/data/tensorflow_mnist/mnist.py

@@ -159,7 +159,7 @@ def serving_input_fn():
 
     # Train the model
     train_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(
-        x={"x": train_data}, y=train_labels, batch_size=50, num_epochs=None, shuffle=True
+        x={"x": train_data}, y=train_labels, batch_size=50, num_epochs=None, shuffle=False
     )
 
     # Evaluate the model and print results

tests/integ/test_spark_processing.py

@@ -302,19 +302,13 @@ def test_integ_history_server(spark_py_processor, sagemaker_session):
             sagemaker_session=sagemaker_session,
         )
 
+    # sleep 3 seconds to avoid s3 eventual consistency issue
+    time.sleep(3)
     spark_py_processor.start_history_server(spark_event_logs_s3_uri=spark_event_logs_s3_uri)
 
     try:
         response = _request_with_retry(HISTORY_SERVER_ENDPOINT)
         assert response.status == 200
-
-        # spark has redirect behavior, this request verify that page navigation works with redirect
-        response = _request_with_retry(f"{HISTORY_SERVER_ENDPOINT}{SPARK_APPLICATION_URL_SUFFIX}")
-        assert response.status == 200
-
-        html_content = response.data.decode("utf-8")
-        assert "Completed Jobs (4)" in html_content
-        assert "collect at /opt/ml/processing/input/code/test_long_duration.py:32" in html_content
     finally:
         spark_py_processor.terminate_history_server()
 

tests/integ/test_transformer.py

@@ -420,7 +420,7 @@ def test_transform_tf_kms_network_isolation(
         with open(os.path.join(tmpdir, "tf-batch-output", "data.csv.out")) as f:
             result = json.load(f)
             assert len(result["predictions"][0]["probabilities"]) == 10
-            assert result["predictions"][0]["classes"] == 1
+            assert result["predictions"][0]["classes"] >= 1
 
 
 def _create_transformer_and_transform_job(

tests/unit/sagemaker/feature_store/test_feature_store.py

@@ -146,7 +146,7 @@ def test_load_feature_definition_unsupported_types(sagemaker_session_mock):
 @patch("sagemaker.feature_store.feature_group.IngestionManagerPandas")
 def test_ingest(ingestion_manager_init, sagemaker_session_mock):
     feature_group = FeatureGroup(name="MyGroup", sagemaker_session=sagemaker_session_mock)
-    df = pd.DataFrame({"float": pd.Series([2.0], dtype="float64")})
+    df = pd.DataFrame(dict((f"float{i}", pd.Series([2.0], dtype="float64")) for i in range(300)))
 
     mock_ingestion_manager_instance = Mock()
     ingestion_manager_init.return_value = mock_ingestion_manager_instance