Merge branch 'master' into tf-distribution

Author: ajaykarpur

doc/frameworks/pytorch/using_pytorch.rst

@@ -132,7 +132,7 @@ Using third-party libraries
 ---------------------------
 
 When running your training script on SageMaker, it will have access to some pre-installed third-party libraries including ``torch``, ``torchvision``, and ``numpy``.
-For more information on the runtime environment, including specific package versions, see `SageMaker PyTorch Docker containers <https://github.com/aws/sagemaker-python-sdk/tree/master/src/sagemaker/pytorch#sagemaker-pytorch-docker-containers>`_.
+For more information on the runtime environment, including specific package versions, see `SageMaker PyTorch Docker containers <https://github.com/aws/deep-learning-containers/tree/master/pytorch>`_.
 
 If there are other packages you want to use with your script, you can include a ``requirements.txt`` file in the same directory as your training script to install other dependencies at runtime. Both ``requirements.txt`` and your training script should be put in the same folder. You must specify this folder in ``source_dir`` argument when creating PyTorch estimator.
 

src/sagemaker/lineage/artifact.py

@@ -81,6 +81,9 @@ class Artifact(_base_types.Record):
     def save(self):
         """Save the state of this Artifact to SageMaker.
 
+        Note that this method must be run from a SageMaker context such as Studio or a training job
+        due to restrictions on the CreateArtifact API.
+
         Returns:
             Artifact: A SageMaker `Artifact` object.
         """

tests/data/tensorflow_mnist/mnist.py

@@ -159,7 +159,7 @@ def serving_input_fn():
 
     # Train the model
     train_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(
-        x={"x": train_data}, y=train_labels, batch_size=50, num_epochs=None, shuffle=True
+        x={"x": train_data}, y=train_labels, batch_size=50, num_epochs=None, shuffle=False
     )
 
     # Evaluate the model and print results

tests/integ/test_spark_processing.py

@@ -302,19 +302,13 @@ def test_integ_history_server(spark_py_processor, sagemaker_session):
             sagemaker_session=sagemaker_session,
         )
 
+    # sleep 3 seconds to avoid s3 eventual consistency issue
+    time.sleep(3)
     spark_py_processor.start_history_server(spark_event_logs_s3_uri=spark_event_logs_s3_uri)
 
     try:
         response = _request_with_retry(HISTORY_SERVER_ENDPOINT)
         assert response.status == 200
-
-        # spark has redirect behavior, this request verify that page navigation works with redirect
-        response = _request_with_retry(f"{HISTORY_SERVER_ENDPOINT}{SPARK_APPLICATION_URL_SUFFIX}")
-        assert response.status == 200
-
-        html_content = response.data.decode("utf-8")
-        assert "Completed Jobs (4)" in html_content
-        assert "collect at /opt/ml/processing/input/code/test_long_duration.py:32" in html_content
     finally:
         spark_py_processor.terminate_history_server()
 

tests/integ/test_transformer.py

@@ -420,7 +420,7 @@ def test_transform_tf_kms_network_isolation(
         with open(os.path.join(tmpdir, "tf-batch-output", "data.csv.out")) as f:
             result = json.load(f)
             assert len(result["predictions"][0]["probabilities"]) == 10
-            assert result["predictions"][0]["classes"] == 1
+            assert result["predictions"][0]["classes"] >= 1
 
 
 def _create_transformer_and_transform_job(