Merge branch 'master' into smddp_custom_mpi_support

Author: ahsan-z-khan

CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## v2.32.1 (2021-04-01)
+
+### Bug Fixes and Other Changes
+
+ * disable profiler in some release tests
+ * remove outdated notebook from test
+ * add compilation option for ml_eia2
+ * add short version to smdataparallel supported list
+
+### Documentation Changes
+
+ * creating a "latest" version sm distributed docs
+ * add docs for Sagemaker Model Parallel 1.3, released with PT 1.8
+ * update PyTorch version in doc
+
 ## v2.32.0 (2021-03-26)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.32.1.dev0
+2.32.2.dev0

doc/api/training/sdp_versions/latest.rst

@@ -0,0 +1,9 @@
+
+Version 1.1.0 (Latest)
+======================
+
+.. toctree::
+   :maxdepth: 1
+
+   latest/smd_data_parallel_pytorch.rst
+   latest/smd_data_parallel_tensorflow.rst

doc/api/training/sdp_versions/v1.1.0/smd_data_parallel_pytorch.rst renamed to doc/api/training/sdp_versions/latest/smd_data_parallel_pytorch.rst

@@ -84,7 +84,7 @@ Select a version to see the API documentation for version.
 .. toctree::
    :maxdepth: 1
 
-   sdp_versions/v1_1_0.rst
+   sdp_versions/latest.rst
    sdp_versions/v1_0_0.rst
 
 .. important::

doc/api/training/sdp_versions/v1.1.0/smd_data_parallel_tensorflow.rst renamed to doc/api/training/sdp_versions/latest/smd_data_parallel_tensorflow.rst

@@ -34,7 +34,7 @@ Select a version to see the API documentation for version. To use the library, r
 .. toctree::
    :maxdepth: 1
 
-   smp_versions/v1_3_0.rst
+   smp_versions/latest.rst
    smp_versions/v1_2_0.rst
    smp_versions/v1_1_0.rst
 

doc/api/training/sdp_versions/v1_1_0.rst

@@ -7,6 +7,6 @@ To use the library, reference the Common API documentation alongside the framewo
 .. toctree::
    :maxdepth: 1
 
-   v1.3.0/smd_model_parallel_common_api
-   v1.3.0/smd_model_parallel_pytorch
-   v1.3.0/smd_model_parallel_tensorflow
+   latest/smd_model_parallel_common_api
+   latest/smd_model_parallel_pytorch
+   latest/smd_model_parallel_tensorflow

doc/api/training/smd_data_parallel.rst

@@ -123,6 +123,7 @@ def __init__(
         enable_network_isolation=False,
         profiler_config=None,
         disable_profiler=False,
+        environment=None,
         **kwargs,
     ):
         """Initialize an ``EstimatorBase`` instance.
@@ -266,6 +267,8 @@ def __init__(
                 ``disable_profiler`` parameter to ``True``.
             disable_profiler (bool): Specifies whether Debugger monitoring and profiling
                 will be disabled (default: ``False``).
+            environment (dict[str, str]) : Environment variables to be set for
+                use during training job (default: ``None``)
 
         """
         instance_count = renamed_kwargs(
@@ -352,6 +355,8 @@ def __init__(
         self.profiler_config = profiler_config
         self.disable_profiler = disable_profiler
 
+        self.environment = environment
+
         if not _region_supports_profiler(self.sagemaker_session.boto_region_name):
             self.disable_profiler = True
 
@@ -1471,6 +1476,7 @@ def _get_train_args(cls, estimator, inputs, experiment_config):
         train_args["tags"] = estimator.tags
         train_args["metric_definitions"] = estimator.metric_definitions
         train_args["experiment_config"] = experiment_config
+        train_args["environment"] = estimator.environment
 
         if isinstance(inputs, TrainingInput):
             if "InputMode" in inputs.config:
@@ -1659,6 +1665,7 @@ def __init__(
         enable_sagemaker_metrics=None,
         profiler_config=None,
         disable_profiler=False,
+        environment=None,
         **kwargs,
     ):
         """Initialize an ``Estimator`` instance.
@@ -1807,6 +1814,8 @@ def __init__(
                 ``disable_profiler`` parameter to ``True``.
             disable_profiler (bool): Specifies whether Debugger monitoring and profiling
                 will be disabled (default: ``False``).
+            environment (dict[str, str]) : Environment variables to be set for
+                use during training job (default: ``None``)
         """
         self.image_uri = image_uri
         self.hyperparam_dict = hyperparameters.copy() if hyperparameters else {}
@@ -1840,6 +1849,7 @@ def __init__(
             enable_network_isolation=enable_network_isolation,
             profiler_config=profiler_config,
             disable_profiler=disable_profiler,
+            environment=environment,
             **kwargs,
         )
 

doc/api/training/smd_model_parallel.rst

@@ -10,7 +10,8 @@
             "1.13": "1.13.0",
             "1.14": "1.14.0",
             "1.15": "1.15.0",
-            "2.0": "2.0.0"
+            "2.0": "2.0.0",
+            "2.3": "2.3.0"
         },
         "versions": {
             "1.10.0": {
@@ -218,6 +219,35 @@
                     "us-west-2": "763104351884"
                 },
                 "repository": "tensorflow-inference-eia"
+            },
+            "2.3.0": {
+                "registries": {
+                    "af-south-1": "626614931356",
+                    "ap-east-1": "871362719292",
+                    "ap-northeast-1": "763104351884",
+                    "ap-northeast-2": "763104351884",
+                    "ap-south-1": "763104351884",
+                    "ap-southeast-1": "763104351884",
+                    "ap-southeast-2": "763104351884",
+                    "ca-central-1": "763104351884",
+                    "cn-north-1": "727897471807",
+                    "cn-northwest-1": "727897471807",
+                    "eu-central-1": "763104351884",
+                    "eu-north-1": "763104351884",
+                    "eu-south-1": "692866216735",
+                    "eu-west-1": "763104351884",
+                    "eu-west-2": "763104351884",
+                    "eu-west-3": "763104351884",
+                    "me-south-1": "217643126080",
+                    "sa-east-1": "763104351884",
+                    "us-east-1": "763104351884",
+                    "us-east-2": "763104351884",
+                    "us-gov-west-1": "442386744353",
+                    "us-iso-east-1": "886529160074",
+                    "us-west-1": "763104351884",
+                    "us-west-2": "763104351884"
+                },
+                "repository": "tensorflow-inference-eia"
             }
         }
     },

doc/api/training/smp_versions/v1_3_0.rst renamed to doc/api/training/smp_versions/latest.rst

@@ -651,7 +651,9 @@ def compile(
         job_status = self.sagemaker_session.wait_for_compilation_job(job_name)
         self.model_data = job_status["ModelArtifacts"]["S3ModelArtifacts"]
         if target_instance_family is not None:
-            if target_instance_family.startswith("ml_"):
+            if target_instance_family == "ml_eia2":
+                pass
+            elif target_instance_family.startswith("ml_"):
                 self.image_uri = self._compilation_image_uri(
                     self.sagemaker_session.boto_region_name,
                     target_instance_family,

doc/api/training/smp_versions/v1.3.0/smd_model_parallel_common_api.rst renamed to doc/api/training/smp_versions/latest/smd_model_parallel_common_api.rst

@@ -456,6 +456,7 @@ def train(  # noqa: C901
         enable_sagemaker_metrics=None,
         profiler_rule_configs=None,
         profiler_config=None,
+        environment=None,
     ):
         """Create an Amazon SageMaker training job.
 
@@ -522,9 +523,12 @@ def train(  # noqa: C901
                 Series. For more information see:
                 https://docs.aws.amazon.com/sagemaker/latest/dg/API_AlgorithmSpecification.html#SageMaker-Type-AlgorithmSpecification-EnableSageMakerMetricsTimeSeries
                 (default: ``None``).
-            profiler_rule_configs (list[dict]): A list of profiler rule configurations.
+            profiler_rule_configs (list[dict]): A list of profiler rule
+                configurations.src/sagemaker/lineage/artifact.py:285
             profiler_config (dict): Configuration for how profiling information is emitted
                 with SageMaker Profiler. (default: ``None``).
+            environment (dict[str, str]) : Environment variables to be set for
+                use during training job (default: ``None``)
 
         Returns:
             str: ARN of the training job, if it is created.
@@ -556,6 +560,7 @@ def train(  # noqa: C901
             enable_sagemaker_metrics=enable_sagemaker_metrics,
             profiler_rule_configs=profiler_rule_configs,
             profiler_config=profiler_config,
+            environment=environment,
         )
         LOGGER.info("Creating training-job with name: %s", job_name)
         LOGGER.debug("train request: %s", json.dumps(train_request, indent=4))
@@ -588,6 +593,7 @@ def _get_train_request(  # noqa: C901
         enable_sagemaker_metrics=None,
         profiler_rule_configs=None,
         profiler_config=None,
+        environment=None,
     ):
         """Constructs a request compatible for creating an Amazon SageMaker training job.
 
@@ -657,6 +663,8 @@ def _get_train_request(  # noqa: C901
             profiler_rule_configs (list[dict]): A list of profiler rule configurations.
             profiler_config(dict): Configuration for how profiling information is emitted with
                 SageMaker Profiler. (default: ``None``).
+            environment (dict[str, str]) : Environment variables to be set for
+                use during training job (default: ``None``)
 
         Returns:
             Dict: a training request dict
@@ -699,6 +707,9 @@ def _get_train_request(  # noqa: C901
         if hyperparameters and len(hyperparameters) > 0:
             train_request["HyperParameters"] = hyperparameters
 
+        if environment is not None:
+            train_request["Environment"] = environment
+
         if tags is not None:
             train_request["Tags"] = tags
 

doc/api/training/smp_versions/v1.3.0/smd_model_parallel_pytorch.rst renamed to doc/api/training/smp_versions/latest/smd_model_parallel_pytorch.rst

@@ -118,7 +118,7 @@ class TensorFlowModel(sagemaker.model.FrameworkModel):
         logging.ERROR: "error",
         logging.CRITICAL: "crit",
     }
-    LATEST_EIA_VERSION = [2, 0]
+    LATEST_EIA_VERSION = [2, 3]
 
     def __init__(
         self,
@@ -289,7 +289,12 @@ def deploy(
 
     def _eia_supported(self):
         """Return true if TF version is EIA enabled"""
-        return [int(s) for s in self.framework_version.split(".")][:2] <= self.LATEST_EIA_VERSION
+        framework_version = [int(s) for s in self.framework_version.split(".")][:2]
+        return (
+            framework_version != [2, 1]
+            and framework_version != [2, 2]
+            and framework_version <= self.LATEST_EIA_VERSION
+        )
 
     def prepare_container_def(self, instance_type=None, accelerator_type=None):
         """Prepare the container definition.

doc/api/training/smp_versions/v1.3.0/smd_model_parallel_tensorflow.rst renamed to doc/api/training/smp_versions/latest/smd_model_parallel_tensorflow.rst

@@ -91,6 +91,7 @@ def _create_and_fit_estimator(sagemaker_session, tf_version, py_version, instanc
         py_version=py_version,
         framework_version=tf_version,
         distribution={"mpi": {"enabled": True}},
+        disable_profiler=True,
     )
 
     with timeout.timeout(minutes=integ.TRAINING_DEFAULT_TIMEOUT_MINUTES):

src/sagemaker/estimator.py

@@ -92,6 +92,7 @@ def _create_and_fit_estimator(mxnet_version, py_version, sagemaker_session, inst
         py_version=py_version,
         framework_version=mxnet_version,
         distribution={"mpi": {"enabled": True}},
+        disable_profiler=True,
     )
 
     with timeout.timeout(minutes=integ.TRAINING_DEFAULT_TIMEOUT_MINUTES):

src/sagemaker/image_uri_config/tensorflow.json

@@ -58,6 +58,7 @@ def test_huggingface_training(
                 "repo": "https://github.com/huggingface/transformers.git",
                 "branch": f"v{huggingface_training_latest_version}",
             },
+            disable_profiler=True,
         )
 
         train_input = hf.sagemaker_session.upload_data(

src/sagemaker/model.py

@@ -36,6 +36,7 @@
 PARAMETER_SERVER_DISTRIBUTION = {"parameter_server": {"enabled": True}}
 MPI_DISTRIBUTION = {"mpi": {"enabled": True}}
 TAGS = [{"Key": "some-key", "Value": "some-value"}]
+ENV_INPUT = {"env_key1": "env_val1", "env_key2": "env_val2", "env_key3": "env_val3"}
 
 
 def test_mnist_with_checkpoint_config(
@@ -59,6 +60,7 @@ def test_mnist_with_checkpoint_config(
         metric_definitions=[{"Name": "train:global_steps", "Regex": r"global_step\/sec:\s(.*)"}],
         checkpoint_s3_uri=checkpoint_s3_uri,
         checkpoint_local_path=checkpoint_local_path,
+        environment=ENV_INPUT,
     )
     inputs = estimator.sagemaker_session.upload_data(
         path=os.path.join(MNIST_RESOURCE_PATH, "data"), key_prefix="scriptmode/mnist"
@@ -82,7 +84,13 @@ def test_mnist_with_checkpoint_config(
     actual_training_checkpoint_config = sagemaker_session.sagemaker_client.describe_training_job(
         TrainingJobName=training_job_name
     )["CheckpointConfig"]
+    actual_training_environment_variable_config = (
+        sagemaker_session.sagemaker_client.describe_training_job(TrainingJobName=training_job_name)[
+            "Environment"
+        ]
+    )
     assert actual_training_checkpoint_config == expected_training_checkpoint_config
+    assert actual_training_environment_variable_config == ENV_INPUT
 
 
 def test_server_side_encryption(sagemaker_session, tf_full_version, tf_full_py_version):
@@ -141,6 +149,7 @@ def test_mnist_distributed(
         framework_version=tensorflow_training_latest_version,
         py_version=tensorflow_training_latest_py_version,
         distribution=PARAMETER_SERVER_DISTRIBUTION,
+        disable_profiler=True,
     )
     inputs = estimator.sagemaker_session.upload_data(
         path=os.path.join(MNIST_RESOURCE_PATH, "data"), key_prefix="scriptmode/distributed_mnist"

src/sagemaker/session.py

@@ -32,5 +32,4 @@ echo "set SAGEMAKER_ROLE_ARN=$SAGEMAKER_ROLE_ARN"
 ./amazon-sagemaker-examples/sagemaker-python-sdk/tensorflow_moving_from_framework_mode_to_script_mode/tensorflow_moving_from_framework_mode_to_script_mode.ipynb \
 ./amazon-sagemaker-examples/sagemaker-python-sdk/tensorflow_script_mode_pipe_mode/tensorflow_script_mode_pipe_mode.ipynb \
 ./amazon-sagemaker-examples/sagemaker-python-sdk/tensorflow_script_mode_quickstart/tensorflow_script_mode_quickstart.ipynb \
-./amazon-sagemaker-examples/sagemaker-python-sdk/tensorflow_script_mode_using_shell_commands/tensorflow_script_mode_using_shell_commands.ipynb \
 ./amazon-sagemaker-examples/sagemaker-python-sdk/tensorflow_serving_using_elastic_inference_with_your_own_model/tensorflow_serving_pretrained_model_elastic_inference.ipynb

src/sagemaker/tensorflow/model.py

@@ -149,6 +149,7 @@ def _create_train_job(version, base_framework_version):
         "tags": None,
         "vpc_config": None,
         "metric_definitions": None,
+        "environment": None,
         "experiment_config": None,
         "debugger_hook_config": {
             "CollectionConfigurations": [],

tests/data/cuteCat.jpg

@@ -130,6 +130,7 @@ def _create_train_job(tf_version, horovod=False, ps=False, py_version="py2", smd
         "tags": None,
         "vpc_config": None,
         "metric_definitions": None,
+        "environment": None,
         "experiment_config": None,
         "profiler_rule_configs": [
             {

tests/integ/test_horovod.py

@@ -20,6 +20,8 @@
 
 REGION = "us-west-2"
 
+ENV_INPUT = {"env_key1": "env_val1", "env_key2": "env_val2", "env_key3": "env_val3"}
+
 
 @pytest.fixture()
 def sagemaker_session():
@@ -68,6 +70,26 @@ def test_framework_name(sagemaker_session):
     assert tf._framework_name == "tensorflow"
 
 
+def test_tf_add_environment_variables(sagemaker_session):
+    tf = _build_tf(
+        sagemaker_session,
+        framework_version="1.15.2",
+        py_version="py3",
+        environment=ENV_INPUT,
+    )
+    assert tf.environment == ENV_INPUT
+
+
+def test_tf_miss_environment_variables(sagemaker_session):
+    tf = _build_tf(
+        sagemaker_session,
+        framework_version="1.15.2",
+        py_version="py3",
+        environment=None,
+    )
+    assert not tf.environment
+
+
 def test_enable_sm_metrics(sagemaker_session):
     tf = _build_tf(
         sagemaker_session,

tests/integ/test_horovod_mx.py

@@ -143,6 +143,7 @@ def _create_train_job(version, py_version):
         "tags": None,
         "vpc_config": None,
         "metric_definitions": None,
+        "environment": None,
         "experiment_config": None,
         "debugger_hook_config": {
             "CollectionConfigurations": [],