Merge branch 'zwei' into master

Author: laurenyu

doc/v2.rst

@@ -203,6 +203,12 @@ To view logs after attaching a training job to an estimator, use :func:`sagemake
 until the completion of the Hyperparameter Tuning Job or Batch Transform Job, respectively.
 To make the function non-blocking, use ``wait=False``.
 
+XGBoost Predictor
+-----------------
+
+The default serializer of ``sagemaker.xgboost.model.XGBoostPredictor`` has been changed from ``NumpySerializer`` to ``LibSVMSerializer``.
+
+
 Parameter and Class Name Changes
 ================================
 
@@ -263,6 +269,8 @@ The follow serializer/deserializer classes have been renamed and/or moved:
 | ``sagemaker.predictor._JsonDeserializer``              | ``sagemaker.deserializers.JSONDeserializer``          |
 +--------------------------------------------------------+-------------------------------------------------------+
 
+``sagemaker.serializers.LibSVMSerializer`` has been added in v2.0.
+
 ``distributions``
 ~~~~~~~~~~~~~~~~~
 

src/sagemaker/algorithm.py

@@ -229,13 +229,13 @@ def hyperparameters(self):
         """
         return self.hyperparam_dict
 
-    def train_image(self):
+    def training_image_uri(self):
         """Returns the docker image to use for training.
 
         The fit() method, that does the model training, calls this method to
         find the image to use for model training.
         """
-        raise RuntimeError("train_image is never meant to be called on Algorithm Estimators")
+        raise RuntimeError("training_image_uri is never meant to be called on Algorithm Estimators")
 
     def enable_network_isolation(self):
         """Return True if this Estimator will need network isolation to run.

src/sagemaker/amazon/amazon_estimator.py

@@ -91,7 +91,7 @@ def __init__(
         )
         self._data_location = data_location
 
-    def train_image(self):
+    def training_image_uri(self):
         """Placeholder docstring"""
         return image_uris.retrieve(
             self.repo_name, self.sagemaker_session.boto_region_name, version=self.repo_version,

src/sagemaker/cli/compatibility/v2/ast_transformer.py

@@ -35,6 +35,7 @@
     modifiers.renamed_params.SessionCreateEndpointImageURIRenamer(),
     modifiers.training_params.TrainPrefixRemover(),
     modifiers.training_input.TrainingInputConstructorRefactor(),
+    modifiers.training_input.ShuffleConfigModuleRenamer(),
     modifiers.serde.SerdeConstructorRenamer(),
 ]
 
@@ -51,6 +52,7 @@
     modifiers.predictors.PredictorImportFromRenamer(),
     modifiers.tfs.TensorFlowServingImportFromRenamer(),
     modifiers.training_input.TrainingInputImportFromRenamer(),
+    modifiers.training_input.ShuffleConfigImportFromRenamer(),
     modifiers.serde.SerdeImportFromAmazonCommonRenamer(),
     modifiers.serde.SerdeImportFromPredictorRenamer(),
 ]

src/sagemaker/cli/compatibility/v2/modifiers/training_input.py

@@ -100,3 +100,73 @@ def modify_node(self, node):
             if node.module == "sagemaker.session":
                 node.module = "sagemaker.inputs"
         return node
+
+
+class ShuffleConfigModuleRenamer(Modifier):
+    """A class to change ``ShuffleConfig`` usage to use ``sagemaker.inputs.ShuffleConfig``."""
+
+    def node_should_be_modified(self, node):
+        """Checks if the ``ast.Call`` node instantiates a class of interest.
+
+        This looks for the following calls:
+
+        - ``sagemaker.session.ShuffleConfig``
+        - ``session.ShuffleConfig``
+
+        Args:
+            node (ast.Call): a node that represents a function call. For more,
+                see https://docs.python.org/3/library/ast.html#abstract-grammar.
+
+        Returns:
+            bool: If the ``ast.Call`` instantiates a class of interest.
+        """
+        if isinstance(node.func, ast.Name):
+            return False
+
+        return matching.matches_name_or_namespaces(
+            node, "ShuffleConfig", ("sagemaker.session", "session")
+        )
+
+    def modify_node(self, node):
+        """Modifies the ``ast.Call`` node to call ``sagemaker.inputs.ShuffleConfig``.
+
+        Args:
+            node (ast.Call): a node that represents a ``sagemaker.session.ShuffleConfig``
+                constructor.
+
+        Returns:
+            ast.Call: the original node, with its namespace changed to use the ``inputs`` module.
+        """
+        _rename_namespace(node, "session")
+        return node
+
+
+class ShuffleConfigImportFromRenamer(Modifier):
+    """A class to update import statements of ``ShuffleConfig``."""
+
+    def node_should_be_modified(self, node):
+        """Checks if the import statement imports ``sagemaker.session.ShuffleConfig``.
+
+        Args:
+            node (ast.ImportFrom): a node that represents a ``from ... import ... `` statement.
+                For more, see https://docs.python.org/3/library/ast.html#abstract-grammar.
+
+        Returns:
+            bool: If the import statement imports ``sagemaker.session.ShuffleConfig``.
+        """
+        return node.module == "sagemaker.session" and any(
+            name.name == "ShuffleConfig" for name in node.names
+        )
+
+    def modify_node(self, node):
+        """Changes the ``ast.ImportFrom`` node's namespace to ``sagemaker.inputs``.
+
+        Args:
+            node (ast.ImportFrom): a node that represents a ``from ... import ... `` statement.
+                For more, see https://docs.python.org/3/library/ast.html#abstract-grammar.
+
+        Returns:
+            ast.ImportFrom: the original node, with its module modified to ``"sagemaker.inputs"``.
+        """
+        node.module = "sagemaker.inputs"
+        return node

src/sagemaker/cli/framework_upgrade.py

@@ -41,9 +41,9 @@ def get_latest_values(existing_content, scope=None):
             )
 
     latest_version = list(existing_content["versions"].keys())[-1]
-    registries = existing_content["versions"][latest_version]["registries"]
-    py_versions = existing_content["versions"][latest_version]["py_versions"]
-    repository = existing_content["versions"][latest_version]["repository"]
+    registries = existing_content["versions"][latest_version].get("registries", None)
+    py_versions = existing_content["versions"][latest_version].get("py_versions", None)
+    repository = existing_content["versions"][latest_version].get("repository", None)
 
     return registries, py_versions, repository
 
@@ -92,8 +92,9 @@ def add_dlc_framework_version(
     new_version = {
         "registries": registries,
         "repository": repository,
-        "py_versions": py_versions,
     }
+    if py_versions:
+        new_version["py_versions"] = py_versions
     existing_content[scope]["versions"][full_version] = new_version
 
 
@@ -128,10 +129,11 @@ def add_algo_version(
             existing_content["scope"].append(scope)
 
     new_version = {
-        "py_versions": py_versions,
         "registries": registries,
         "repository": repository,
     }
+    if py_versions:
+        new_version["py_versions"] = py_versions
     if tag_prefix:
         new_version["tag_prefix"] = tag_prefix
     existing_content["versions"][full_version] = new_version
@@ -171,7 +173,8 @@ def add_version(
         py_versions (str): Supported Python versions (e.g. "py3,py37").
         tag_prefix (str): Algorithm image's tag prefix.
      """
-    py_versions = py_versions.split(",")
+    if py_versions:
+        py_versions = py_versions.split(",")
     processors = processors.split(",")
     latest_registries, latest_py_versions, latest_repository = get_latest_values(
         existing_content, scope

src/sagemaker/estimator.py

@@ -285,7 +285,7 @@ def __init__(
         self._enable_network_isolation = enable_network_isolation
 
     @abstractmethod
-    def train_image(self):
+    def training_image_uri(self):
         """Return the Docker image to use for training.
 
         The :meth:`~sagemaker.estimator.EstimatorBase.fit` method, which does
@@ -329,7 +329,7 @@ def _ensure_base_job_name(self):
         """Set ``self.base_job_name`` if it is not set already."""
         # honor supplied base_job_name or generate it
         if self.base_job_name is None:
-            self.base_job_name = base_name_from_image(self.train_image())
+            self.base_job_name = base_name_from_image(self.training_image_uri())
 
     def _get_or_create_name(self, name=None):
         """Generate a name based on the base job name or training image if needed.
@@ -507,7 +507,7 @@ def fit(self, inputs=None, wait=True, logs="All", job_name=None, experiment_conf
 
     def _compilation_job_name(self):
         """Placeholder docstring"""
-        base_name = self.base_job_name or base_name_from_image(self.train_image())
+        base_name = self.base_job_name or base_name_from_image(self.training_image_uri())
         return name_from_base("compilation-" + base_name)
 
     def compile_model(
@@ -1083,7 +1083,7 @@ def start_new(cls, estimator, inputs, experiment_config):
         if isinstance(estimator, sagemaker.algorithm.AlgorithmEstimator):
             train_args["algorithm_arn"] = estimator.algorithm_arn
         else:
-            train_args["image_uri"] = estimator.train_image()
+            train_args["image_uri"] = estimator.training_image_uri()
 
         if estimator.debugger_rule_configs:
             train_args["debugger_rule_configs"] = estimator.debugger_rule_configs
@@ -1350,7 +1350,7 @@ def __init__(
             enable_network_isolation=enable_network_isolation,
         )
 
-    def train_image(self):
+    def training_image_uri(self):
         """Returns the docker image to use for training.
 
         The fit() method, that does the model training, calls this method to
@@ -1424,7 +1424,7 @@ def predict_wrapper(endpoint, session):
             kwargs["enable_network_isolation"] = self.enable_network_isolation()
 
         return Model(
-            image_uri or self.train_image(),
+            image_uri or self.training_image_uri(),
             self.model_data,
             role,
             vpc_config=self.get_vpc_config(vpc_config_override),
@@ -1826,7 +1826,7 @@ class constructor
 
         return init_params
 
-    def train_image(self):
+    def training_image_uri(self):
         """Return the Docker image to use for training.
 
         The :meth:`~sagemaker.estimator.EstimatorBase.fit` method, which does

src/sagemaker/fw_utils.py

@@ -49,40 +49,6 @@
 SINGLE_GPU_INSTANCE_TYPES = ("ml.p2.xlarge", "ml.p3.2xlarge")
 
 
-def is_version_equal_or_higher(lowest_version, framework_version):
-    """Determine whether the ``framework_version`` is equal to or higher than
-    ``lowest_version``
-
-    Args:
-        lowest_version (List[int]): lowest version represented in an integer
-            list
-        framework_version (str): framework version string
-
-    Returns:
-        bool: Whether or not ``framework_version`` is equal to or higher than
-            ``lowest_version``
-    """
-    version_list = [int(s) for s in framework_version.split(".")]
-    return version_list >= lowest_version[0 : len(version_list)]
-
-
-def is_version_equal_or_lower(highest_version, framework_version):
-    """Determine whether the ``framework_version`` is equal to or lower than
-    ``highest_version``
-
-    Args:
-        highest_version (List[int]): highest version represented in an integer
-            list
-        framework_version (str): framework version string
-
-    Returns:
-        bool: Whether or not ``framework_version`` is equal to or lower than
-            ``highest_version``
-    """
-    version_list = [int(s) for s in framework_version.split(".")]
-    return version_list <= highest_version[0 : len(version_list)]
-
-
 def validate_source_dir(script, directory):
     """Validate that the source directory exists and it contains the user script
     Args: