Merge branch 'dev' into feat/jumpstart-docs-model-table

Author: bencrabtree

CHANGELOG.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## v2.75.0 (2022-02-05)
+
+### Features
+
+ * JumpStart Integration
+ * Adds support for async inference
+ * Update instance types for integ test
+
+### Bug Fixes and Other Changes
+
+ * Revert "feature: CompilationStep support for Sagemaker Pipelines
+ * gpu use p3/p2 per avail for region
+ * jumpstart typo
+ * pin pytest-xdist to avoid release failures
+ * set sagemaker_connection and image_uri in register method
+ * update to incorporate black v22, pin tox versions
+ * Add deprecation warning in Clarify DataConfig
+
+### Documentation Changes
+
+ * Jumpstart doc strings and added new sections
+ * Add Jumpstart support documentation
+
 ## v2.74.0 (2022-01-26)
 
 ### Features

README.rst

@@ -90,6 +90,7 @@ SageMaker Python SDK is tested on:
 - Python 3.6
 - Python 3.7
 - Python 3.8
+- Python 3.9
 
 AWS Permissions
 ~~~~~~~~~~~~~~~

VERSION

@@ -1 +1 @@
-2.74.1.dev0
+2.75.1.dev0

doc/workflows/pipelines/sagemaker.workflow.pipelines.rst

@@ -140,12 +140,12 @@ Steps
 
 .. autoclass:: sagemaker.workflow.lambda_step.LambdaStep
 
-.. autoclass:: sagemaker.workflow.steps.CompilationStep
-
 .. autoclass:: sagemaker.workflow.quality_check_step.QualityCheckConfig
 
 .. autoclass:: sagemaker.workflow.quality_check_step.QualityCheckStep
 
 .. autoclass:: sagemaker.workflow.clarify_check_step.ClarifyCheckConfig
 
 .. autoclass:: sagemaker.workflow.clarify_check_step.ClarifyCheckStep
+
+.. autoclass:: sagemaker.workflow.fail_step.FailStep

setup.py

@@ -33,7 +33,7 @@ def read_version():
 
 # Declare minimal set for installation
 required_packages = [
-    "attrs",
+    "attrs==20.3.0",
     "boto3>=1.20.21",
     "google-pasta",
     "numpy>=1.9.0",
@@ -49,36 +49,38 @@ def read_version():
 # Specific use case dependencies
 extras = {
     "local": [
-        "urllib3>=1.21.1,!=1.25,!=1.25.1",
-        "docker-compose>=1.25.2",
-        "docker==5.0.0",
-        "PyYAML>=5.3, <6",  # PyYAML version has to match docker-compose requirements
+        "urllib3==1.26.8",
+        "docker-compose==1.29.2",
+        "docker~=5.0.0",
+        "PyYAML==5.4.1",  # PyYAML version has to match docker-compose requirements
     ],
-    "scipy": ["scipy>=0.19.0"],
+    "scipy": ["scipy==1.5.4"],
 }
 # Meta dependency groups
 extras["all"] = [item for group in extras.values() for item in group]
 # Tests specific dependencies (do not need to be included in 'all')
 extras["test"] = (
     [
         extras["all"],
-        "tox",
-        "flake8",
-        "pytest<6.1.0",
-        "pytest-cov",
-        "pytest-rerunfailures",
-        "pytest-timeout",
+        "tox==3.24.5",
+        "flake8==4.0.1",
+        "pytest==6.0.2",
+        "pytest-cov==3.0.0",
+        "pytest-rerunfailures==10.2",
+        "pytest-timeout==2.1.0",
         "pytest-xdist==2.4.0",
-        "coverage<6.2",
-        "mock",
-        "contextlib2",
-        "awslogs",
-        "black",
+        "coverage>=5.2, <6.2",
+        "mock==4.0.3",
+        "contextlib2==21.6.0",
+        "awslogs==0.14.0",
+        "black==22.1.0",
         "stopit==1.1.2",
-        "apache-airflow==1.10.11",
-        "fabric>=2.0",
-        "requests>=2.20.0, <3",
-        "sagemaker-experiments",
+        "apache-airflow==2.2.3",
+        "apache-airflow-providers-amazon==3.0.0",
+        "attrs==20.3.0",
+        "fabric==2.6.0",
+        "requests==2.27.1",
+        "sagemaker-experiments==0.1.35",
     ],
 )
 

src/sagemaker/inputs.py

@@ -136,67 +136,6 @@ class CreateModelInput(object):
     accelerator_type: str = attr.ib(default=None)
 
 
-@attr.s
-class CompilationInput(object):
-    """Create a class containing all the parameters.
-
-    It can be used when calling ``sagemaker.model.Model.compile_model()``
-
-    Parameters:
-        target_instance_type(str): Identifies the device that you want to
-                run your model after compilation, for example: ml_c5. For allowed
-                strings see
-                https://docs.aws.amazon.com/sagemaker/latest/dg/API_OutputConfig.html.
-        input_shape(str): Specifies the name and shape of the expected
-                inputs for your trained model in json dictionary form, for
-                example: {'data': [1,3,1024,1024]}, or {'var1': [1,1,28,28],
-                'var2': [1,1,28,28]}
-        output_path(str): Specifies where to store the compiled model
-        framework (str, optional): The framework that is used to train the original
-                model. Allowed values: 'mxnet', 'tensorflow', 'keras', 'pytorch',
-                'onnx', 'xgboost' (default: None)
-        framework_version (str, optional): The version of the framework (default: None)
-        compile_max_run (int, optional): Timeout in seconds for compilation (default:
-            15 * 60). After this amount of time Amazon SageMaker Neo
-            terminates the compilation job regardless of its current status.
-        tags (list[dict], optional): List of tags for labeling a compilation job.
-            For more, see
-            https://docs.aws.amazon.com/sagemaker/latest/dg/API_Tag.html.
-        job_name (str, optional): The name of the compilation job (default: None)
-        target_platform_os (str, optional): Target Platform OS, for example: 'LINUX'.
-            (default: None)
-            For allowed strings see
-            https://docs.aws.amazon.com/sagemaker/latest/dg/API_OutputConfig.html.
-            It can be used instead of target_instance_family.
-        target_platform_arch (str, optional): Target Platform Architecture, for example: 'X86_64'.
-            (default: None)
-            For allowed strings see
-            https://docs.aws.amazon.com/sagemaker/latest/dg/API_OutputConfig.html.
-            It can be used instead of target_instance_family.
-        target_platform_accelerator (str, optional): Target Platform Accelerator,
-            for example: 'NVIDIA'. (default: None)
-            For allowed strings see
-            https://docs.aws.amazon.com/sagemaker/latest/dg/API_OutputConfig.html.
-            It can be used instead of target_instance_family.
-        compiler_options (dict, optional): Additional parameters for compiler. (default: None)
-            Compiler Options are TargetPlatform / target_instance_family specific. See
-            https://docs.aws.amazon.com/sagemaker/latest/dg/API_OutputConfig.html for details.
-    """
-
-    target_instance_type: str = attr.ib(default=None)
-    input_shape: dict = attr.ib(factory=dict)
-    output_path: str = attr.ib(default=None)
-    framework: str = attr.ib(default=None)
-    framework_version: str = attr.ib(default=None)
-    compile_max_run: int = attr.ib(default=15 * 60)
-    tags: list = attr.ib(factory=list)
-    job_name: str = attr.ib(default=None)
-    target_platform_os: str = attr.ib(default=None)
-    target_platform_arch: str = attr.ib(default=None)
-    target_platform_accelerator: str = attr.ib(default=None)
-    compiler_options: dict = attr.ib(default=None)
-
-
 @attr.s
 class TransformInput(object):
     """Create a class containing all the parameters.

src/sagemaker/model.py

@@ -23,14 +23,12 @@
 import sagemaker
 from sagemaker import (
     fw_utils,
-    image_uris,
     local,
     s3,
     session,
     utils,
     git_utils,
 )
-from sagemaker.inputs import CompilationInput
 from sagemaker.deprecations import removed_kwargs
 from sagemaker.predictor import PredictorBase
 from sagemaker.serverless import ServerlessInferenceConfig
@@ -658,86 +656,6 @@ def _compilation_job_config(
             "job_name": job_name,
         }
 
-    def _get_compilation_args(self, estimator, inputs):
-        """Constructs a dict of arguments for an Amazon SageMaker compilation job from estimator.
-
-        Args:
-            estimator (sagemaker.estimator.EstimatorBase): Estimator object
-                created by the user.
-            inputs (CompilationInput): class containing all the parameters that
-                can be used when calling ``sagemaker.model.Model.compile_model()``
-        """
-
-        if not isinstance(inputs, CompilationInput):
-            raise TypeError("Your inputs must be provided as CompilationInput objects.")
-        target_instance_family = inputs.target_instance_type
-        input_shape = inputs.input_shape
-        output_path = inputs.output_path
-        role = estimator.role
-        compile_max_run = inputs.compile_max_run
-        job_name = estimator._compilation_job_name()
-        framework = inputs.framework or self._framework()
-        if framework is None:
-            raise ValueError(
-                "You must specify framework, allowed values {}".format(NEO_ALLOWED_FRAMEWORKS)
-            )
-        if framework not in NEO_ALLOWED_FRAMEWORKS:
-            raise ValueError(
-                "You must provide valid framework, allowed values {}".format(NEO_ALLOWED_FRAMEWORKS)
-            )
-        if self.model_data is None:
-            raise ValueError("You must provide an S3 path to the compressed model artifacts.")
-        tags = inputs.tags
-        target_platform_os = inputs.target_platform_os
-        target_platform_arch = inputs.target_platform_arch
-        target_platform_accelerator = inputs.target_platform_accelerator
-        compiler_options = inputs.compiler_options
-        framework_version = inputs.framework_version or self._get_framework_version()
-
-        return self._compilation_job_config(
-            target_instance_family,
-            input_shape,
-            output_path,
-            role,
-            compile_max_run,
-            job_name,
-            framework,
-            tags,
-            target_platform_os,
-            target_platform_arch,
-            target_platform_accelerator,
-            compiler_options,
-            framework_version,
-        )
-
-    def _compilation_image_uri(self, region, target_instance_type, framework, framework_version):
-        """Retrieve the Neo or Inferentia image URI.
-
-        Args:
-            region (str): The AWS region.
-            target_instance_type (str): Identifies the device on which you want to run
-                your model after compilation, for example: ml_c5. For valid values, see
-                https://docs.aws.amazon.com/sagemaker/latest/dg/API_OutputConfig.html.
-            framework (str): The framework name.
-            framework_version (str): The framework version.
-        """
-        framework_prefix = ""
-        framework_suffix = ""
-
-        if framework == "xgboost":
-            framework_suffix = "-neo"
-        elif target_instance_type.startswith("ml_inf"):
-            framework_prefix = "inferentia-"
-        else:
-            framework_prefix = "neo-"
-
-        return image_uris.retrieve(
-            "{}{}{}".format(framework_prefix, framework, framework_suffix),
-            region,
-            instance_type=target_instance_type,
-            version=framework_version,
-        )
-
     def package_for_edge(
         self,
         output_path,
@@ -902,12 +820,7 @@ def compile(
             if target_instance_family == "ml_eia2":
                 pass
             elif target_instance_family.startswith("ml_"):
-                self.image_uri = self._compilation_image_uri(
-                    self.sagemaker_session.boto_region_name,
-                    target_instance_family,
-                    framework,
-                    framework_version,
-                )
+                self.image_uri = job_status.get("InferenceImage", None)
                 self._is_compiled_model = True
             else:
                 LOGGER.warning(

src/sagemaker/session.py

@@ -1845,57 +1845,6 @@ def compile_model(
         LOGGER.info("Creating compilation-job with name: %s", job_name)
         self.sagemaker_client.create_compilation_job(**compilation_job_request)
 
-    def _get_compilation_request(
-        self,
-        job_name,
-        input_model_config,
-        output_model_config,
-        role,
-        stop_condition,
-        tags=None,
-        vpc_config=None,
-    ):
-        """Construct CreateCompilationJob request
-
-        Args:
-            input_model_config (dict): the trained model and the Amazon S3 location where it is
-                stored.
-            output_model_config (dict): Identifies the Amazon S3 location where you want Amazon
-                SageMaker Neo to save the results of compilation job
-            role (str): An AWS IAM role (either name or full ARN). The Amazon SageMaker Neo
-                compilation jobs use this role to access model artifacts. You must grant
-                sufficient permissions to this role.
-            job_name (str): Name of the compilation job being created.
-            stop_condition (dict): Defines when compilation job shall finish. Contains entries
-                that can be understood by the service like ``MaxRuntimeInSeconds``.
-            tags (list[dict]): List of tags for labeling a compile model job. For more, see
-                https://docs.aws.amazon.com/sagemaker/latest/dg/API_Tag.html.
-            vpc_config (dict): Contains values for VpcConfig:
-                * subnets (list[str]): List of subnet ids.
-                The key in vpc_config is 'Subnets'.
-                * security_group_ids (list[str]): List of security group ids.
-                The key in vpc_config is 'SecurityGroupIds'.
-        Returns:
-            dict: A dictionary for CreateCompilationJob request
-        """
-
-        compilation_request = {
-            "InputConfig": input_model_config,
-            "OutputConfig": output_model_config,
-            "RoleArn": role,
-            "StoppingCondition": stop_condition,
-            "CompilationJobName": job_name,
-        }
-
-        tags = _append_project_tags(tags)
-        if tags is not None:
-            compilation_request["Tags"] = tags
-
-        if vpc_config is not None:
-            compilation_request["VpcConfig"] = vpc_config
-
-        return compilation_request
-
     def package_model_for_edge(
         self,
         output_model_config,