Merge branch 'master' into pt_171_smddp

Author: ChaiBapchya

.pydocstylerc

@@ -1,4 +1,4 @@
 [pydocstyle]
 inherit = false
-ignore = D104,D107,D202,D203,D205,D212,D213,D214,D400,D401,D404,D406,D407,D411,D413,D414,D415,D417
+ignore = D104,D107,D202,D203,D213,D214,D400,D401,D404,D406,D407,D411,D413,D414,D415,D417
 match = (?!record_pb2).*\.py

CHANGELOG.md

@@ -1,5 +1,182 @@
 # Changelog
 
+## v2.24.5 (2021-02-12)
+
+### Bug Fixes and Other Changes
+
+ * test_tag/test_tags method assert fix in association tests
+
+### Documentation Changes
+
+ * removing mention of TF 2.4 from SM distributed model parallel docs
+ * adding details about mpi options, other small updates
+
+## v2.24.4 (2021-02-09)
+
+### Bug Fixes and Other Changes
+
+ * add integration test for listing artifacts by type
+ * List Associations integ tests
+
+## v2.24.3 (2021-02-04)
+
+### Bug Fixes and Other Changes
+
+ * Remove pytest fixture and fix test_tag/s method
+
+## v2.24.2 (2021-02-03)
+
+### Bug Fixes and Other Changes
+
+ * use 3.5 version of get-pip.py
+ * SM DDP release notes/changelog files
+
+### Documentation Changes
+
+ * adding versioning to sm distributed data parallel docs
+
+## v2.24.1 (2021-01-28)
+
+### Bug Fixes and Other Changes
+
+ * fix collect-tests tox env
+ * create profiler specific unsupported regions
+ * Update smd_model_parallel_pytorch.rst
+
+## v2.24.0 (2021-01-22)
+
+### Features
+
+ * add support for Std:Join for pipelines
+ * Map image name to image uri
+ * friendly names for short URIs
+
+### Bug Fixes and Other Changes
+
+ * increase allowed time for search to get updated
+ * refactor distribution config construction
+
+### Documentation Changes
+
+ * Add SMP 1.2.0 API docs
+
+## v2.23.6 (2021-01-20)
+
+### Bug Fixes and Other Changes
+
+ * add artifact, action, context to virsualizer
+
+## v2.23.5 (2021-01-18)
+
+### Bug Fixes and Other Changes
+
+ * increase time allowed for trial components to index
+
+## v2.23.4.post0 (2021-01-14)
+
+### Documentation Changes
+
+ * update predict_fn implementation for PyTorch EIA 1.5.1
+
+## v2.23.4 (2021-01-13)
+
+### Bug Fixes and Other Changes
+
+ * remove captureWarninig setting
+
+## v2.23.3 (2021-01-12)
+
+### Bug Fixes and Other Changes
+
+ * improve optional dependency error message
+ * add debugger rule container account in PDT
+ * assert step execution first in pipeline test
+ * add service inserted fields to generated Hive DDL
+
+### Documentation Changes
+
+ * fix description for max_wait
+ * use correct classpath in V2 alias documentation.
+ * Bad arg name in feat-store ingestion manager
+
+## v2.23.2 (2021-01-06)
+
+### Bug Fixes and Other Changes
+
+ * remove shell=True in subprocess.check_output
+ * use SecurityConfig dict key
+
+### Documentation Changes
+
+ * remove D212 from ignore to comply with PEP257 standards
+
+## v2.23.1 (2020-12-29)
+
+### Bug Fixes and Other Changes
+
+ * update git utils temp file
+ * Allow online store only FeatureGroups
+
+### Documentation Changes
+
+ * inform contributors when not to mark integration tests as canaries
+ * adding change log for smd model parallel
+
+## v2.23.0 (2020-12-23)
+
+### Features
+
+ * Add support for actions in debugger rules.
+
+### Bug Fixes and Other Changes
+
+ * include sparkml 2.4 in image uri config properly
+ * Mount metadata dir only if it exists
+ * allow urllib3 1.26
+
+## v2.22.0 (2020-12-22)
+
+### Features
+
+ * Support local mode for Amazon SageMaker Processing jobs
+
+### Bug Fixes and Other Changes
+
+ * Add API enhancements for SMP
+ * adjust naming convention; fix links
+ * lower value used in featurestore test
+
+### Documentation Changes
+
+ * Update GTDD instructions
+
+## v2.21.0 (2020-12-21)
+
+### Features
+
+ * remove D205 to enable PEP257 Docstring Conventions
+
+### Bug Fixes and Other Changes
+
+ * Pin smdebug-rulesconfig to 1.0.0
+ * use itertuples to ingest pandas dataframe to FeatureStore
+
+## v2.20.0 (2020-12-16)
+
+### Features
+
+ * add dataset definition support for processing jobs
+
+### Bug Fixes and Other Changes
+
+ * include workflow integ tests with clarify and debugger enabled
+ * only run DataParallel and EdgePackaging tests in supported regions
+
+### Documentation Changes
+
+ * fix smp code example, add note for CUDA 11 to sdp
+ * adding note about CUDA 11 to SMP. Small title update PyTorch
+
 ## v2.19.0 (2020-12-08)
 
 ### Features

CONTRIBUTING.md

@@ -111,6 +111,7 @@ If you are writing or modifying a test that creates a SageMaker job (training, t
 1. Run all the unit tests as per [Run the Unit Tests](#run-the-unit-tests), and verify that all checks and tests pass.
    1. Note that this also runs tools that may be necessary for the automated build to pass (ex: code reformatting by 'black').
 1. If your changes include documentation changes, please see the [Documentation Guidelines](#documentation-guidelines).
+1. If you include integration tests, do not mark them as canaries if they will not run in all regions.
 
 
 ### Commit Your Change

VERSION

@@ -1 +1 @@
-2.19.1.dev0
+2.24.6.dev0

buildspec-deploy.yml

@@ -3,7 +3,16 @@ version: 0.2
 phases:
   build:
     commands:
-      - PACKAGE_FILE="$CODEBUILD_SRC_DIR_ARTIFACT_1/sagemaker-*.tar.gz"
+      # prepare the release (update versions, changelog etc.)
+      - git-release --prepare
+
+      # generate the distribution package
+      - python3 setup.py sdist
+
+      # publish the release to github
+      - git-release --publish
+
+      - PACKAGE_FILE="dist/sagemaker-*.tar.gz"
       - PYPI_USER=$(aws secretsmanager get-secret-value --secret-id /codebuild/pypi/user --query SecretString --output text)
       - PYPI_PASSWORD=$(aws secretsmanager get-secret-value --secret-id /codebuild/pypi/password --query SecretString --output text)
       - GPG_PRIVATE_KEY=$(aws secretsmanager get-secret-value --secret-id /codebuild/gpg/private_key --query SecretString --output text)

buildspec-release.yml

@@ -3,9 +3,6 @@ version: 0.2
 phases:
   build:
     commands:
-      # prepare the release (update versions, changelog etc.)
-      - git-release --prepare
-
       # run linters
       - tox -e flake8,pylint
 
@@ -21,16 +18,4 @@ phases:
         tox -e py36,py37,py38 -- tests/unit
 
       # run a subset of the integration tests
-      - IGNORE_COVERAGE=- tox -e py36 -- tests/integ -m canary_quick -n 64 --boxed --reruns 2
-
-      # generate the distribution package
-      - python3 setup.py sdist
-
-      # publish the release to github
-      - git-release --publish
-
-artifacts:
-  files:
-    - dist/sagemaker-*.tar.gz
-  name: ARTIFACT_1
-  discard-paths: yes
+      - IGNORE_COVERAGE=- tox -e py36 -- tests/integ -m not (local_mode or slow_test) -n 32 --boxed --reruns 2

buildspec-slowtests.yml

@@ -0,0 +1,15 @@
+version: 0.2
+
+phases:
+  pre_build:
+    commands:
+      - start-dockerd
+
+  build:
+    commands:
+      - IGNORE_COVERAGE=-
+
+      # slow tests
+      - start_time=`date +%s`
+      - execute-command-if-has-matching-changes "tox -e py38 -- tests/integ -m slow_test -n 16 --durations 0" "tests/integ" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec-slowtests.yml"
+      - ./ci-scripts/displaytime.sh 'py38 slow tests' $start_time

buildspec.yml

@@ -16,7 +16,7 @@ phases:
 
       - start_time=`date +%s`
       - |
-        execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py38 -- tests/integ -m \"not local_mode and not cron\" -n 384 --reruns 3 --reruns-delay 15 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "src/sagemaker/image_uri_config/*.json" "setup.py" "setup.cfg" "buildspec.yml"
+        execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py38 -- tests/integ -m \"not local_mode and not cron and not slow_test\" -n 384 --reruns 3 --reruns-delay 15 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "src/sagemaker/image_uri_config/*.json" "setup.py" "setup.cfg" "buildspec.yml"
       - ./ci-scripts/displaytime.sh 'py38 tests/integ' $start_time
 
   post_build:

doc/api/training/smd_data_parallel_pytorch.rst renamed to doc/api/training/sdp_versions/v1.0.0/smd_data_parallel_pytorch.rst

@@ -1,6 +1,6 @@
-####################
-PyTorch Guide to SDP
-####################
+##############################################################
+PyTorch Guide to SageMaker's distributed data parallel library
+##############################################################
 
 .. admonition:: Contents
 
@@ -13,16 +13,16 @@ Modify a PyTorch training script to use SageMaker data parallel
 ======================================================================
 
 The following steps show you how to convert a PyTorch training script to
-utilize SageMaker Distributed Data Parallel (SDP).
+utilize SageMaker's distributed data parallel library.
 
-The SDP APIs are designed to be close to PyTorch Distributed Data
-Parallel (DDP) APIs. Please see `SageMaker Distributed Data Parallel
-PyTorch API documentation <http://#>`__ for additional details on each
-API SDP offers for PyTorch.
+The distributed data parallel library APIs are designed to be close to PyTorch Distributed Data
+Parallel (DDP) APIs.
+See `SageMaker distributed data parallel PyTorch examples <https://sagemaker-examples.readthedocs.io/en/latest/training/distributed_training/index.html#pytorch-distributed>`__ for additional details on how to implement the data parallel library
+API offered for PyTorch.
 
 
--  First import SDP’s PyTorch client and initialize it. You also import
-   the SDP module for distributed training.
+-  First import the distributed data parallel library’s PyTorch client and initialize it. You also import
+   the distributed data parallel library module for distributed training.
 
    .. code:: python
 
@@ -33,7 +33,7 @@ API SDP offers for PyTorch.
       dist.init_process_group()
 
 
--  Pin each GPU to a single SDP process with ``local_rank`` - this
+-  Pin each GPU to a single distributed data parallel library process with ``local_rank`` - this
    refers to the relative rank of the process within a given node.
    ``smdistributed.dataparallel.torch.get_local_rank()`` API provides
    you the local rank of the device. The leader node will be rank 0, and
@@ -45,12 +45,12 @@ API SDP offers for PyTorch.
       torch.cuda.set_device(dist.get_local_rank())
 
 
--  Then wrap the PyTorch model with SDP’s DDP.
+-  Then wrap the PyTorch model with the distributed data parallel library’s DDP.
 
    .. code:: python
 
       model = ...
-      # Wrap model with SDP DistributedDataParallel
+      # Wrap model with SageMaker's DistributedDataParallel
       model = DDP(model)
 
 
@@ -82,17 +82,17 @@ API SDP offers for PyTorch.
 
 
 All put together, the following is an example PyTorch training script
-you will have for distributed training with SDP:
+you will have for distributed training with the distributed data parallel library:
 
 .. code:: python
 
-   # SDP: Import SDP PyTorch API
+   # Import distributed data parallel library PyTorch API
    import smdistributed.dataparallel.torch.distributed as dist
 
-   # SDP: Import SDP PyTorch DDP
+   # Import distributed data parallel library PyTorch DDP
    from smdistributed.dataparallel.torch.parallel.distributed import DistributedDataParallel as DDP
 
-   # SDP: Initialize SDP
+   # Initialize distributed data parallel library
    dist.init_process_group()
 
    class Net(nn.Module):
@@ -109,25 +109,25 @@ you will have for distributed training with SDP:
 
    def main():
 
-       # SDP: Scale batch size by world size
+       # Scale batch size by world size
        batch_size //= dist.get_world_size() // 8
        batch_size = max(batch_size, 1)
 
        # Prepare dataset
        train_dataset = torchvision.datasets.MNIST(...)
 
-       # SDP: Set num_replicas and rank in DistributedSampler
+       # Set num_replicas and rank in DistributedSampler
        train_sampler = torch.utils.data.distributed.DistributedSampler(
                train_dataset,
                num_replicas=dist.get_world_size(),
                rank=dist.get_rank())
 
        train_loader = torch.utils.data.DataLoader(..)
 
-       # SDP: Wrap the PyTorch model with SDP’s DDP
+       # Wrap the PyTorch model with distributed data parallel library’s DDP
        model = DDP(Net().to(device))
 
-       # SDP: Pin each GPU to a single SDP process.
+       # Pin each GPU to a single distributed data parallel library process.
        torch.cuda.set_device(local_rank)
        model.cuda(local_rank)
 
@@ -140,7 +140,7 @@ you will have for distributed training with SDP:
                test(...)
            scheduler.step()
 
-       # SDP: Save model on master node.
+       # Save model on master node.
        if dist.get_rank() == 0:
            torch.save(...)