Merge branch 'master' into MLX-1269

Author: sage-maker

.githooks/pre-push

@@ -12,5 +12,5 @@ start_time=`date +%s`
 tox -e sphinx,doc8 --parallel all
 ./ci-scripts/displaytime.sh 'sphinx,doc8' $start_time
 start_time=`date +%s`
-tox -e py38,py39,py310 --parallel all -- tests/unit
-./ci-scripts/displaytime.sh 'py38,py39,py310 unit' $start_time
+tox -e py39,py310,py311,py312 --parallel all -- tests/unit
+./ci-scripts/displaytime.sh 'py39,py310,py311,py312 unit' $start_time

.github/PULL_REQUEST_TEMPLATE.md

@@ -22,5 +22,6 @@ _Put an `x` in the boxes that apply. You can also fill these out after creating
 - [ ] I have added unit and/or integration tests as appropriate to ensure backward compatibility of the changes
 - [ ] I have checked that my tests are not configured for a specific region or account (if appropriate)
 - [ ] I have used [`unique_name_from_base`](https://github.com/aws/sagemaker-python-sdk/blob/master/src/sagemaker/utils.py#L77) to create resource names in integ tests (if appropriate)
+- [ ] If adding any dependency in requirements.txt files, I have spell checked and ensured they exist in PyPi
 
 By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

.github/workflows/codebuild-canaries.yml

@@ -0,0 +1,24 @@
+name: Canaries
+on:
+  schedule:
+    - cron: "0 */3 * * *"
+  workflow_dispatch:
+
+permissions:
+    id-token: write # This is required for requesting the JWT
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
+          aws-region: us-west-2
+          role-duration-seconds: 10800
+      - name: Run Integ Tests
+        uses: aws-actions/aws-codebuild-run-build@v1
+        id: codebuild
+        with:
+          project-name: sagemaker-python-sdk-canaries

.github/workflows/codebuild-ci-health.yml

@@ -26,7 +26,7 @@ jobs:
     strategy:
         fail-fast: false
         matrix:
-          python-version: ["py38", "py39", "py310", "py311"]
+          python-version: ["py39", "py310", "py311","py312"]
     steps:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v4

.github/workflows/codebuild-ci.yml

@@ -55,15 +55,15 @@ jobs:
       - name: Run Codestyle & Doc Tests
         uses: aws-actions/aws-codebuild-run-build@v1
         with:
-          project-name: sagemaker-python-sdk-ci-codestyle-doc-tests
+          project-name: ${{ github.event.repository.name }}-ci-codestyle-doc-tests
           source-version-override: 'refs/pull/${{ github.event.pull_request.number }}/head^{${{ github.event.pull_request.head.sha }}}'
   unit-tests:
     runs-on: ubuntu-latest
     needs: [wait-for-approval]
     strategy:
         fail-fast: false
         matrix:
-          python-version: ["py38","py39","py310","py311"]
+          python-version: ["py39","py310","py311","py312"]
     steps:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v4
@@ -74,7 +74,7 @@ jobs:
       - name: Run Unit Tests
         uses: aws-actions/aws-codebuild-run-build@v1
         with:
-          project-name: sagemaker-python-sdk-ci-unit-tests
+          project-name: ${{ github.event.repository.name }}-ci-unit-tests
           source-version-override: 'refs/pull/${{ github.event.pull_request.number }}/head^{${{ github.event.pull_request.head.sha }}}'
           env-vars-for-codebuild: |
             PY_VERSION
@@ -93,5 +93,5 @@ jobs:
       - name: Run Integ Tests
         uses: aws-actions/aws-codebuild-run-build@v1
         with:
-          project-name: sagemaker-python-sdk-ci-integ-tests
+          project-name: ${{ github.event.repository.name }}-ci-integ-tests
           source-version-override: 'refs/pull/${{ github.event.pull_request.number }}/head^{${{ github.event.pull_request.head.sha }}}'

.github/workflows/codeql.yml

@@ -0,0 +1,35 @@
+name: "CodeQL"
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+  schedule:
+    - cron: '30 15 * * *'
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    runs-on: ${{ 'ubuntu-latest' }}
+    permissions:
+      security-events: write
+      packages: read
+
+    strategy:
+      matrix:
+        include:
+          - language: python
+            build-mode: none
+          - language: java-kotlin
+            build-mode: none
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@6ccd57f4c5d15bdc2fef309bd9fb6cc9db2ef1c6
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@4b1d7da102ff94aca014c0245062b1a463356d72
+        with:
+          languages: ${{ matrix.language }}
+          build-mode: ${{ matrix.build-mode }}
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@4b1d7da102ff94aca014c0245062b1a463356d72
+        with:
+          category: "/language:${{matrix.language}}"

.github/workflows/security-monitoring.yml

@@ -0,0 +1,121 @@
+name: Security Monitoring
+
+on:
+  schedule:
+    - cron: '0 16 * * *'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  id-token: write
+
+jobs:
+  check-code-scanning-alerts:
+    runs-on: ubuntu-latest
+    outputs:
+      code_scanning_alert_status: ${{ steps.check-code-scanning-alerts.outputs.code_scanning_alert_status }}
+    steps:
+      - name: Check for security alerts
+        id: check-code-scanning-alerts
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea
+        with:
+          github-token: ${{ secrets.GH_PAT }}
+          script: |
+            async function checkAlerts() {
+              const owner = '${{ github.repository_owner }}';
+              const repo = '${{ github.event.repository.name }}';
+              const ref = 'refs/heads/master';
+            
+              const codeScanningAlerts = await github.rest.codeScanning.listAlertsForRepo({
+                owner,
+                repo,
+                ref: ref
+              });
+              const activeCodeScanningAlerts = codeScanningAlerts.data.filter(alert => alert.state === 'open');
+              core.setOutput('code_scanning_alert_status', activeCodeScanningAlerts.length > 0 ? '1': '0');
+            }
+            await checkAlerts();
+
+  check-dependabot-alerts:
+    runs-on: ubuntu-latest
+    outputs:
+      dependabot_alert_status: ${{ steps.check-dependabot-alerts.outputs.dependabot_alert_status }}
+    steps:
+      - name: Check for dependabot alerts
+        id: check-dependabot-alerts
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea
+        with:
+          github-token: ${{ secrets.GH_PAT }}
+          script: |
+            async function checkAlerts() {
+              const owner = '${{ github.repository_owner }}';
+              const repo = '${{ github.event.repository.name }}';
+            
+              const dependabotAlerts = await github.rest.dependabot.listAlertsForRepo({
+                owner,
+                repo,
+                headers: {
+                  'accept': 'applications/vnd.github+json'
+                }
+              }); 
+              const activeDependabotAlerts = dependabotAlerts.data.filter(alert => alert.state === 'open');
+              core.setOutput('dependabot_alert_status', activeDependabotAlerts.length > 0 ? '1': '0');
+            }
+            await checkAlerts();
+
+  check-secret-scanning-alerts:
+    runs-on: ubuntu-latest
+    outputs:
+      secret_scanning_alert_status: ${{ steps.check-secret-scanning-alerts.outputs.secret_scanning_alert_status }}
+    steps:
+      - name: Check for secret scanning alerts
+        id: check-secret-scanning-alerts
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea
+        with:
+          github-token: ${{ secrets.GH_PAT }}
+          script: |
+            async function checkAlerts() {
+              const owner = '${{ github.repository_owner }}';
+              const repo = '${{ github.event.repository.name }}';
+
+              const secretScanningAlerts = await github.rest.secretScanning.listAlertsForRepo({
+                owner,
+                repo,
+              }); 
+              const activeSecretScanningAlerts = secretScanningAlerts.data.filter(alert => alert.state === 'open');
+              core.setOutput('secret_scanning_alert_status', activeSecretScanningAlerts.length > 0 ? '1': '0');
+            }
+            await checkAlerts();
+
+  put-metric-data:
+    runs-on: ubuntu-latest
+    needs: [check-code-scanning-alerts, check-dependabot-alerts, check-secret-scanning-alerts]
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@12e3392609eaaceb7ae6191b3f54bbcb85b5002b
+        with:
+          role-to-assume: ${{ secrets.MONITORING_ROLE_ARN }}
+          aws-region: us-west-2
+      - name: Put Code Scanning Alert Metric Data
+        run: |
+          if [ "${{ needs.check-code-scanning-alerts.outputs.code_scanning_alert_status }}" == "1" ]; then
+            aws cloudwatch put-metric-data --metric-name CodeScanningAlert --namespace SecurityMonitoringMetrics --value 1 --unit Count --dimensions ProjectName=sagemaker-python-sdk
+          else
+            aws cloudwatch put-metric-data --metric-name CodeScanningAlert --namespace SecurityMonitoringMetrics --value 0 --unit Count --dimensions ProjectName=sagemaker-python-sdk
+          fi
+      - name: Put Dependabot Alert Metric Data
+        run: |
+          if [ "${{ needs.check-dependabot-alerts.outputs.dependabot_alert_status }}" == "1" ]; then
+            aws cloudwatch put-metric-data --metric-name DependabotAlert --namespace SecurityMonitoringMetrics --value 1 --unit Count --dimensions ProjectName=sagemaker-python-sdk
+          else
+            aws cloudwatch put-metric-data --metric-name DependabotAlert --namespace SecurityMonitoringMetrics --value 0 --unit Count --dimensions ProjectName=sagemaker-python-sdk
+          fi
+      - name: Put Secret Scanning Alert Metric Data
+        run: |
+          if [ "${{ needs.check-secret-scanning-alerts.outputs.secret_scanning_alert_status }}" == "1" ]; then
+            aws cloudwatch put-metric-data --metric-name SecretScanningAlert --namespace SecurityMonitoringMetrics --value 1 --unit Count --dimensions ProjectName=sagemaker-python-sdk
+          else
+            aws cloudwatch put-metric-data --metric-name SecretScanningAlert --namespace SecurityMonitoringMetrics --value 0 --unit Count --dimensions ProjectName=sagemaker-python-sdk
+          fi

.gitignore

@@ -32,6 +32,9 @@ env/
 .python-version
 *.html
 **/_repack_script_launcher.sh
+src/sagemaker/modules/train/container_drivers/sm_train.sh
+src/sagemaker/modules/train/container_drivers/sourcecode.json
+src/sagemaker/modules/train/container_drivers/distributed.json
 tests/data/**/_repack_model.py
 tests/data/experiment/sagemaker-dev-1.0.tar.gz
 src/sagemaker/serve/tmp_workspace

.pydocstylerc

@@ -2,3 +2,4 @@
 inherit = false
 ignore = D104,D107,D202,D203,D213,D214,D400,D401,D404,D406,D407,D411,D413,D414,D415,D417
 match = (?!record_pb2).*\.py
+match-dir = (?!.*test).*

.pylintrc

@@ -94,7 +94,24 @@ disable=
     useless-object-inheritance, # TODO: Enable this check and fix code once Python 2 is no longer supported.
     super-with-arguments,
     raise-missing-from,
-    E1136,
+    C0116,  # Missing function or method docstring
+    C0209,  # Use f-string instead of format
+    E0015,  # Unrecognized option found in config
+    E0702,  # Raising a string instead of an exception
+    E1101,  # Module has no member (likely dynamic attr)
+    E1136,  # Value assigned to something inferred as None
+    R0022,  # Useless option value in config
+    R1710,  # Inconsistent return statements
+    R1714,  # Consider using `in` with comparisons
+    R1729,  # Use a generator
+    R1732,
+    R1735,  # Consider using a dict or list literal
+    W0237,  # Argument renamed in override
+    W0613,  # Unused argument
+    W0621,  # Redefining name from outer scope
+    W0719
+    W1404,  # Implicit string concatenation
+    W1514,  # `open()` used without encoding
 
 [REPORTS]
 # Set the output format. Available formats are text, parseable, colorized, msvs
@@ -310,7 +327,7 @@ ignore-mixin-members=yes
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis. It
 # supports qualified module names, as well as Unix pattern matching.
-ignored-modules=distutils
+ignored-modules=
 
 # List of class names for which member attributes should not be checked (useful
 # for classes with dynamically set attributes). This supports the use of
@@ -384,7 +401,7 @@ max-returns=6
 max-branches=12
 
 # Maximum number of statements in function / method body
-max-statements=100
+max-statements=105
 
 # Maximum number of parents for a class (see R0901).
 max-parents=7
@@ -436,4 +453,4 @@ analyse-fallback-blocks=no
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "Exception"
-overgeneral-exceptions=Exception
+overgeneral-exceptions=builtins.Exception

.readthedocs.yaml

@@ -5,9 +5,9 @@
 version: 2
 
 build:
-  os: ubuntu-20.04
+  os: ubuntu-22.04
   tools:
-    python: "3.9"
+    python: "3.12"
 
 
 python: