Nightly RC Verification Build for Python 3.13 (#37400)

scbedd · web-flow · commit 6697c475710e · 2024-09-20T16:53:26.000-07:00
* refactor the steps a bit. can we still build and test?
diff --git a/eng/pipelines/templates/jobs/tests-nightly-python.yml b/eng/pipelines/templates/jobs/tests-nightly-python.yml
@@ -1,150 +1,84 @@
 trigger:
   - main
 
-jobs:
-
-  - job: Validate_RC_Python_Build_Windows
-    displayName: Validate Release Candidate Python - Windows
-    variables:
-      skipComponentGovernanceDetection: true
-      PythonVersion: '3.11.0-rc.1'
-
-    timeoutInMinutes: 90
-
-    pool:
-      name: 'azsdk-pool-mms-win-2022-general'
+parameters:
+  - name: RunNightlyTests
+    type: boolean
+    default: false
+  - name: TargetedPackages
+    type: string
+    default: "azure-core,azure-keyvault*, azure-identity*, azure-storage-queue"
 
-    steps:
-      - template: /eng/pipelines/templates/steps/use-python-version.yml
-        parameters:
-          versionSpec: $(PythonVersion)
-
-      - script: |
-          python -m pip freeze
-          python -m pip --version
-          python -m pip install setuptools==72.2.0 wheel==0.37.0 tox==4.5.0 packaging==23.1 requests
-          python -m pip install $(Build.SourcesDirectory)/tools/azure-sdk-tools[build]
-        displayName: Install Dependencies
-
-      - template: /eng/common/testproxy/test-proxy-tool.yml
-        parameters:
-          runProxy: false
-
-      - script: |
-          python ./scripts/devops_tasks/dispatch_tox.py "$(BuildTargetingString)" --junitxml="junit/test_results.xml" --toxenv="whl" --filter-type="None"
-        displayName: 'Setup - Run Filtered Tests For Python $(PythonVersion)'
-        env:
-          YARL_NO_EXTENSIONS: 1
-          AIOHTTP_NO_EXTENSIONS: 1
-          PROXY_URL: "http://localhost:5000"
-        continueOnError: true
-
-      - task: PublishTestResults@2
-        condition: always()
-        inputs:
-          testResultsFiles: '**/*test*.xml'
-          testRunTitle: 'Python $(PythonVersion)'
-          failTaskOnFailedTests: true
+jobs:
 
   - job: Validate_RC_Python_Build_Linux
     displayName: Validate Release Candidate Python - Linux
     variables:
       skipComponentGovernanceDetection: true
-      PythonVersion: '3.11.0-rc.1'
-
-    timeoutInMinutes: 90
-
-    pool:
-      name: 'azsdk-pool-mms-ubuntu-2004-general'
-
-    steps:
-      - template: /eng/pipelines/templates/steps/use-python-version.yml
-        parameters:
-          versionSpec: $(PythonVersion)
-
-      - script: |
-          sudo apt-get update
-          sudo apt-get install build-essential -y
-          python -m pip freeze
-          python -m pip --version
-          python -m pip install setuptools==72.2.0 wheel==0.37.0 tox==4.5.0 packaging==23.1 requests
-          python -m pip install $(Build.SourcesDirectory)/tools/azure-sdk-tools[build]
-        displayName: Install Dependencies
-
-      - template: /eng/common/testproxy/test-proxy-tool.yml
-        parameters:
-          runProxy: false
-
-      - script: |
-          find /usr/lib -name "libffi.so*"
-          ln -s /usr/lib/x86_64-linux-gnu/libffi.so.7 /usr/lib/x86_64-linux-gnu/libffi.so.6
-          python ./scripts/devops_tasks/dispatch_tox.py "$(BuildTargetingString)" --junitxml="junit/test_results.xml" --toxenv="whl" --filter-type="None"
-        displayName: 'Setup - Run Filtered Tests For Python $(PythonVersion)'
-        env:
-          YARL_NO_EXTENSIONS: 1
-          AIOHTTP_NO_EXTENSIONS: 1
-          PROXY_URL: "http://localhost:5000"
-        continueOnError: true
-
-      - task: PublishTestResults@2
-        condition: always()
-        inputs:
-          testResultsFiles: '**/*test*.xml'
-          testRunTitle: 'Python $(PythonVersion)'
-          failTaskOnFailedTests: true
-
-  - job: Validate_Nightly_Python_Build
-    displayName: Validate Nightly Dev Python Build
-    variables:
-      skipComponentGovernanceDetection: true
+      PythonVersion: '3.13.0-rc.2'
+      TargetedPackages: ${{ parameters.TargetedPackages }}
 
     timeoutInMinutes: 90
 
     pool:
       name: 'azsdk-pool-mms-ubuntu-2004-general'
 
     steps:
-      - task: UsePythonVersion@0
-        displayName: 'Use Python 3.9 For Build Tools'
-        inputs:
-          versionSpec: '3.9'
-
-
-      - template: /eng/common/testproxy/test-proxy-tool.yml
-        parameters:
-          runProxy: false
-
-      - script: |
-          sudo apt-get update
-          sudo apt-get install build-essential libsqlite3-dev sqlite3 bzip2 libbz2-dev zlib1g-dev libssl-dev openssl libgdbm-dev liblzma-dev libreadline-dev libncursesw5-dev libffi-dev uuid-dev
-          cd ~/
-          git clone https://github.com/python/cpython.git
-          cd cpython
-          mkdir debug
-          cd debug
-          ../configure --enable-optimizations --prefix=$HOME
-          make
-          make install
-          export PATH=~/bin:$PATH
-          export PATH=~/lib:$PATH
-          export PATH=~/.local/bin:$PATH
-          curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-          python3 get-pip.py
-          python3 -m pip install setuptools==72.2.0 wheel
-          python3 -m pip install tox packaging twine beautifulsoup4
-          python3 --version
-          cd $(Build.SourcesDirectory)
-          python3 ./scripts/devops_tasks/dispatch_tox.py "$(BuildTargetingString)" --junitxml="junit/test_results_38.xml" --toxenv="whl" --filter-type="None"
-        displayName: 'Setup - Run Filtered Tests "Nightly" using Python Edge'
-        env:
-          YARL_NO_EXTENSIONS: 1
-          PROXY_URL: "http://localhost:5000"
-          AIOHTTP_NO_EXTENSIONS: 1
-        continueOnError: true
-
-      - task: PublishTestResults@2
-        condition: always()
-        inputs:
-          testResultsFiles: '**/junit/test-results.xml'
-          testRunTitle: 'Python Nightly'
-          failTaskOnFailedTests: true
+      - template: ../steps/release-candidate-steps.yml
+
+  - ${{ if eq(parameters.RunNightlyTests, true) }}:
+    - job: Validate_Nightly_Python_Build
+      displayName: Validate Nightly Dev Python Build
+      variables:
+        skipComponentGovernanceDetection: true
+
+      timeoutInMinutes: 90
+
+      pool:
+        name: 'azsdk-pool-mms-ubuntu-2004-general'
+
+      steps:
+        - task: UsePythonVersion@0
+          displayName: 'Use Python 3.9 For Build Tools'
+          inputs:
+            versionSpec: '3.9'
+
+
+        - template: /eng/common/testproxy/test-proxy-tool.yml
+          parameters:
+            runProxy: false
+
+        - script: |
+            sudo apt-get update
+            sudo apt-get install build-essential libsqlite3-dev sqlite3 bzip2 libbz2-dev zlib1g-dev libssl-dev openssl libgdbm-dev liblzma-dev libreadline-dev libncursesw5-dev libffi-dev uuid-dev
+            cd ~/
+            git clone https://github.com/python/cpython.git
+            cd cpython
+            mkdir debug
+            cd debug
+            ../configure --enable-optimizations --prefix=$HOME
+            make
+            make install
+            export PATH=~/bin:$PATH
+            export PATH=~/lib:$PATH
+            export PATH=~/.local/bin:$PATH
+            curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+            python3 get-pip.py
+            python3 -m pip install setuptools==72.2.0 wheel
+            python3 -m pip install tox packaging twine beautifulsoup4
+            python3 --version
+            cd $(Build.SourcesDirectory)
+            python3 ./scripts/devops_tasks/dispatch_tox.py "$(BuildTargetingString)" --junitxml="junit/test_results_38.xml" --toxenv="whl" --filter-type="None"
+          displayName: 'Setup - Run Filtered Tests "Nightly" using Python Edge'
+          env:
+            YARL_NO_EXTENSIONS: 1
+            PROXY_URL: "http://localhost:5000"
+            AIOHTTP_NO_EXTENSIONS: 1
+          continueOnError: true
+
+        - task: PublishTestResults@2
+          condition: always()
+          inputs:
+            testResultsFiles: '**/junit/test-results.xml'
+            testRunTitle: 'Python Nightly'
+            failTaskOnFailedTests: true
diff --git a/eng/pipelines/templates/steps/release-candidate-steps.yml b/eng/pipelines/templates/steps/release-candidate-steps.yml
@@ -0,0 +1,38 @@
+steps:
+  - template: /eng/pipelines/templates/steps/use-python-version.yml
+    parameters:
+      versionSpec: $(PythonVersion)
+
+  - template: /eng/pipelines/templates/steps/use-venv.yml
+    parameters:
+      Activate: false
+
+  - pwsh: |
+      $(VENV_ACTIVATION_SCRIPT)
+      $ErrorActionPreference = 'Stop'
+      $PSNativeCommandUseErrorActionPreference = $true
+      python -m pip install -r $(Build.SourcesDirectory)/eng/ci_tools.txt
+      python -m pip freeze --all
+      Write-Host (Get-Command python).Source
+    displayName: 'Install Dependencies'
+
+  - template: /eng/common/testproxy/test-proxy-tool.yml
+    parameters:
+      runProxy: false
+
+  - pwsh: |
+      $(VENV_ACTIVATION_SCRIPT)
+      python ./scripts/devops_tasks/dispatch_tox.py "$(TargetedPackages)" --junitxml="junit/test_results.xml" --toxenv="whl" --filter-type="Build"
+    displayName: 'Setup - Run Filtered Tests For Python $(PythonVersion)'
+    env:
+      YARL_NO_EXTENSIONS: 1
+      AIOHTTP_NO_EXTENSIONS: 1
+      PROXY_URL: "http://localhost:5000"
+    continueOnError: true
+
+  - task: PublishTestResults@2
+    condition: always()
+    inputs:
+      testResultsFiles: '**/*test*.xml'
+      testRunTitle: 'Python $(PythonVersion)'
+      failTaskOnFailedTests: true
diff --git a/scripts/devops_tasks/tox_harness.py b/scripts/devops_tasks/tox_harness.py
@@ -82,7 +82,7 @@ def compare_req_to_injected_reqs(parsed_req, injected_packages):
 
 def inject_custom_reqs(file, injected_packages, package_dir):
     req_lines = []
-    injected_packages = [p for p in re.split("[\s,]", injected_packages) if p]
+    injected_packages = [p for p in re.split(r"[\s,]", injected_packages) if p]
 
     if injected_packages:
         logging.info("Adding custom packages to requirements for {}".format(package_dir))
