Reuse env var code in the tests, add test runner

cryos · cryos · commit 91c07c9f6cfa · 2025-05-20T11:25:12.000-04:00
diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml
@@ -154,57 +154,14 @@ jobs:
           dependent_exes: "jq wget"
 
       - name: Set environment variables
-        run: |
-          PYTHON_VERSION_FORMATTED=$(echo '${{ matrix.PY_VER }}' | tr -d '.')
-          if [[ "${{ inputs.host-platform }}" == linux* ]]; then
-            REPO_DIR=$(pwd)
-          elif [[ "${{ inputs.host-platform }}" == win* ]]; then
-            PWD=$(pwd)
-            REPO_DIR=$(cygpath -w $PWD)
-          fi
-
-          BUILD_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${{ inputs.build-ctk-ver }})"
-          TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${{ matrix.CUDA_VER }})"
-          if [[ $BUILD_CUDA_MAJOR != $TEST_CUDA_MAJOR ]]; then
-            SKIP_CUDA_BINDINGS_TEST=1
-            SKIP_CYTHON_TEST=1
-          else
-            SKIP_CUDA_BINDINGS_TEST=0
-            BUILD_CUDA_MINOR="$(cut -d '.' -f 2 <<< ${{ inputs.build-ctk-ver }})"
-            TEST_CUDA_MINOR="$(cut -d '.' -f 2 <<< ${{ matrix.CUDA_VER }})"
-            if [[ $BUILD_CUDA_MINOR != $TEST_CUDA_MINOR ]]; then
-              SKIP_CYTHON_TEST=1
-            else
-              SKIP_CYTHON_TEST=0
-            fi
-          fi
-
-          # We don't test compute-sanitizer on CTK<12 because backporting fixes is too much effort
-          # We only test compute-sanitizer on python 3.12 arbitrarily; we don't need to use sanitizer on the entire matrix
-          # Only local ctk installs have compute-sanitizer; there is not wheel for it
-          if [[ "${{ inputs.python-version }}" == "3.12" && "${{ inputs.cuda-version }}" != "11.8.0" && "${{ inputs.local-ctk }}" == 1 ]]; then
-            SETUP_SANITIZER=1
-            echo "LATEST_CUDA_VERSION=$(bash .github/workflows/guess_latest.sh)" >> $GITHUB_ENV
-          else
-            SETUP_SANITIZER=0
-          fi
-          echo "SETUP_SANITIZER=${SETUP_SANITIZER}" >> $GITHUB_ENV
-
-          # make outputs from the previous job as env vars
-          CUDA_CORE_ARTIFACT_BASENAME="cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.host-platform }}"
-          echo "PYTHON_VERSION_FORMATTED=${PYTHON_VERSION_FORMATTED}" >> $GITHUB_ENV
-          echo "CUDA_CORE_ARTIFACT_BASENAME=${CUDA_CORE_ARTIFACT_BASENAME}" >> $GITHUB_ENV
-          echo "CUDA_CORE_ARTIFACT_NAME=${CUDA_CORE_ARTIFACT_BASENAME}-${{ github.sha }}" >> $GITHUB_ENV
-          echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_core/dist")" >> $GITHUB_ENV
-          echo "CUDA_CORE_CYTHON_TESTS_DIR=$(realpath "$REPO_DIR/cuda_core/tests/cython")" >> $GITHUB_ENV
-          CUDA_BINDINGS_ARTIFACT_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ inputs.build-ctk-ver }}-${{ inputs.host-platform }}"
-          echo "CUDA_BINDINGS_ARTIFACT_BASENAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}" >> $GITHUB_ENV
-          echo "CUDA_BINDINGS_ARTIFACT_NAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}-${{ github.sha }}" >> $GITHUB_ENV
-          echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV
-          echo "CUDA_BINDINGS_CYTHON_TESTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/tests/cython")" >> $GITHUB_ENV
-
-          echo "SKIP_CUDA_BINDINGS_TEST=${SKIP_CUDA_BINDINGS_TEST}" >> $GITHUB_ENV
-          echo "SKIP_CYTHON_TEST=${SKIP_CYTHON_TEST}" >> $GITHUB_ENV
+        env:
+          BUILD_CUDA_VER: ${{ inputs.build-ctk-ver }}
+          CUDA_VER: ${{ matrix.CUDA_VER }}
+          HOST_PLATFORM: ${{ inputs.host-platform }}
+          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
+          PY_VER: ${{ matrix.PY_VER }}
+          SHA: ${{ github.sha }}
+        run: ./ci/tools/env-vars test
 
       - name: Download cuda-python build artifacts
         if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0'}}
@@ -343,50 +300,16 @@ jobs:
 
       - name: Run cuda.bindings tests
         if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0' }}
-        run: |
-          pushd "${CUDA_BINDINGS_ARTIFACTS_DIR}"
-          if [[ "${{ matrix.LOCAL_CTK }}" == 1 ]]; then
-            ls $CUDA_PATH
-            pip install $(ls *.whl)[test]
-          else
-            pip install $(ls *.whl)[all,test]
-          fi
-          popd
-
-          pushd ./cuda_bindings
-          ${SANITIZER_CMD} pytest -rxXs -v tests/
-          if [[ "${SKIP_CYTHON_TEST}" == 0 ]]; then
-            ${SANITIZER_CMD} pytest -rxXs -v tests/cython
-          fi
-          popd
+        env:
+          CUDA_VER: ${{ matrix.CUDA_VER }}
+          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
+        run: run-tests bindings
 
       - name: Run cuda.core tests
-        run: |
-          # If build/test majors match: cuda.bindings is installed in the previous step.
-          # If mismatch: cuda.bindings is installed from the backport branch.
-          if [[ "${SKIP_CUDA_BINDINGS_TEST}" == 1 ]]; then
-            pushd "${CUDA_BINDINGS_ARTIFACTS_DIR}"
-            if [[ "${{ matrix.LOCAL_CTK }}" == 1 ]]; then
-              pip install *.whl
-            else
-              pip install $(ls *.whl)[all]
-            fi
-            popd
-          fi
-          TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${{ matrix.CUDA_VER }})"
-          pushd "${CUDA_CORE_ARTIFACTS_DIR}"
-          pip install $(ls *.whl)["cu${TEST_CUDA_MAJOR}","test-cu${TEST_CUDA_MAJOR}"]
-          popd
-
-          pushd ./cuda_core
-          ${SANITIZER_CMD} pytest -rxXs -v tests/
-
-          # Currently our CI always installs the latest bindings (from either major version).
-          # This is not compatible with the test requirements.
-          if [[ "${SKIP_CYTHON_TEST}" == 0 ]]; then
-            ${SANITIZER_CMD} pytest -rxXs -v tests/cython
-          fi
-          popd
+        env:
+          CUDA_VER: ${{ matrix.CUDA_VER }}
+          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
+        run: run-tests core
 
       - name: Ensure cuda-python installable
         run: |
diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml
@@ -96,41 +96,30 @@ jobs:
       - name: Ensure GPU is working
         run: nvidia-smi
 
-      - name: Set environment variables
+      - name: Install Git for Windows
+        # the GPU runner image does not have Git Bash pre-installed...
+        env:
+          # doesn't seem there's an easy way to avoid hard-coding it?
+          GFW_EXE_URL: https://github.com/git-for-windows/git/releases/download/v2.49.0.windows.1/PortableGit-2.49.0-64-bit.7z.exe
         run: |
-          $PYTHON_VERSION_FORMATTED = '${{ matrix.PY_VER }}' -replace '\.'
-          $REPO_DIR = $PWD.Path
-
-          $BUILD_CUDA_MAJOR = '${{ inputs.build-ctk-ver }}' -split '\.' | Select-Object -First 1
-          $TEST_CUDA_MAJOR = '${{ matrix.CUDA_VER }}' -split '\.' | Select-Object -First 1
-          if ($BUILD_CUDA_MAJOR -ne $TEST_CUDA_MAJOR) {
-            $SKIP_CUDA_BINDINGS_TEST = 1
-            $SKIP_CYTHON_TEST = 1
-          } else {
-            $SKIP_CUDA_BINDINGS_TEST = 0
-            $BUILD_CUDA_MINOR = '${{ inputs.build-ctk-ver }}' -split '\.' | Select-Object -Skip 1 -First 1
-            $TEST_CUDA_MINOR = '${{ matrix.CUDA_VER }}' -split '\.' | Select-Object -Skip 1 -First 1
-            if ($BUILD_CUDA_MINOR -ne $TEST_CUDA_MINOR) {
-              $SKIP_CYTHON_TEST = 1
-            } else {
-              $SKIP_CYTHON_TEST = 0
-            }
-          }
+          Invoke-WebRequest -Uri "$env:GFW_EXE_URL" -OutFile "PortableGit.7z.exe"
+          # Self-extracting, see https://gitforwindows.org/zip-archives-extracting-the-released-archives.html
+          Start-Process .\PortableGit.7z.exe -Wait -Verbose -ArgumentList '-y -gm2'
+          ls -l PortableGit
+          echo "$((Get-Location).Path)\\PortableGit\\bin" >> $env:GITHUB_PATH
+          $env:Path += ";$((Get-Location).Path)\\PortableGit\\bin"
+          bash --version
 
-          # Make outputs from the previous job as env vars
-          $CUDA_CORE_ARTIFACT_BASENAME = "cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.host-platform }}"
-          "PYTHON_VERSION_FORMATTED=${PYTHON_VERSION_FORMATTED}" >> $env:GITHUB_ENV
-          "CUDA_CORE_ARTIFACT_BASENAME=${CUDA_CORE_ARTIFACT_BASENAME}" >> $env:GITHUB_ENV
-          "CUDA_CORE_ARTIFACT_NAME=${CUDA_CORE_ARTIFACT_BASENAME}-${{ github.sha }}" >> $env:GITHUB_ENV
-          "CUDA_CORE_ARTIFACTS_DIR=$($ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath("$REPO_DIR\cuda_core\dist"))" >> $env:GITHUB_ENV
-          "CUDA_CORE_CYTHON_TESTS_DIR=$($ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath("$REPO_DIR\cuda_core\tests\cython"))" >> $env:GITHUB_ENV
-          $CUDA_BINDINGS_ARTIFACT_BASENAME = "cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ inputs.build-ctk-ver }}-${{ inputs.host-platform }}"
-          "CUDA_BINDINGS_ARTIFACT_BASENAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}" >> $env:GITHUB_ENV
-          "CUDA_BINDINGS_ARTIFACT_NAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}-${{ github.sha }}" >> $env:GITHUB_ENV
-          "CUDA_BINDINGS_ARTIFACTS_DIR=$($ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath("$REPO_DIR\cuda_bindings\dist"))" >> $env:GITHUB_ENV
-          "CUDA_BINDINGS_CYTHON_TESTS_DIR=$($ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath("$REPO_DIR\cuda_bindings\tests\cython"))" >> $env:GITHUB_ENV
-          "SKIP_CUDA_BINDINGS_TEST=${SKIP_CUDA_BINDINGS_TEST}" >> $env:GITHUB_ENV
-          "SKIP_CYTHON_TEST=${SKIP_CYTHON_TEST}" >> $env:GITHUB_ENV
+      - name: Set environment variables
+        env:
+          BUILD_CUDA_VER: ${{ inputs.build-ctk-ver }}
+          CUDA_VER: ${{ matrix.CUDA_VER }}
+          HOST_PLATFORM: ${{ inputs.host-platform }}
+          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
+          PY_VER: ${{ matrix.PY_VER }}
+          SHA: ${{ github.sha }}
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: ./ci/tools/env-vars test
 
       - name: Download cuda-python build artifacts
         if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0'}}
@@ -161,21 +150,6 @@ jobs:
           }
           gh --version
 
-      - name: Install Git for Windows
-        # the GPU runner image does not have Git Bash pre-installed...
-        if: ${{ matrix.LOCAL_CTK == '1' }}
-        env:
-          # doesn't seem there's an easy way to avoid hard-coding it?
-          GFW_EXE_URL: https://github.com/git-for-windows/git/releases/download/v2.49.0.windows.1/PortableGit-2.49.0-64-bit.7z.exe
-        run: |
-          Invoke-WebRequest -Uri "$env:GFW_EXE_URL" -OutFile "PortableGit.7z.exe"
-          # Self-extracting, see https://gitforwindows.org/zip-archives-extracting-the-released-archives.html
-          Start-Process .\PortableGit.7z.exe -Wait -Verbose -ArgumentList '-y -gm2'
-          ls -l PortableGit
-          echo "$((Get-Location).Path)\\PortableGit\\bin" >> $env:GITHUB_PATH
-          $env:Path += ";$((Get-Location).Path)\\PortableGit\\bin"
-          bash --version
-
       - name: Install zstd
         # the GPU runner image does not have zstd pre-installed... and it's needed by actions/cache
         if: ${{ matrix.LOCAL_CTK == '1' }}
@@ -278,48 +252,18 @@ jobs:
 
       - name: Run cuda.bindings tests
         if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0' }}
-        run: |
-          Push-Location $env:CUDA_BINDINGS_ARTIFACTS_DIR
-          if ('${{ matrix.LOCAL_CTK }}' -eq '1') {
-            Get-ChildItem $env:CUDA_PATH
-            echo $PATH
-            pip install "$((Get-ChildItem -Filter *.whl).FullName)[test]"
-          } else {
-            pip install "$((Get-ChildItem -Filter *.whl).FullName)[all,test]"
-          }
-          Pop-Location
-
-          Push-Location ./cuda_bindings
-          pytest -rxXs -v tests/
-          if ($env:SKIP_CYTHON_TEST -eq '0') {
-            pytest -rxXs -v tests/cython
-          }
-          Pop-Location
+        env:
+          CUDA_VER: ${{ matrix.CUDA_VER }}
+          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: run-tests bindings
 
       - name: Run cuda.core tests
-        run: |
-          # If build/test majors match: cuda.bindings is installed in the previous step.
-          # If mismatch: cuda.bindings is installed from the backport branch.
-          if ($env:SKIP_CUDA_BINDINGS_TEST -eq '1') {
-            Push-Location $env:CUDA_BINDINGS_ARTIFACTS_DIR
-            if ('${{ matrix.LOCAL_CTK }}' -eq '1') {
-              pip install (Get-ChildItem -Filter *.whl).FullName
-            } else {
-              pip install "$((Get-ChildItem -Filter *.whl).FullName)[all]"
-            }
-            Pop-Location
-          }
-          $TEST_CUDA_MAJOR = '${{ matrix.CUDA_VER }}' -split '\.' | Select-Object -First 1
-          Push-Location $env:CUDA_CORE_ARTIFACTS_DIR
-          pip install "$((Get-ChildItem -Filter *.whl).FullName)[cu${TEST_CUDA_MAJOR},test-cu${TEST_CUDA_MAJOR}]"
-          Pop-Location
-
-          Push-Location ./cuda_core
-          pytest -rxXs -v tests/
-          if ($env:SKIP_CYTHON_TEST -eq '0') {
-            pytest -rxXs -v tests/cython
-          }
-          Pop-Location
+        env:
+          CUDA_VER: ${{ matrix.CUDA_VER }}
+          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: run-tests core
 
       - name: Ensure cuda-python installable
         run: |
diff --git a/ci/tools/env-vars b/ci/tools/env-vars
@@ -39,7 +39,36 @@ echo "CUDA_BINDINGS_ARTIFACT_BASENAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}" >> $GI
 echo "CUDA_BINDINGS_ARTIFACT_NAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}-${SHA}" >> $GITHUB_ENV
 echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "${REPO_DIR}/cuda_bindings/dist")" >> $GITHUB_ENV
 echo "CUDA_BINDINGS_CYTHON_TESTS_DIR=$(realpath "${REPO_DIR}/cuda_bindings/tests/cython")" >> $GITHUB_ENV
+echo "PYTHON_VERSION_FORMATTED=${PYTHON_VERSION_FORMATTED}" >> $GITHUB_ENV
 
 if [[ "${1}" == "build" ]]; then
   echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV
+elif [[ "${1}" == "test" ]]; then
+  BUILD_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${BUILD_CUDA_VER})"
+  TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${CUDA_VER})"
+  if [[ ${BUILD_CUDA_MAJOR} != ${TEST_CUDA_MAJOR} ]]; then
+    SKIP_CUDA_BINDINGS_TEST=1
+    SKIP_CYTHON_TEST=1
+  else
+    SKIP_CUDA_BINDINGS_TEST=0
+    BUILD_CUDA_MINOR="$(cut -d '.' -f 2 <<< ${BUILD_CUDA_VER})"
+    TEST_CUDA_MINOR="$(cut -d '.' -f 2 <<< ${CUDA_VER})"
+    if [[ ${BUILD_CUDA_MINOR} != ${TEST_CUDA_MINOR} ]]; then
+      SKIP_CYTHON_TEST=1
+    else
+      SKIP_CYTHON_TEST=0
+    fi
+  fi
+  # We don't test compute-sanitizer on CTK<12 because backporting fixes is too much effort
+  # We only test compute-sanitizer on python 3.12 arbitrarily; we don't need to use sanitizer on the entire matrix
+  # Only local ctk installs have compute-sanitizer; there is no wheel for it
+  if [[ "${PY_VER}" == "3.12" && "${CUDA_VER}" != "11.8.0" && "${LOCAL_CTK}" == 1 && "${HOST_PLATFORM}" == linux* ]]; then
+    SETUP_SANITIZER=1
+    echo "LATEST_CUDA_VERSION=$(bash .github/workflows/guess_latest.sh)" >> $GITHUB_ENV
+  else
+    SETUP_SANITIZER=0
+  fi
+  echo "SETUP_SANITIZER=${SETUP_SANITIZER}" >> $GITHUB_ENV
+  echo "SKIP_CUDA_BINDINGS_TEST=${SKIP_CUDA_BINDINGS_TEST}" >> $GITHUB_ENV
+  echo "SKIP_CYTHON_TEST=${SKIP_CYTHON_TEST}" >> $GITHUB_ENV
 fi
diff --git a/ci/tools/run-tests b/ci/tools/run-tests
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# A utility script to install the correct packages and run the tests.
+
+set -euo pipefail
+
+# Check if the script was called with exactly 1 argument
+if [[ ${#} -ne 1 && ( "${1}" == "bindings" || "${1}" == "core" ) ]]; then
+  echo "Error: This script requires exactly 1 argument (what is tested). You provided ${#}"
+  echo "Usage: ${0} test_module[bindings or core]"
+  exit 1
+fi
+
+test_module=${1}
+
+if [[ "${test_module}" == "bindings" ]]; then
+  pushd "${CUDA_BINDINGS_ARTIFACTS_DIR}"
+  echo "Installing bindings wheel"
+  pwd
+  ls
+  if [[ "${LOCAL_CTK}" == 1 ]]; then
+    ls "${CUDA_PATH}"
+    pip install $(ls *.whl)[test]
+  else
+    pip install $(ls *.whl)[all,test]
+  fi
+  popd
+  pushd ./cuda_bindings
+  echo "Running bindinds tests"
+  pwd
+  ${SANITIZER_CMD} pytest -rxXs -v tests/
+  if [[ "${SKIP_CYTHON_TEST}" == 0 ]]; then
+    ${SANITIZER_CMD} pytest -rxXs -v tests/cython
+  fi
+  popd
+elif [[ "${test_module}" == "core" ]]; then
+  # If build/test majors match: cuda.bindings is installed in the previous step.
+  # If mismatch: cuda.bindings is installed from the backport branch.
+  if [[ "${SKIP_CUDA_BINDINGS_TEST}" == 1 ]]; then
+    pushd "${CUDA_BINDINGS_ARTIFACTS_DIR}"
+    echo "Installing bindings wheel"
+    pwd
+    ls
+    if [[ "${LOCAL_CTK}" == 1 ]]; then
+      pip install *.whl
+    else
+      pip install $(ls *.whl)[all]
+    fi
+    popd
+  fi
+  TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${CUDA_VER})"
+  pushd "${CUDA_CORE_ARTIFACTS_DIR}"
+  echo "Installing core wheel"
+  pwd
+  ls
+  pip install $(ls *.whl)["cu${TEST_CUDA_MAJOR}","test-cu${TEST_CUDA_MAJOR}"]
+  popd
+  pushd ./cuda_core
+  echo "Running core tests"
+  pwd
+  ${SANITIZER_CMD} pytest -rxXs -v tests/
+  # Currently our CI always installs the latest bindings (from either major version).
+  # This is not compatible with the test requirements.
+  if [[ "${SKIP_CYTHON_TEST}" == 0 ]]; then
+    ${SANITIZER_CMD} pytest -rxXs -v tests/cython
+  fi
+  popd
+fi
