[CI] Move CUDA AWS job to nightly (#14845)

There have been no issues with the self-hosted runners over the last 1+
week since we enabled them (July 19), and the only test failures on them
have been actual product issues. Removing this job saves resources and
developer time waiting for CI (self-hosted runners are faster).

---------

Signed-off-by: Sarnie, Nick <[email protected]>

Author: sarnex

.github/workflows/sycl-aws.yml

@@ -14,22 +14,18 @@ on:
         # See devops/actions/aws-ec2/action.yml for more details.
         description: "JSON string with array of objects with aws-type, runs-on, aws-ami, aws-spot, aws-disk, aws-timebomb, one-job properties"
         type: string
-        required: true
+        default: '[{"runs-on":"aws_cuda-${{ github.run_id }}-${{ github.run_attempt }}","aws-ami":"ami-01cb0573cb039ab24","aws-type":["g5.2xlarge","g5.4xlarge"],"aws-disk":"/dev/sda1:64","aws-spot":"false"}]'
 
 jobs:
   aws:
     runs-on: ubuntu-20.04
     environment: aws
     steps:
-      - name: Setup script
-        run: |
-          mkdir -p ./aws-ec2
-          wget raw.githubusercontent.com/intel/llvm/sycl/devops/actions/aws-ec2/action.yml   -P ./aws-ec2
-          wget raw.githubusercontent.com/intel/llvm/sycl/devops/actions/aws-ec2/aws-ec2.js   -P ./aws-ec2
-          wget raw.githubusercontent.com/intel/llvm/sycl/devops/actions/aws-ec2/package.json -P ./aws-ec2
-          npm install ./aws-ec2
-      - name: Start AWS EC2 runners
-        uses: ./aws-ec2
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: devops/actions/aws-ec2
+      - run: npm install ./devops/actions/aws-ec2
+      - uses: ./devops/actions/aws-ec2
         with:
           mode: ${{ inputs.mode }}
           runs-on-list: ${{ inputs.runs-on-list }}

.github/workflows/sycl-linux-precommit-aws.yml

@@ -1,5 +1,7 @@
 name: E2E on AWS CUDA
 run-name: E2E on AWS CUDA - ${{ github.event.workflow_run.display_title }}
+# Note: This workflow is currently disabled in the Github Actions UI because
+# we run CUDA testing on the self-hosted runners.
 # We have to keep pre-commit AWS CUDA testing in a separate workflow because we
 # need access to AWS secret and that isn't available on pull_request jobs for
 # PRs from forks. And github's "require approval for all outside collaborators"

.github/workflows/sycl-nightly.yml

@@ -37,13 +37,6 @@ jobs:
             target_devices: ext_oneapi_hip:gpu
             tests_selector: e2e
 
-          - name: NVIDIA/CUDA
-            runner: '["Linux", "cuda"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_build:latest
-            image_options: -u 1001 --gpus all --cap-add SYS_ADMIN
-            target_devices: ext_oneapi_cuda:gpu
-            tests_selector: e2e
-
           - name: Intel L0 GPU
             runner: '["Linux", "gen12"]'
             image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
@@ -135,6 +128,39 @@ jobs:
       sycl_toolchain_archive: ${{ needs.build-win.outputs.artifact_archive_name }}
       extra_lit_opts: --param gpu-intel-gen12=True
 
+  cuda-aws-start:
+    needs: [ubuntu2204_build]
+    if: ${{ always() && !cancelled() && needs.ubuntu2204_build.outputs.build_conclusion == 'success' }}
+    uses: ./.github/workflows/sycl-aws.yml
+    secrets: inherit
+    with:
+      mode: start
+
+  cuda-run-tests:
+    needs: [ubuntu2204_build, cuda-aws-start]
+    if: ${{ always() && !cancelled() && needs.ubuntu2204_build.outputs.build_conclusion == 'success' }}
+    uses: ./.github/workflows/sycl-linux-run-tests.yml
+    with:
+      name: CUDA E2E
+      runner: '["aws_cuda-${{ github.run_id }}-${{ github.run_attempt }}"]'
+      image: ghcr.io/intel/llvm/ubuntu2204_build:latest-0300ac924620a51f76c4929794637b82790f12ab
+      image_options: -u 1001 --gpus all --cap-add SYS_ADMIN --env NVIDIA_DISABLE_REQUIRE=1
+      target_devices: ext_oneapi_cuda:gpu
+      ref: ${{ github.sha }}
+      merge_ref: ''
+
+      sycl_toolchain_artifact: sycl_linux_default
+      sycl_toolchain_archive: ${{ needs.ubuntu2204_build.outputs.artifact_archive_name }}
+      sycl_toolchain_decompress_command: ${{ needs.ubuntu2204_build.outputs.artifact_decompress_command }}
+
+  cuda-aws-stop:
+    needs: [cuda-aws-start, cuda-run-tests]
+    if: always()
+    uses: ./.github/workflows/sycl-aws.yml
+    secrets: inherit
+    with:
+      mode: stop
+
   nightly_build_upload:
     name: Nightly Build Upload
     if: ${{ github.ref_name == 'sycl' }}