[CI][Benchmark] Add comparison script to benchmark scripts + benchmark dashboard via sycl-docs.yml (#17522)

This PR:
- Introduces new CI to replace existing nightly benchmarking workflow
- Adds a preliminary implementation of a comparison script alongside the
existing benchmarking scripts: This comparison script is used by the CI
to decide whether or not a benchmark result is over a (predefined)
tolerance above historical median of benchmark results
  
This is not a permanent solution: I am looking into exponential moving
averages in the future
- Adds code in sycl-docs.yml for bringing up new benchmark results
dashboard on https://intel.github.io

---------

Co-authored-by: Piotr Balcer <[email protected]>
Co-authored-by: Łukasz Stolarczuk <[email protected]>

Author: 3 people

.github/workflows/sycl-docs.yml

@@ -49,7 +49,13 @@ jobs:
         mkdir clang
         mv $GITHUB_WORKSPACE/build/tools/sycl/doc/html/* .
         mv $GITHUB_WORKSPACE/build/tools/clang/docs/html/* clang/
+        cp -r $GITHUB_WORKSPACE/repo/devops/scripts/benchmarks/html benchmarks
         touch .nojekyll
+        # Update benchmarking dashboard configuration
+        cat << EOF > benchmarks/config.js
+        remoteDataUrl = 'https://raw.githubusercontent.com/intel/llvm-ci-perf-results/refs/heads/unify-ci/data.json';
+        defaultCompareNames = ["Baseline_PVC_L0"];
+        EOF
     # Upload the generated docs as an artifact and deploy to GitHub Pages.
     - name: Upload artifact
       uses: actions/upload-pages-artifact@v3

.github/workflows/sycl-linux-run-tests.yml

@@ -25,7 +25,7 @@ on:
         required: False
       tests_selector:
         description: |
-          Three possible options: "e2e", "cts", and "compute-benchmarks".
+          Three possible options: "e2e", "cts", and "benchmarks".
         type: string
         default: "e2e"
 
@@ -111,6 +111,33 @@ on:
         default: ''
         required: False
 
+      benchmark_upload_results:
+        description: |
+          Set to true to upload results to git repository storing benchmarking
+          results.
+        type: string
+        default: 'false'
+        required: False
+      benchmark_save_name:
+        description: |
+          Save name to use for benchmark results: Save names are stored in
+          metadata of result file, and are used to identify benchmark results in
+          the same series (e.g. same configuration, same device, etc.).
+
+          Note: Currently, benchmark result filenames are in the format of
+          <benchmark_save_name>_<Device>_<Backend>_YYYYMMDD_HHMMSS.json
+        type: string
+        default: ''
+        required: False
+      benchmark_preset:
+        description: |
+          Name of benchmark preset to run.
+
+          See /devops/scripts/benchmarks/presets.py for all presets available.
+        type: string
+        default: 'Minimal'
+        required: False
+
   workflow_dispatch:
     inputs:
       runner:
@@ -150,7 +177,7 @@ on:
         options:
           - e2e
           - cts
-          - compute-benchmarks
+          - benchmarks
 
       env:
         description: |
@@ -303,11 +330,14 @@ jobs:
         target_devices: ${{ inputs.target_devices }}
         retention-days: ${{ inputs.retention-days }}
 
-    - name: Run compute-benchmarks on SYCL
-      if: inputs.tests_selector == 'compute-benchmarks'
+    - name: Run benchmarks
+      if: inputs.tests_selector == 'benchmarks'
       uses: ./devops/actions/run-tests/benchmark
       with:
         target_devices: ${{ inputs.target_devices }}
+        upload_results: ${{ inputs.benchmark_upload_results }}
+        save_name: ${{ inputs.benchmark_save_name }}
+        preset: ${{ inputs.benchmark_preset }}
       env:
         RUNNER_TAG: ${{ inputs.runner }}
         GITHUB_TOKEN: ${{ secrets.LLVM_SYCL_BENCHMARK_TOKEN }}

.github/workflows/sycl-nightly.yml

@@ -277,35 +277,30 @@ jobs:
       sycl_toolchain_archive: ${{ needs.build-win.outputs.artifact_archive_name }}
       sycl_cts_artifact: sycl_cts_bin_win
 
-  aggregate_benchmark_results:
-    if: github.repository == 'intel/llvm' && !cancelled()
-    name: Aggregate benchmark results and produce historical averages
-    uses: ./.github/workflows/sycl-benchmark-aggregate.yml
-    secrets:
-      LLVM_SYCL_BENCHMARK_TOKEN: ${{ secrets.LLVM_SYCL_BENCHMARK_TOKEN }}
-    with:
-      lookback_days: 100
-      
   run-sycl-benchmarks:
-    needs: [ubuntu2204_build, aggregate_benchmark_results]
+    needs: [ubuntu2204_build]
     if: ${{ always() && !cancelled() && needs.ubuntu2204_build.outputs.build_conclusion == 'success' }}
     strategy:
-      fail-fast: false
       matrix:
         include:
-          - name: Run compute-benchmarks on L0 PVC
+          - ref: ${{ github.sha }}
+            save_name: Baseline
             runner: '["PVC_PERF"]'
-            image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
-            target_devices: level_zero:gpu
+            backend: 'level_zero:gpu'
+            preset: Minimal
     uses: ./.github/workflows/sycl-linux-run-tests.yml
     secrets: inherit
     with:
-      name: ${{ matrix.name }}
+      name: Run compute-benchmarks (${{ matrix.runner }}, ${{ matrix.backend }})
       runner: ${{ matrix.runner }}
-      image_options: ${{ matrix.image_options }}
-      target_devices: ${{ matrix.target_devices }}
-      tests_selector: compute-benchmarks
-      repo_ref: ${{ github.sha }}
+      image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest
+      image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
+      target_devices: ${{ matrix.backend }}
+      tests_selector: benchmarks
+      benchmark_upload_results: true
+      benchmark_save_name: ${{ matrix.save_name }}
+      benchmark_preset: ${{ matrix.preset }}
+      repo_ref: ${{ matrix.ref }}
       sycl_toolchain_artifact: sycl_linux_default
       sycl_toolchain_archive: ${{ needs.ubuntu2204_build.outputs.artifact_archive_name }}
       sycl_toolchain_decompress_command: ${{ needs.ubuntu2204_build.outputs.artifact_decompress_command }}

.github/workflows/sycl-ur-perf-benchmarking.yml

@@ -1,12 +1,138 @@
-name: Benchmarks
+name: Run Benchmarks
 
-# This workflow is a WIP: this workflow file acts as a placeholder.
+on:
+  workflow_call:
+    inputs:
+      preset:
+        type: string
+        description: |
+          Benchmark presets to run: See /devops/scripts/benchmarks/presets.py
+        required: false
+        default: 'Minimal'  # Only compute-benchmarks
+      pr_no:
+        type: string
+        description: |
+          PR no. to build SYCL from if specified: SYCL will be built from HEAD
+          of incoming branch used by the specified PR no.
 
-on: [ workflow_dispatch ]
+          If both pr_no and commit_hash are empty, the latest SYCL nightly build
+          will be used.
+        required: false
+        default: ''
+      commit_hash:
+        type: string
+        description: |
+          Commit hash (within intel/llvm) to build SYCL from if specified.
+
+          If both pr_no and commit_hash are empty, the latest commit in
+          deployment branch will be used.
+        required: false
+        default: ''
+      upload_results:
+        type: string # true/false: workflow_dispatch does not support booleans
+        required: true
+      runner:
+        type: string
+        required: true
+      backend:
+        type: string
+        required: true
+
+  workflow_dispatch:
+    inputs:
+      preset:
+        type: choice
+        description: |
+          Benchmark presets to run, See /devops/scripts/benchmarks/presets.py. Hint: Minimal is compute-benchmarks only.
+        options:
+          - Full
+          - SYCL
+          - Minimal
+          - Normal
+          - Test
+        default: 'Minimal'  # Only compute-benchmarks
+      pr_no:
+        type: string
+        description: |
+          PR no. to build SYCL from:
+            
+          SYCL will be built from HEAD of incoming branch.
+        required: false
+        default: ''
+      commit_hash:
+        type: string
+        description: |
+          Commit hash (within intel/llvm) to build SYCL from:
+
+          Leave both pr_no and commit_hash empty to use latest commit.
+        required: false
+        default: ''
+      upload_results:
+        description: 'Save and upload results'
+        type: choice
+        options:
+          - false
+          - true
+        default: true
+      runner:
+        type: choice
+        options:
+          - '["PVC_PERF"]'
+      backend:
+        description: Backend to use
+        type: choice
+        options:
+          - 'level_zero:gpu'
+          - 'level_zero_v2:gpu'
+            # As of #17407, sycl-linux-build now builds v2 by default
+
+permissions: read-all
 
 jobs:
-  do-nothing:
-    runs-on: ubuntu-latest
-    steps:
-      - run: echo 'This workflow is a WIP.'
+  build_sycl:
+    name: Build SYCL
+    uses: ./.github/workflows/sycl-linux-build.yml
+    with:
+      build_ref: |
+        ${{
+          inputs.commit_hash != '' && inputs.commit_hash ||
+          inputs.pr_no != '' && format('refs/pull/{0}/head', inputs.pr_no) ||
+          github.ref
+        }}
+      build_cache_root: "/__w/"
+      build_artifact_suffix: "prod_noassert"
+      build_cache_suffix: "prod_noassert"
+      build_configure_extra_args: "--no-assertions"
+      build_image: "ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest"
+      cc: clang
+      cxx: clang++
+      changes: '[]'
 
+  run_benchmarks_build:
+    name: Run Benchmarks on Build
+    needs: [ build_sycl ]
+    strategy:
+      matrix:
+        include:
+          - ref: ${{ inputs.commit_hash != '' && inputs.commit_hash || format('refs/pull/{0}/head', inputs.pr_no) }}
+            save_name: ${{ inputs.commit_hash != '' && format('Commit{0}', inputs.commit_hash) || format('PR{0}', inputs.pr_no) }}
+            # Set default values if not specified:
+            runner: ${{ inputs.runner || '["PVC_PERF"]' }}
+            backend: ${{ inputs.backend || 'level_zero:gpu' }}
+    uses: ./.github/workflows/sycl-linux-run-tests.yml
+    secrets: inherit
+    with:
+      name: Run compute-benchmarks (${{ matrix.save_name }}, ${{ matrix.runner }}, ${{ matrix.backend }})
+      runner: ${{ matrix.runner }}
+      image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest
+      image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
+      target_devices: ${{ matrix.backend }}
+      tests_selector: benchmarks
+      benchmark_upload_results: ${{ inputs.upload_results }}
+      benchmark_save_name: ${{ matrix.save_name }}
+      benchmark_preset: ${{ inputs.preset }}
+      repo_ref: ${{ matrix.ref }}
+      devops_ref: ${{ github.ref }}
+      sycl_toolchain_artifact: sycl_linux_prod_noassert
+      sycl_toolchain_archive: ${{ needs.build_sycl.outputs.artifact_archive_name }}
+      sycl_toolchain_decompress_command: ${{ needs.build_sycl.outputs.artifact_decompress_command }}