Merge branch 'sycl' into bf16-cvt-ext

Author: JackAKirk

.github/CODEOWNERS

@@ -25,8 +25,8 @@ sycl/ @intel/llvm-reviewers-runtime
 sycl/ReleaseNotes.md @intel/dpcpp-doc-reviewers
 sycl/doc/ @intel/dpcpp-doc-reviewers
 sycl/doc/design/ @intel/dpcpp-specification-reviewers
+sycl/doc/design/spirv-extensions/ @intel/dpcpp-spirv-doc-reviewers
 sycl/doc/extensions/ @intel/dpcpp-specification-reviewers
-sycl/doc/extensions/SPIRV/ @intel/dpcpp-spirv-doc-reviewers
 
 # Level Zero plugin
 sycl/plugins/level_zero/ @intel/dpcpp-l0-pi-reviewers
@@ -51,4 +51,4 @@ clang/tools/clang-offload-*/ @intel/dpcpp-tools-reviewers
 ESIMD/ @intel/dpcpp-esimd-reviewers
 esimd/ @intel/dpcpp-esimd-reviewers
 sycl/include/sycl/ext/intel/experimental/esimd.hpp @intel/dpcpp-esimd-reviewers
-sycl/doc/extensions/ExplicitSIMD/ @intel/dpcpp-esimd-reviewers
+sycl/doc/extensions/experimental/sycl_ext_intel_esimd/ @intel/dpcpp-esimd-reviewers

.github/workflows/issue-release-workflow.yml

@@ -19,9 +19,12 @@ on:
     types:
       - created
       - edited
+  issues:
+    types:
+      - opened
 
 env:
-  COMMENT_BODY: ${{ github.event.comment.body }}
+  COMMENT_BODY: ${{ github.event.action == 'opened' && github.event.issue.body || github.event.comment.body  }}
 
 jobs:
   backport-commits:
@@ -30,7 +33,7 @@ jobs:
     if: >-
         (github.repository == 'llvm/llvm-project') &&
         !startswith(github.event.comment.body, '<!--IGNORE-->') &&
-        contains(github.event.comment.body, '/cherry-pick')
+        contains(github.event.action == 'opened' && github.event.issue.body || github.event.comment.body, '/cherry-pick')
     steps:
       - name: Fetch LLVM sources
         uses: actions/checkout@v2

.github/workflows/issue-subscriber.yml

@@ -18,9 +18,12 @@ jobs:
         pip install -r requirements.txt
 
     - name: Update watchers
+      # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
+      env:
+        LABEL_NAME: ${{ github.event.label.name }}
       run: |
         ./github-automation.py \
-          --token ${{ secrets.ISSUE_SUBSCRIBER_TOKEN }} \
+          --token '${{ secrets.ISSUE_SUBSCRIBER_TOKEN }}' \
           issue-subscriber \
-          --issue-number ${{ github.event.issue.number }} \
-          --label-name ${{ github.event.label.name }}
+          --issue-number '${{ github.event.issue.number }}' \
+          --label-name "$LABEL_NAME"

.github/workflows/sycl_cleanup.yml

@@ -0,0 +1,32 @@
+name: Cleanup
+
+on:
+  workflow_run:
+    workflows:
+    - SYCL
+    - SYCL Nightly Builds
+    - SYCL Post Commit
+    types: [completed]
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/github-script@v6
+      with:
+        script: |
+          let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            run_id: context.payload.workflow_run.id,
+          });
+          let matchArtifacts = allArtifacts.data.artifacts.filter((artifact) => {
+            return artifact.name.startsWith("sycl_linux_");
+          });
+          matchArtifacts.forEach(async (artifact) => {
+            await github.rest.actions.deleteArtifact({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              artifact_id: artifact.id
+            });
+          });

.github/workflows/sycl_linux_build_and_test.yml

@@ -1,9 +1,6 @@
 name: Reusable SYCL Linux build and test workflow
 
 on:
-  pull_request:
-    paths:
-    - '.github/workflows/sycl_linux_build_and_test.yml'
   workflow_call:
     inputs:
       cc:
@@ -21,11 +18,10 @@ on:
       build_runs_on:
         type: string
         required: false
-        default: "ubuntu-latest"
-      build_github_cache:
-        type: boolean
+        default: "build"
+      build_ref:
+        type: string
         required: false
-        default: false
       build_cache_root:
         type: string
         required: true
@@ -36,7 +32,7 @@ on:
       build_cache_size:
         type: string
         required: false
-        default: 2G
+        default: 8G
       build_configure_extra_args:
         type: string
         required: false
@@ -52,61 +48,29 @@ on:
         type: string
         required: false
         default: ""
-      gen9_runs_on:
+      lts_cmake_extra_args:
         type: string
         required: false
-        default: "gen9"
+        default: ""
       amdgpu_image:
         type: string
         required: false
         default: "ghcr.io/intel/llvm/ubuntu2004_build:latest"
-      amdgpu_runs_on:
+      lts_ref:
         type: string
         required: false
-        default: "amdgpu"
+        default: 'intel'
+      max_parallel:
+        type: number
+        required: false
+        default: 4
 
 jobs:
-  configure:
-    name: Pre-build configuration
-    runs-on: ubuntu-latest
-    outputs:
-      params: ${{ steps.input-parameters.outputs.params }}
-    steps:
-      - id: input-parameters
-        env:
-          INPUTS: ${{ toJSON(inputs) }}
-        run: |
-          if [[ "$GITHUB_WORKFLOW" == "Reusable SYCL Linux build and test workflow" ]]; then
-            INPUTS="{
-              \"cc\":\"gcc\",
-              \"cxx\":\"g++\",
-              \"build_runs_on\":\"build\",
-              \"build_image\":\"ghcr.io/intel/llvm/ubuntu2004_build:latest\",
-              \"build_github_cache\":\"false\",
-              \"build_cache_root\":\"/__w/\",
-              \"build_cache_suffix\":\"default\",
-              \"build_cache_size\":\"2G\",
-              \"build_configure_extra_args\":\"--hip --cuda\",
-              \"build_artifact_suffix\":\"default\",
-              \"build_upload_artifact\":\"false\",
-              \"intel_drivers_image\":\"ghcr.io/intel/llvm/ubuntu2004_intel_drivers:latest\",
-              \"amdgpu_image\":\"ghcr.io/intel/llvm/ubuntu2004_build:latest\",
-              \"lts_config\":\"ocl_x64;hip_amdgpu\",
-              \"gen9_runs_on\":\"gen9\",
-              \"amdgpu_runs_on\":\"amdgpu\"
-            }"
-          fi
-          INPUTS="${INPUTS//'%'/'%25'}"
-          INPUTS="${INPUTS//$'\n'/'%0A'}"
-          INPUTS="${INPUTS//$'\r'/'%0D'}"
-          echo "::set-output name=params::$INPUTS"
-          echo "$INPUTS"
   build:
     name: Build + LIT
-    needs: configure
-    runs-on: ${{ fromJSON(needs.configure.outputs.params).build_runs_on }}
+    runs-on: ${{ inputs.build_runs_on }}
     container:
-      image: ${{ fromJSON(needs.configure.outputs.params).build_image }}
+      image: ${{ inputs.build_image }}
       options: -u 1001:1001
     steps:
     # GHA requires relative paths for actions. Copy actions from container root
@@ -118,24 +82,16 @@ jobs:
     - uses: ./actions/cached_checkout
       with:
         path: src
+        ref: ${{ inputs.build_ref || github.ref }}
         cache_path: "/__w/repo_cache/"
-    - name: Setup Cache
-      uses: actions/cache@v2
-      if: ${{ steps.parameters.build_github_cache }}
-      id: cache
-      with:
-        path: ${{ fromJSON(needs.configure.outputs.params).build_cache_root }}/build_cache_${{ fromJSON(needs.configure.outputs.params).build_cache_suffix }}
-        key: sycl-build-${{ fromJSON(needs.configure.outputs.params).build_cache_suffix }}-${{ github.sha }}
-        restore-keys: |
-          sycl-build-${{ fromJSON(needs.configure.outputs.params).build_cache_suffix }}-
     - name: Configure
       env:
-        CC: ${{ fromJSON(needs.configure.outputs.params).cc }}
-        CXX: ${{ fromJSON(needs.configure.outputs.params).cxx }}
-        CACHE_ROOT: ${{ fromJSON(needs.configure.outputs.params).build_cache_root }}
-        CACHE_SUFFIX: ${{ fromJSON(needs.configure.outputs.params).build_cache_suffix }}
-        CACHE_SIZE: ${{ fromJSON(needs.configure.outputs.params).build_cache_size }}
-        ARGS: ${{ fromJSON(needs.configure.outputs.params).build_configure_extra_args }}
+        CC: ${{ inputs.cc }}
+        CXX: ${{ inputs.cxx }}
+        CACHE_ROOT: ${{ inputs.build_cache_root }}
+        CACHE_SUFFIX: ${{ inputs.build_cache_suffix }}
+        CACHE_SIZE: ${{ inputs.build_cache_size }}
+        ARGS: ${{ inputs.build_configure_extra_args }}
         CUDA_LIB_PATH: "/usr/local/cuda/lib64/stubs"
       run: |
         mkdir -p $CACHE_ROOT/build_cache_$CACHE_SUFFIX
@@ -203,83 +159,50 @@ jobs:
     - name: Pack LIT
       run: tar -cJf lit.tar.xz -C $GITHUB_WORKSPACE/src/llvm/utils/lit .
     - name: Upload toolchain
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v2
       with:
-        name: sycl_linux_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
+        name: sycl_linux_${{ inputs.build_artifact_suffix }}
         path: llvm_sycl.tar.xz
     - name: Upload LIT
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v2
       with:
-        name: sycl_lit_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
+        name: sycl_lit_${{ inputs.build_artifact_suffix }}
         path: lit.tar.xz
 
-  llvm_test_suite_l0_gen9:
-    name: L0 GEN9 Test Suite
-    needs: [build, configure]
-    if: ${{ contains(fromJSON(needs.configure.outputs.params).lts_config, 'l0_gen9') }}
-    runs-on: ${{ fromJSON(needs.configure.outputs.params).gen9_runs_on }}
-    container:
-      image: ${{ fromJSON(needs.configure.outputs.params).intel_drivers_image }}
-      options: -u 1001 --device=/dev/dri
-    steps:
-    - run: cp -r /actions .
-    - name: Register cleanup after job is finished
-      uses: ./actions/cleanup
-    # FIXME cached_checkout fails here, but works everywhere else
-    - uses: actions/checkout@v2
-      with:
-        path: llvm
-    # TODO should this action be packed into container as well?
-    - uses: ./llvm/devops/actions/llvm_test_suite
-      name: Run LLVM Test Suite
-      with:
-        # TODO allow custom test references
-        test_ref: 'intel'
-        sycl_artifact: sycl_linux_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
-        sycl_archive: llvm_sycl.tar.xz
-        lit_artifact: sycl_lit_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
-        lit_archive: lit.tar.xz
-        check_sycl_all: 'level_zero:gpu,host'
-        results_name_suffix: l0_gpu_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
-        cmake_args: '-DGPU_AOT_TARGET_OPTS="\\\"-device gen9\\\""'
-
-  llvm_test_suite_ocl_gen9:
-    name: OCL GEN9 Test Suite
-    needs: [build, configure]
-    if: ${{ contains(fromJSON(needs.configure.outputs.params).lts_config, 'ocl_gen9') }}
-    runs-on: ${{ fromJSON(needs.configure.outputs.params).gen9_runs_on }}
-    container:
-      image: ${{ fromJSON(needs.configure.outputs.params).intel_drivers_image }}
-      options: -u 1001 --device=/dev/dri
+  # This job generates matrix of tests for LLVM Test Suite
+  resolve_matrix:
+    name: Resolve Test Matrix
+    runs-on: ubuntu-latest
+    outputs:
+      lts: ${{ steps.work.outputs.lts }}
     steps:
-    - run: cp -r /actions .
-    - name: Register cleanup after job is finished
-      uses: ./actions/cleanup
-    # FIXME cached_checkout fails here, but works everywhere else
-    - uses: actions/checkout@v2
-      with:
-        path: llvm
-    - uses: ./llvm/devops/actions/llvm_test_suite
-      name: Run LLVM Test Suite
+    - name: Download scripts
+      run: |
+        wget raw.githubusercontent.com/intel/llvm/${{ github.sha }}/devops/scripts/generate_test_matrix.js
+        wget raw.githubusercontent.com/intel/llvm/${{ github.sha }}/devops/test_configs.json
+    - id: work
+      uses: actions/github-script@v6
+      name: Generate matrix
+      env:
+        GHA_INPUTS: ${{ toJSON(inputs) }}
       with:
-        # TODO allow custom test references
-        test_ref: 'intel'
-        sycl_artifact: sycl_linux_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
-        sycl_archive: llvm_sycl.tar.xz
-        lit_artifact: sycl_lit_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
-        lit_archive: lit.tar.xz
-        check_sycl_all: 'opencl:gpu,host'
-        results_name_suffix: ocl_gpu_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
-        cmake_args: '-DGPU_AOT_TARGET_OPTS="\\\"-device gen9\\\""'
+        script: |
+          const script = require('./generate_test_matrix.js');
+          script({core, process});
 
-  llvm_test_suite_ocl_x64:
-    name: OCL x64 Test Suite
-    needs: [build, configure]
-    if: ${{ contains(fromJSON(needs.configure.outputs.params).lts_config, 'ocl_x64') }}
-    runs-on: x64
+  llvm_test_suite:
+    needs: [build, resolve_matrix]
+    if: ${{ inputs.lts_config != '' }}
+    strategy:
+      fail-fast: false
+      max-parallel: ${{ inputs.max_parallel }}
+      matrix:
+        include: ${{ fromJSON(needs.resolve_matrix.outputs.lts) }}
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.runs-on }}
     container:
-      image: ${{ fromJSON(needs.configure.outputs.params).intel_drivers_image }}
-      options: -u 1001
+      image: ${{ matrix.image }}
+      options: ${{ matrix.container_options }}
     steps:
     - run: cp -r /actions .
     - name: Register cleanup after job is finished
@@ -288,45 +211,16 @@ jobs:
     - uses: actions/checkout@v2
       with:
         path: llvm
+    # TODO should this action be packed into container as well?
     - uses: ./llvm/devops/actions/llvm_test_suite
       name: Run LLVM Test Suite
       with:
-        # TODO allow custom test references
-        test_ref: 'intel'
-        sycl_artifact: sycl_linux_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
+        test_ref: ${{ inputs.lts_ref }}
+        sycl_artifact: sycl_linux_${{ inputs.build_artifact_suffix }}
         sycl_archive: llvm_sycl.tar.xz
-        lit_artifact: sycl_lit_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
+        lit_artifact: sycl_lit_${{ inputs.build_artifact_suffix }}
         lit_archive: lit.tar.xz
-        check_sycl_all: 'opencl:cpu,host'
-        results_name_suffix: ocl_x64_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
+        check_sycl_all: ${{ matrix.check_sycl_all }}
+        results_name_suffix: ${{ matrix.config }}_${{ inputs.build_artifact_suffix }}
+        cmake_args: '${{ matrix.cmake_args }} ${{ inputs.lts_cmake_extra_args }}'
 
-  llvm_test_suite_hip_amdgpu:
-    name: HIP AMD GPU Test Suite
-    needs: [build, configure]
-    if: ${{ contains(fromJSON(needs.configure.outputs.params).lts_config, 'hip_amdgpu') }}
-    runs-on: ${{ fromJSON(needs.configure.outputs.params).amdgpu_runs_on }}
-    container:
-      image: ${{ fromJSON(needs.configure.outputs.params).amdgpu_image }}
-      options: --device=/dev/dri --device=/dev/kfd
-    steps:
-    - run: cp -r /actions .
-    - name: Register cleanup after job is finished
-      uses: ./actions/cleanup
-    # TODO remove this step one LLVM Test Suite action is settled and packed
-    # into container.
-    # FIXME cached_checkout fails here, but works everywhere else
-    - uses: actions/checkout@v2
-      with:
-        path: llvm
-    - uses: ./llvm/devops/actions/llvm_test_suite
-      name: Run LLVM Test Suite
-      with:
-        # TODO allow custom test references
-        test_ref: 'intel'
-        sycl_artifact: sycl_linux_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
-        sycl_archive: llvm_sycl.tar.xz
-        lit_artifact: sycl_lit_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
-        lit_archive: lit.tar.xz
-        check_sycl_all: 'hip:gpu,host'
-        results_name_suffix: hip_amdgpu_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
-        cmake_args: '-DHIP_PLATFORM="AMD" -DAMD_ARCH="gfx1031"'