Merge remote-tracking branch 'origin/main' into yc-PR/241107-misc-minor-fix

Author: yingcong-wu

.github/labeler.yml

@@ -31,7 +31,7 @@ loader:
 opencl:
   - changed-files:
     - any-glob-to-any-file:
-      - source/adapter/opencl/**
+      - source/adapters/opencl/**
 
 level-zero:
   - changed-files:

.github/scripts/0001-travis-fix-travisci_build_coverity_scan.sh.patch

@@ -176,7 +176,7 @@ jobs:
           let markdown = ""
           try {
             const fs = require('fs');
-            markdown = fs.readFileSync('benchmark_results.md', 'utf8');
+            markdown = fs.readFileSync('ur-repo/benchmark_results.md', 'utf8');
           } catch(err) {
           }
 
@@ -199,5 +199,5 @@ jobs:
       if: ${{ always() && inputs.upload_report }}
       uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
       with:
-        path: ${{ github.workspace }}/ur-repo/benchmark_results.html
+        path: ur-repo/benchmark_results.html
         key: benchmark-results-${{ matrix.adapter.str_name }}-${{ github.run_id }}

.github/workflows/benchmarks-reusable.yml

@@ -41,12 +41,14 @@ jobs:
 
     - name: Build level zero with gcc
       run: |
-        git clone -b v1.17.6 --depth=1 https://github.com/oneapi-src/level-zero.git ${{github.workspace}}/level-zero
+        git clone -b v1.18.5 --depth=1 https://github.com/oneapi-src/level-zero.git ${{github.workspace}}/level-zero
         cd ${{github.workspace}}/level-zero
         cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++
         cmake --build build -j $(nproc)
 
     - name: Configure CMake
+      # CFI sanitization (or flto?) seems to cause linking to fail
+      # https://github.com/oneapi-src/unified-runtime/issues/2323
       run: >
         cmake
         -B${{github.workspace}}/build
@@ -58,6 +60,7 @@ jobs:
         -DUR_USE_ASAN=ON
         -DUR_USE_UBSAN=ON
         -DUR_BUILD_ADAPTER_L0=ON
+        -DUR_USE_CFI=OFF
         -DUR_LEVEL_ZERO_LOADER_LIBRARY=${{github.workspace}}/level-zero/build/lib/libze_loader.so
         -DUR_LEVEL_ZERO_INCLUDE_DIR=${{github.workspace}}/level-zero/include/
         -DUR_DPCXX=${{github.workspace}}/dpcpp_compiler/bin/clang++

.github/workflows/build-fuzz-reusable.yml

@@ -256,10 +256,12 @@ jobs:
            compiler: {c: clang-cl, cxx: clang-cl}
 
         build_type: [Debug, Release]
-        compiler: [{c: cl, cxx: cl}, {c: clang-cl, cxx: clang-cl}]
+        # TODO: clang-cl seems to be fully broken (https://github.com/oneapi-src/unified-runtime/issues/2348)
+        #compiler: [{c: cl, cxx: cl}, {c: clang-cl, cxx: clang-cl}]
+        compiler: [{c: cl, cxx: cl}]
         include:
-          - compiler: {c: clang-cl, cxx: clang-cl}
-            toolset: "-T ClangCL"
+          #- compiler: {c: clang-cl, cxx: clang-cl}
+          #  toolset: "-T ClangCL"
           - os: 'windows-2022'
             adapter: {name: L0, var: '-DUR_BUILD_ADAPTER_L0=ON -DUR_STATIC_ADAPTER_L0=ON'}
             build_type: 'Release'

.github/workflows/cmake.yml

@@ -1,63 +1,81 @@
-#
-# Copyright (C) 2023-2024 Intel Corporation
-#
-# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
-# See LICENSE.TXT
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
+# Coverity - static analysis build. It requires Coverity's token (set in CI's secret).
 name: coverity-unified-runtime
-# It runs static analysis build - Coverity. It requires special token (set in CI's secret).
 
 on:
   workflow_dispatch:
   schedule:
     # Run every day at 22:00 UTC
     - cron: '0 22 * * *'
 
-env:
-  WORKDIR:                           ${{ github.workspace }}
-  COVERITY_SCAN_NOTIFICATION_EMAIL:  ${{ secrets.COVERITY_SCAN_NOTIFICATION_EMAIL }}
-  COVERITY_SCAN_TOKEN:               ${{ secrets.COVERITY_SCAN_TOKEN }}
-  COVERITY_SCAN_PROJECT_NAME:        ${{ github.repository }}
-  COVERITY_SCAN_BUILD_COMMAND:       "cmake --build ${{github.workspace}}/build"
-  COVERITY_SCAN_BRANCH_PATTERN:      "main"
-  TRAVIS_BRANCH:                     ${{ github.ref_name }}
-
 permissions:
   contents: read
 
 jobs:
-  linux:
+  coverity:
     name: Coverity
-    runs-on: coverity
+    # run only on upstream; forks don't have token for upstream's cov project
+    if: github.repository == 'oneapi-src/unified-runtime'
+    runs-on: ubuntu-latest
 
     steps:
-      - name: Clone the git repo
+      - name: Checkout repository
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          fetch-depth: 0
+
+      - name: Install dependencies
+        run: |
+          wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+          sudo dpkg -i cuda-keyring_1.1-1_all.deb
+          sudo apt-get update
+          sudo apt-get install -y libhwloc-dev libtbb-dev cuda-toolkit-12-6
 
       - name: Install pip packages
         run: pip install -r third_party/requirements.txt
 
+      - name: Download Coverity
+        run: |
+          wget -O coverity_tool.tgz -nv https://scan.coverity.com/download/linux64 \
+            --post-data "token=${{ secrets.COVERITY_SCAN_TOKEN }}&project=oneapi-src%2Funified-runtime"
+
+      - name: Extract Coverity
+        run: tar xzf coverity_tool.tgz
+
+      # TODO: enable HIP adapter as well (requires proper package(s) installation)
       - name: Configure CMake
         run: >
           cmake
-          -B $WORKDIR/build
+          -B ${{github.workspace}}/build
+          -DCMAKE_BUILD_TYPE=Release
+          -DUR_DEVELOPER_MODE=OFF
+          -DUR_FORMAT_CPP_STYLE=ON
           -DUR_ENABLE_TRACING=ON
-          -DUR_DEVELOPER_MODE=ON
           -DUR_BUILD_TESTS=ON
-          -DUMF_ENABLE_POOL_TRACKING=ON
-          -DUR_FORMAT_CPP_STYLE=ON
-          -DCMAKE_BUILD_TYPE=Debug
           -DUR_BUILD_ADAPTER_L0=ON
           -DUR_BUILD_ADAPTER_CUDA=ON
-          -DCUDA_CUDA_LIBRARY=/usr/local/cuda/lib64/stubs/libcuda.so
+          -DCUDA_CUDA_LIBRARY=/usr/local/cuda-12.6/targets/x86_64-linux/lib/stubs/libcuda.so
           -DUR_BUILD_ADAPTER_NATIVE_CPU=ON
-          -DUR_BUILD_ADAPTER_HIP=ON
+          -DUR_BUILD_ADAPTER_HIP=OFF
           -DUR_BUILD_ADAPTER_OPENCL=ON
 
-      - name: Run Coverity
+      - name: Build
+        run: |
+          export COVERITY_DIR=$(find . -maxdepth 1 -type d -name "cov-analysis-linux64-*" | head -n 1)
+          if [ -n "$COVERITY_DIR" ]; then
+            export PATH="$PATH:$COVERITY_DIR/bin"
+          fi
+          cov-build --dir ${{github.workspace}}/cov-int cmake --build ${{github.workspace}}/build --config Release -j$(nproc)
+
+      - name: Create tarball to analyze
+        run: tar czvf cov-int_ur.tgz cov-int
+
+      - name: Push tarball to scan
         run: |
-          cd $WORKDIR/build
-          wget https://scan.coverity.com/scripts/travisci_build_coverity_scan.sh
-          patch < "../.github/scripts/0001-travis-fix-travisci_build_coverity_scan.sh.patch"
-          bash ./travisci_build_coverity_scan.sh
+          BRANCH_NAME=$(echo ${GITHUB_REF_NAME})
+          COMMIT_ID=$(echo $GITHUB_SHA)
+          curl --form token=${{ secrets.COVERITY_SCAN_TOKEN }} \
+          --form [email protected] \
+          --form file=@cov-int_ur.tgz \
+          --form version="$COMMIT_ID" \
+          --form description="$BRANCH_NAME:$COMMIT_ID" \
+          https://scan.coverity.com/builds\?project\=oneapi-src%2Funified-runtime

.github/workflows/coverity.yml

@@ -54,9 +54,8 @@ jobs:
         id: download-bench-html
         uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
         with:
-          path: ${{ github.workspace }}/ur-repo/benchmark_results.html
+          path: ur-repo/benchmark_results.html
           key: benchmark-results-
-          restore-keys: benchmark-results-
 
       - name: Move benchmark HTML
         # exact or partial cache hit

.github/workflows/docs.yml

@@ -16,19 +16,6 @@ include(CheckCXXSourceCompiles)
 include(CMakePackageConfigHelpers)
 include(CTest)
 
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
-include(helpers)
-
-if(CMAKE_SYSTEM_NAME STREQUAL Darwin)
-    set(Python3_FIND_FRAMEWORK NEVER)
-    set(Python3_FIND_STRATEGY LOCATION)
-endif()
-
-find_package(Python3 COMPONENTS Interpreter REQUIRED)
-
-set(CMAKE_CXX_STANDARD 17)
-set(CMAKE_CXX_STANDARD_REQUIRED YES)
-
 # Build Options
 option(UR_BUILD_EXAMPLES "Build example applications." ON)
 option(UR_BUILD_TESTS "Build unit tests." ON)
@@ -40,6 +27,7 @@ option(UR_USE_ASAN "enable AddressSanitizer" OFF)
 option(UR_USE_UBSAN "enable UndefinedBehaviorSanitizer" OFF)
 option(UR_USE_MSAN "enable MemorySanitizer" OFF)
 option(UR_USE_TSAN "enable ThreadSanitizer" OFF)
+option(UR_USE_CFI "enable Control Flow Integrity checks (requires clang and implies -flto)" ON)
 option(UR_ENABLE_TRACING "enable api tracing through xpti" OFF)
 option(UR_ENABLE_SANITIZER "enable device sanitizer" ON)
 option(UR_ENABLE_SYMBOLIZER "enable symoblizer for sanitizer" OFF)
@@ -80,6 +68,19 @@ set(UR_ADAPTER_HIP_SOURCE_DIR "" CACHE PATH
 set(UR_ADAPTER_NATIVE_CPU_SOURCE_DIR "" CACHE PATH
     "Path to external 'native_cpu' adapter source dir")
 
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+include(helpers)
+
+if(CMAKE_SYSTEM_NAME STREQUAL Darwin)
+    set(Python3_FIND_FRAMEWORK NEVER)
+    set(Python3_FIND_STRATEGY LOCATION)
+endif()
+
+find_package(Python3 COMPONENTS Interpreter REQUIRED)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED YES)
+
 # There's little reason not to generate the compile_commands.json
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 

CMakeLists.txt

@@ -3,7 +3,8 @@
 [![Build and test](https://github.com/oneapi-src/unified-runtime/actions/workflows/cmake.yml/badge.svg)](https://github.com/oneapi-src/unified-runtime/actions/workflows/cmake.yml)
 [![Bandit](https://github.com/oneapi-src/unified-runtime/actions/workflows/bandit.yml/badge.svg)](https://github.com/oneapi-src/unified-runtime/actions/workflows/bandit.yml)
 [![CodeQL](https://github.com/oneapi-src/unified-runtime/actions/workflows/codeql.yml/badge.svg)](https://github.com/oneapi-src/unified-runtime/actions/workflows/codeql.yml)
-[![Coverity](https://scan.coverity.com/projects/28213/badge.svg)](https://scan.coverity.com/projects/oneapi-src-unified-runtime)
+[![Coverity build](https://github.com/oneapi-src/unified-runtime/actions/workflows/coverity.yml/badge.svg?branch=main)](https://github.com/oneapi-src/unified-runtime/actions/workflows/coverity.yml)
+[![Coverity report](https://scan.coverity.com/projects/28213/badge.svg)](https://scan.coverity.com/projects/oneapi-src-unified-runtime)
 [![Nightly](https://github.com/oneapi-src/unified-runtime/actions/workflows/nightly.yml/badge.svg)](https://github.com/oneapi-src/unified-runtime/actions/workflows/nightly.yml)
 [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/oneapi-src/unified-runtime/badge)](https://securityscorecards.dev/viewer/?uri=github.com/oneapi-src/unified-runtime)
 [![Trivy](https://github.com/oneapi-src/unified-runtime/actions/workflows/trivy.yml/badge.svg)](https://github.com/oneapi-src/unified-runtime/actions/workflows/trivy.yml)
@@ -129,6 +130,7 @@ List of options provided by CMake:
 | UR_USE_TSAN | Enable ThreadSanitizer | ON/OFF | OFF |
 | UR_USE_UBSAN | Enable UndefinedBehavior Sanitizer | ON/OFF | OFF |
 | UR_USE_MSAN | Enable MemorySanitizer (clang only) | ON/OFF | OFF |
+| UR_USE_CFI | Enable Control Flow Integrity checks (clang only, also enables lto) | ON/OFF | ON |
 | UR_ENABLE_TRACING | Enable XPTI-based tracing layer | ON/OFF | OFF |
 | UR_ENABLE_SANITIZER | Enable device sanitizer layer | ON/OFF | ON |
 | UR_CONFORMANCE_TARGET_TRIPLES | SYCL triples to build CTS device binaries for | Comma-separated list | spir64 |

README.md

@@ -40,7 +40,7 @@ if (NOT DEFINED LEVEL_ZERO_LIBRARY OR NOT DEFINED LEVEL_ZERO_INCLUDE_DIR)
         set(UR_LEVEL_ZERO_LOADER_REPO "https://github.com/oneapi-src/level-zero.git")
     endif()
     if (UR_LEVEL_ZERO_LOADER_TAG STREQUAL "")
-        set(UR_LEVEL_ZERO_LOADER_TAG v1.17.39)
+        set(UR_LEVEL_ZERO_LOADER_TAG v1.19.2)
     endif()
 
     # Disable due to a bug https://github.com/oneapi-src/level-zero/issues/104

cmake/FetchLevelZero.cmake

@@ -63,6 +63,16 @@ if(CMAKE_SYSTEM_NAME STREQUAL Linux)
     check_cxx_compiler_flag("-fstack-clash-protection" CXX_HAS_FSTACK_CLASH_PROTECTION)
 endif()
 
+if (UR_USE_CFI)
+    set(SAVED_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
+    set(CMAKE_REQUIRED_FLAGS "-flto -fvisibility=hidden")
+    check_cxx_compiler_flag("-fsanitize=cfi" CXX_HAS_CFI_SANITIZE)
+    set(CMAKE_REQUIRED_FLAGS ${SAVED_CMAKE_REQUIRED_FLAGS})
+else()
+    # If CFI checking is disabled, pretend we don't support it
+    set(CXX_HAS_CFI_SANITIZE OFF)
+endif()
+
 function(add_ur_target_compile_options name)
     if(NOT MSVC)
         target_compile_definitions(${name} PRIVATE -D_FORTIFY_SOURCE=2)
@@ -78,11 +88,10 @@ function(add_ur_target_compile_options name)
             # Hardening options
             -fPIC
             -fstack-protector-strong
-            -fvisibility=hidden # Required for -fsanitize=cfi
-            # -fsanitize=cfi requires -flto, which breaks a lot of things
-            # See: https://github.com/oneapi-src/unified-runtime/issues/2120
-            # -flto
-            # $<$<CXX_COMPILER_ID:Clang,AppleClang>:-fsanitize=cfi>
+            -fvisibility=hidden
+            # cfi-icall requires called functions in shared libraries to also be built with cfi-icall, which we can't
+            # guarantee. -fsanitize=cfi depends on -flto
+            $<$<BOOL:${CXX_HAS_CFI_SANITIZE}>:-flto -fsanitize=cfi -fno-sanitize=cfi-icall>
             $<$<BOOL:${CXX_HAS_FCF_PROTECTION_FULL}>:-fcf-protection=full>
             $<$<BOOL:${CXX_HAS_FSTACK_CLASH_PROTECTION}>:-fstack-clash-protection>
 
@@ -119,7 +128,10 @@ endfunction()
 function(add_ur_target_link_options name)
     if(NOT MSVC)
         if (NOT APPLE)
-            target_link_options(${name} PRIVATE "LINKER:-z,relro,-z,now,-z,noexecstack")
+            target_link_options(${name} PRIVATE
+                $<$<BOOL:${CXX_HAS_CFI_SANITIZE}>:-flto -fsanitize=cfi -fno-sanitize=cfi-icall>
+                "LINKER:-z,relro,-z,now,-z,noexecstack"
+            )
             if (UR_DEVELOPER_MODE)
                 target_link_options(${name} PRIVATE -Werror -Wextra)
             endif()
@@ -131,17 +143,17 @@ function(add_ur_target_link_options name)
         endif()
     elseif(MSVC)
         target_link_options(${name} PRIVATE
-            /DYNAMICBASE
-            /HIGHENTROPYVA
-            /NXCOMPAT
+            LINKER:/DYNAMICBASE
+            LINKER:/HIGHENTROPYVA
+            LINKER:/NXCOMPAT
         )
     endif()
 endfunction()
 
 function(add_ur_target_exec_options name)
     if(MSVC)
         target_link_options(${name} PRIVATE
-            /ALLOWISOLATION
+            LINKER:/ALLOWISOLATION
         )
     endif()
 endfunction()
@@ -159,7 +171,7 @@ function(add_ur_library name)
     add_ur_target_link_options(${name})
     if(MSVC)
         target_link_options(${name} PRIVATE
-            $<$<STREQUAL:$<TARGET_LINKER_FILE_NAME:${name}>,link.exe>:/DEPENDENTLOADFLAG:0x2000>
+            $<$<STREQUAL:$<TARGET_LINKER_FILE_NAME:${name}>,link.exe>:LINKER:/DEPENDENTLOADFLAG:0x2000>
         )
     endif()
 endfunction()