[CI] New workflows order part1

.github/workflows/docs.yml

@@ -29,8 +29,13 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y doxygen
 
-    - name: Install pip requirements
-      run: python3 -m pip install -r third_party/requirements.txt
+    # Latest distros do not allow global pip installation
+    - name: Install Python requirements in venv
+      run: |
+        python3 -m venv .venv
+        . .venv/bin/activate
+        echo "$PATH" >> $GITHUB_PATH
+        python3 -m pip install -r third_party/requirements.txt
 
     - name: Setup PATH for python
       run: echo "$HOME/.local/bin" >> $GITHUB_PATH

.github/workflows/pr_push.yml

@@ -1,4 +1,5 @@
-# Checks required for a PR to merge. This workflow mostly call other workflows.
+# Run checks required for a PR to merge and verify if post-merge commit is valid.
+# This workflow only call other workflows.
 name: PR/push
 
 on:
@@ -15,101 +16,54 @@ permissions:
   contents: read
 
 jobs:
-  CodeStyle:
-    name: Coding style
-    runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-latest' }}
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-      with:
-        fetch-depth: 0
-
-    - name: Install apt packages
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y black cmake clang-format-15 cmake-format libhwloc-dev
-
-    - name: Configure CMake
-      run: >
-        cmake
-        -B ${{github.workspace}}/build
-        -DUMF_FORMAT_CODE_STYLE=ON
-        -DUMF_BUILD_TESTS=OFF
-        -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
-        -DUMF_BUILD_CUDA_PROVIDER=OFF
-        -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=OFF
-
-    - name: Check C/C++ formatting
-      run: cmake --build build --target clang-format-check
-
-    - name: Check CMake formatting
-      run: |
-        cmake --build build --target cmake-format-apply
-        git diff --exit-code
-
-    - name: Check Python formatting
-      run: cmake --build build --target black-format-check
-
+  CodeChecks:
+    uses: ./.github/workflows/reusable_checks.yml
   DocsBuild:
-    name: Build docs
-    runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-latest' }}
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-      with:
-        fetch-depth: 0
-
-    - name: Install doxygen
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y doxygen
-
-    - name: Install pip requirements
-      run: python3 -m pip install -r third_party/requirements.txt
-
-    - name: Setup PATH for python
-      run: echo "$HOME/.local/bin" >> $GITHUB_PATH
-
-    - name: Build the documentation
-      working-directory: scripts
-      run: python3 generate_docs.py
-
-  Spellcheck:
-    uses: ./.github/workflows/spellcheck.yml
+    uses: ./.github/workflows/reusable_docs_build.yml
   FastBuild:
     name: Fast builds
-    needs: [Spellcheck, CodeStyle]
-    uses: ./.github/workflows/fast.yml
+    needs: [CodeChecks, DocsBuild]
+    uses: ./.github/workflows/reusable_fast.yml
   Build:
     name: Basic builds
     needs: [FastBuild]
-    uses: ./.github/workflows/basic.yml
+    uses: ./.github/workflows/reusable_basic.yml
   DevDax:
     needs: [FastBuild]
-    uses: ./.github/workflows/dax.yml
+    uses: ./.github/workflows/reusable_dax.yml
   Sanitizers:
     needs: [FastBuild]
-    uses: ./.github/workflows/sanitizers.yml
+    uses: ./.github/workflows/reusable_sanitizers.yml
   Qemu:
     needs: [FastBuild]
-    uses: ./.github/workflows/qemu.yml
+    uses: ./.github/workflows/reusable_qemu.yml
   Benchmarks:
     needs: [Build]
-    uses: ./.github/workflows/benchmarks.yml
+    uses: ./.github/workflows/reusable_benchmarks.yml
   ProxyLib:
     needs: [Build]
-    uses: ./.github/workflows/proxy_lib.yml
+    uses: ./.github/workflows/reusable_proxy_lib.yml
   GPU:
     needs: [Build]
-    uses: ./.github/workflows/gpu.yml
+    uses: ./.github/workflows/reusable_gpu.yml
   Valgrind:
     needs: [Build]
-    uses: ./.github/workflows/valgrind.yml
+    uses: ./.github/workflows/reusable_valgrind.yml
   MultiNuma:
     needs: [Build]
-    uses: ./.github/workflows/multi_numa.yml
+    uses: ./.github/workflows/reusable_multi_numa.yml
   Coverage:
     needs: [Build, DevDax, GPU, MultiNuma, Qemu, ProxyLib]
-    uses: ./.github/workflows/coverage.yml
+    uses: ./.github/workflows/reusable_coverage.yml
+  CodeQL:
+    needs: [Build]
+    permissions:
+      contents: read
+      security-events: write
+    uses: ./.github/workflows/reusable_codeql.yml
+  Trivy:
+    needs: [Build]
+    permissions:
+      contents: read
+      security-events: write
+    uses: ./.github/workflows/reusable_trivy.yml

.github/workflows/reusable_checks.yml

@@ -0,0 +1,62 @@
+# Basic checks on the code, incl. coding style, spelling, bandit analysis.
+# TODO: add license check
+name: Basic checks
+
+on: workflow_call
+
+permissions:
+  contents: read
+
+jobs:
+  CodeChecks:
+    name: Basic code checks
+    runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-latest' }}
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        fetch-depth: 0
+
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y black cmake clang-format-15 cmake-format libhwloc-dev
+
+    # Latest distros do not allow global pip installation
+    - name: Install Python requirements in venv
+      run: |
+        python3 -m venv .venv
+        . .venv/bin/activate
+        echo "$PATH" >> $GITHUB_PATH
+        python3 -m pip install bandit
+
+    - name: Configure CMake
+      run: >
+        cmake
+        -B ${{github.workspace}}/build
+        -DUMF_FORMAT_CODE_STYLE=ON
+        -DUMF_BUILD_TESTS=OFF
+        -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
+        -DUMF_BUILD_CUDA_PROVIDER=OFF
+        -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=OFF
+
+    - name: Check C/C++ formatting
+      run: cmake --build build --target clang-format-check
+
+    - name: Check CMake formatting
+      run: |
+        cmake --build build --target cmake-format-apply
+        git diff --exit-code
+
+    - name: Check Python formatting
+      run: cmake --build build --target black-format-check
+
+    - name: Run a spell check
+      uses: crate-ci/typos@b63f421581dce830bda2f597a678cb7776b41877 # v1.18.2
+      with:
+        config: ./.github/workflows/.spellcheck-conf.toml
+
+    # Run Bandit recursively, but omit _deps directory (with 3rd party code) and python's venv
+    - name: Run Bandit
+      run: python3 -m bandit -r . -x '/_deps/,/.venv/'

.github/workflows/codeql.yml → .github/workflows/reusable_codeql.yml

@@ -1,19 +1,7 @@
 # CodeQL static analysis
 name: CodeQL
 
-# Due to lower score on Scorecard we're running this separately from
-# "PR/push" workflow. For some reason permissions weren't properly set
-# or recognized (by Scorecard). If Scorecard changes its behavior we can
-# go back to use 'workflow_call' trigger.
-on:
-  push:
-    branches-ignore:
-      - 'dependabot/**'
-  pull_request:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
+on: workflow_call
 
 permissions:
   contents: read
@@ -56,27 +44,34 @@ jobs:
       with:
         languages: cpp
 
-    - name: Initialize vcpkg
-      if: ${{ matrix.os == 'windows-latest' }}
+    - name: "[Win] Initialize vcpkg"
+      if: matrix.os == 'windows-latest'
       uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5
       with:
         vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289
         vcpkgDirectory: ${{env.BUILD_DIR}}/vcpkg
         vcpkgJsonGlob: '**/vcpkg.json'
 
-    - name: Install dependencies
-      if: ${{ matrix.os == 'windows-latest' }}
-      run: vcpkg install
-      shell: pwsh # Specifies PowerShell as the shell for running the script.
+    - name: "[Win] Install dependencies"
+      if: matrix.os == 'windows-latest'
+      run: |
+        vcpkg install
+        python3 -m pip install -r third_party/requirements.txt
 
-    - name: Install apt packages
+    - name: "[Lin] Install apt packages"
       if: matrix.os == 'ubuntu-latest'
       run: |
         sudo apt-get update
         sudo apt-get install -y cmake clang libhwloc-dev libnuma-dev libjemalloc-dev libtbb-dev
 
-    - name: Install pip packages
-      run: python3 -m pip install -r third_party/requirements.txt
+    # Latest distros do not allow global pip installation
+    - name: "[Lin] Install Python requirements in venv"
+      if: matrix.os == 'ubuntu-latest'
+      run: |
+        python3 -m venv .venv
+        . .venv/bin/activate
+        echo "$PATH" >> $GITHUB_PATH
+        python3 -m pip install -r third_party/requirements.txt
 
     - name: Configure CMake
       run: >

.github/workflows/coverage.yml → .github/workflows/reusable_coverage.yml

@@ -1,4 +1,4 @@
-# Coverage build
+# Coverage build - gather artifacts from other builds and merge them into a single report
 name: Coverage
 
 on: workflow_call
@@ -20,7 +20,7 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Install dependencies (ubuntu-latest)
+      - name: Install dependencies
         run: |
           sudo apt-get update
           sudo apt-get install -y lcov

.github/workflows/reusable_docs_build.yml

@@ -0,0 +1,34 @@
+name: Docs build
+
+on: workflow_call
+
+permissions:
+  contents: read
+
+jobs:
+  DocsBuild:
+    name: Docs build
+    runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-latest' }}
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        fetch-depth: 0
+
+    - name: Install doxygen
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y doxygen
+
+    # Latest distros do not allow global pip installation
+    - name: Install Python requirements in venv
+      run: |
+        python3 -m venv .venv
+        . .venv/bin/activate
+        echo "$PATH" >> $GITHUB_PATH
+        python3 -m pip install -r third_party/requirements.txt
+
+    - name: Build the documentation
+      working-directory: scripts
+      run: python3 generate_docs.py

.github/workflows/fast.yml → .github/workflows/reusable_fast.yml

@@ -133,16 +133,6 @@ jobs:
       working-directory: ${{env.BUILD_DIR}}
       run: ctest --output-on-failure --test-dir test -C Release
 
-    - name: check /DEPENDENTLOADFLAG (Windows only)
-      if: matrix.os == 'windows-latest'
-      run: ${{github.workspace}}/.github/scripts/check_dll_flags.ps1 ${{env.BUILD_DIR}}/bin/Release/umf.dll
-      shell: pwsh
-
-    - name: check /DEPENDENTLOADFLAG in umf_proxy.dll
-      if: matrix.os == 'windows-latest'
-      run: ${{github.workspace}}/.github/scripts/check_dll_flags.ps1 ${{env.BUILD_DIR}}/src/proxy_lib/Release/umf_proxy.dll
-      shell: pwsh
-
     # TODO: We could add some script to verify metadata of dll's (selected fields, perhaps)
     # ref. https://superuser.com/questions/381276/what-are-some-nice-command-line-ways-to-inspect-dll-exe-details
     - name: Print metadata of our dll's

.github/workflows/qemu.yml → .github/workflows/reusable_qemu.yml

@@ -29,7 +29,14 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y qemu-system genisoimage qemu-utils \
           libvirt-clients libvirt-daemon-system libvirt-daemon virtinst bridge-utils
-        pip install -r umf/scripts/qemu/requirements.txt
+
+    # Latest distros do not allow global pip installation
+    - name: Install Python requirements in venv
+      run: |
+        python3 -m venv .venv
+        . .venv/bin/activate
+        echo "$PATH" >> $GITHUB_PATH
+        python3 -m pip install -r umf/scripts/qemu/requirements.txt
 
     - name: Add user to kvm group
       run: sudo usermod -a -G kvm,libvirt $USER