[CI] Factor out igc dev related code (#13441)

jsji · web-flow · commit 2336d028e2d6 · 2024-04-17T15:15:37.000-07:00
Per discussion with @aelovikov-intel ,factor out all igc dev related code to new workflows to avoid necessary interaction with other docker images.
diff --git a/.github/workflows/sycl-containers-igc-dev.yaml b/.github/workflows/sycl-containers-igc-dev.yaml
@@ -0,0 +1,52 @@
+name: IGC DEV CI Containers
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - sycl
+    paths:
+      - 'devops/actions/build_container/**'
+      - 'devops/dependencies-igc-dev.json'
+      - '.github/workflows/sycl-containers-igc-dev.yaml'
+  pull_request:
+    paths:
+      - 'devops/actions/build_container/**'
+      - 'devops/dependencies-igc-dev.json'
+      - '.github/workflows/sycl-containers-igc-dev.yaml'
+
+permissions: read-all
+
+jobs:
+  build_and_push_images:
+    if: github.repository == 'intel/llvm'
+    name: Build and Push IGC Dev Docker Images
+    runs-on: ubuntu-22.04
+    permissions:
+      packages: write
+    strategy:
+      matrix:
+        include:
+          - name: Intel Drivers Ubuntu 22.04 Docker image with dev IGC
+            dockerfile: ubuntu2204_intel_drivers_igc_dev
+            imagefile: ubuntu2204_intel_drivers
+            tag: devigc
+            build_args: |
+                  "use_latest=false"
+                  "use_igc_dev=true"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - name: Build and Push Container
+        uses: ./devops/actions/build_container
+        with:
+          push: ${{ github.event_name != 'pull_request' }}
+          file: ${{ matrix.dockerfile }}
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          tags: |
+            ghcr.io/${{ github.repository }}/${{ matrix.imagefile }}:${{ matrix.tag }}-${{ github.sha }}
+            ghcr.io/${{ github.repository }}/${{ matrix.imagefile }}:${{ matrix.tag }}
+          build-args: ${{ matrix.build_args }}
+
diff --git a/.github/workflows/sycl-containers.yaml b/.github/workflows/sycl-containers.yaml
@@ -8,13 +8,15 @@ on:
     branches:
       - sycl
     paths:
+      - 'devops/actions/build_container/**'
       - 'devops/containers/**'
       - 'devops/dependencies.json'
       - 'devops/scripts/install_drivers.sh'
       - 'devops/scripts/install_build_tools.sh'
       - '.github/workflows/sycl-containers.yaml'
   pull_request:
     paths:
+      - 'devops/actions/build_container/**'
       - 'devops/containers/**'
       - 'devops/dependencies.json'
       - 'devops/scripts/install_drivers.sh'
@@ -45,12 +47,6 @@ jobs:
             file: ubuntu2204_intel_drivers
             tag: latest
             build_args: "use_latest=false"
-          - name: Intel Drivers Ubuntu 22.04 Docker image with dev IGC
-            file: ubuntu2204_intel_drivers
-            tag: devigc
-            build_args: |
-                  "use_latest=false"
-                  "use_igc_dev=true"
           - name: Intel Drivers (unstable) Ubuntu 22.04 Docker image
             file: ubuntu2204_intel_drivers
             tag: unstable
diff --git a/.github/workflows/sycl-update-igc-dev-driver.yml b/.github/workflows/sycl-update-igc-dev-driver.yml
@@ -0,0 +1,35 @@
+name: Update IGC dev driver
+
+on:
+  schedule:
+    - cron:  '0 3 * * 1,4'
+  workflow_dispatch:
+
+permissions: read-all
+
+jobs:
+  update_driver_linux:
+    permissions:
+      contents: write  # for Git to git push
+    runs-on: ubuntu-20.04
+    if: github.repository == 'intel/llvm'
+    steps:
+    - uses: actions/checkout@v4
+    - name: Update dependencies file
+      run: |
+        version="$(python3 devops/scripts/update_drivers.py linux --igc-dev-only)"
+        echo 'NEW_DRIVER_VERSION='$version >> $GITHUB_ENV
+    - name: Create Pull Request
+      env:
+        BRANCH: ci/update_gpu_driver-linux-${{ env.NEW_DRIVER_VERSION }}
+        GITHUB_TOKEN: ${{ secrets.LLVM_MAIN_SYNC_BBSYCL_TOKEN }}
+      run: |
+        cd $GITHUB_WORKSPACE
+        git config --global user.name "GitHub Actions"
+        git config --global user.email "actions@github.com"
+        git checkout -B $BRANCH
+        git add -u
+        git commit -m "[GHA] Uplift Linux IGC Dev RT version to $NEW_DRIVER_VERSION" || exit 0   # exit if commit is empty
+        git show
+        git push https://$GITHUB_TOKEN@github.com/${{ github.repository }} ${BRANCH}
+        gh pr create --head $BRANCH --title "[GHA] Uplift Linux IGC Dev RT version to $NEW_DRIVER_VERSION" --body "Scheduled igc dev drivers uplift"
diff --git a/devops/containers/ubuntu2204_intel_drivers.Dockerfile b/devops/containers/ubuntu2204_intel_drivers.Dockerfile
@@ -6,10 +6,8 @@ FROM $base_image:$base_tag
 ENV DEBIAN_FRONTEND=noninteractive
 
 ARG use_latest=true
-ARG use_igc_dev=false
 
-RUN apt update && apt install -yqq wget \
-    && if [ "$use_igc_dev" = "true" ]; then apt-get install -yqq libllvm14; fi
+RUN apt update && apt install -yqq wget
 
 COPY scripts/get_release.py /
 COPY scripts/install_drivers.sh /
@@ -22,9 +20,6 @@ RUN --mount=type=secret,id=github_token \
       install_driver_opt=" --use-latest"; \
     else \
       install_driver_opt=" dependencies.json"; \
-      if [ "$use_igc_dev" = "true" ]; then \
-        install_driver_opt="$install_driver_opt --use-dev-igc"; \
-      fi; \
     fi && \
     GITHUB_TOKEN=$(cat /run/secrets/github_token) /install_drivers.sh $install_driver_opt --all
 
diff --git a/devops/containers/ubuntu2204_intel_drivers_igc_dev.Dockerfile b/devops/containers/ubuntu2204_intel_drivers_igc_dev.Dockerfile
@@ -0,0 +1,24 @@
+ARG base_tag=latest
+ARG base_image=ghcr.io/intel/llvm/ubuntu2204_base
+
+FROM $base_image:$base_tag
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt update && apt install -yqq libllvm14
+
+COPY scripts/get_release.py /
+COPY scripts/install_drivers.sh /
+COPY dependencies.json /
+COPY dependencies-igc-dev.json /
+
+RUN mkdir /runtimes
+ENV INSTALL_LOCATION=/runtimes
+RUN --mount=type=secret,id=github_token \
+    install_driver_opt="dependencies.json dependencies-igc-dev.json --use-dev-igc"; \
+    GITHUB_TOKEN=$(cat /run/secrets/github_token) /install_drivers.sh $install_driver_opt --all
+
+COPY scripts/drivers_entrypoint.sh /drivers_entrypoint.sh
+
+ENTRYPOINT ["/bin/bash", "/drivers_entrypoint.sh"]
+
diff --git a/devops/dependencies-igc-dev.json b/devops/dependencies-igc-dev.json
@@ -0,0 +1,11 @@
+{
+  "linux": {
+    "igc_dev": {
+      "github_tag": "igc-dev-498324a",
+      "version": "498324a",
+      "updated_at": "2024-04-17T13:44:27Z",
+      "url": "https://api.github.com/repos/intel/intel-graphics-compiler/actions/artifacts/1422168296/zip",
+      "root": "{DEPS_ROOT}/opencl/runtime/linux/oclgpu"
+    }
+  }
+}
diff --git a/devops/dependencies.json b/devops/dependencies.json
@@ -12,13 +12,6 @@
       "url": "https://github.com/intel/intel-graphics-compiler/releases/tag/igc-1.0.15985.7",
       "root": "{DEPS_ROOT}/opencl/runtime/linux/oclgpu"
     },
-    "igc_dev": {
-      "github_tag": "igc-dev-db4de5f",
-      "version": "db4de5f",
-      "updated_at": "2024-04-12T11:06:26Z",
-      "url": "https://api.github.com/repos/intel/intel-graphics-compiler/actions/artifacts/1409219375/zip",
-      "root": "{DEPS_ROOT}/opencl/runtime/linux/oclgpu"
-    },
     "cm": {
       "github_tag": "cmclang-1.0.144",
       "version": "1.0.144",
diff --git a/devops/scripts/install_drivers.sh b/devops/scripts/install_drivers.sh
@@ -8,14 +8,17 @@ if [ -f "$1" ]; then
     CONFIG_FILE=$1
     CR_TAG=$(jq -r '.linux.compute_runtime.github_tag' $CONFIG_FILE)
     IGC_TAG=$(jq -r '.linux.igc.github_tag' $CONFIG_FILE)
-    IGC_DEV_TAG=$(jq -r '.linux.igc_dev.github_tag' $CONFIG_FILE)
-    IGC_DEV_VER=$(jq -r '.linux.igc_dev.version' $CONFIG_FILE)
-    IGC_DEV_URL=$(jq -r '.linux.igc_dev.url' $CONFIG_FILE)
     CM_TAG=$(jq -r '.linux.cm.github_tag' $CONFIG_FILE)
     L0_TAG=$(jq -r '.linux.level_zero.github_tag' $CONFIG_FILE)
     TBB_TAG=$(jq -r '.linux.tbb.github_tag' $CONFIG_FILE)
     FPGA_TAG=$(jq -r '.linux.fpgaemu.github_tag' $CONFIG_FILE)
     CPU_TAG=$(jq -r '.linux.oclcpu.github_tag' $CONFIG_FILE)
+    if [[ "$*" == *"--use-dev-igc"* ]]; then
+       CONFIG_FILE_IGC_DEV=$2
+       IGC_DEV_TAG=$(jq -r '.linux.igc_dev.github_tag' $CONFIG_FILE_IGC_DEV)
+       IGC_DEV_VER=$(jq -r '.linux.igc_dev.version' $CONFIG_FILE_IGC_DEV)
+       IGC_DEV_URL=$(jq -r '.linux.igc_dev.url' $CONFIG_FILE_IGC_DEV)
+    fi
 elif [[ "$*" == *"--use-latest"* ]]; then
     CR_TAG=latest
     IGC_TAG=latest
diff --git a/devops/scripts/update_drivers.py b/devops/scripts/update_drivers.py
@@ -3,6 +3,7 @@
 import sys
 import os
 import re
+import argparse
 
 
 def get_latest_release(repo):
@@ -28,7 +29,19 @@ def get_artifacts_download_url(repo, name):
     return json.loads(artifacts)["artifacts"][0]["archive_download_url"]
 
 
-def uplift_linux_igfx_driver(config, platform_tag):
+def uplift_linux_igfx_driver(config, platform_tag, igc_dev_only):
+
+    if igc_dev_only:
+        igc_dev = get_latest_workflow_runs("intel/intel-graphics-compiler", "build-IGC")
+        igcdevver = igc_dev["head_sha"][:7]
+        config[platform_tag]["igc_dev"]["github_tag"] = "igc-dev-" + igcdevver
+        config[platform_tag]["igc_dev"]["version"] = igcdevver
+        config[platform_tag]["igc_dev"]["updated_at"] = igc_dev["updated_at"]
+        config[platform_tag]["igc_dev"]["url"] = get_artifacts_download_url(
+            "intel/intel-graphics-compiler", "IGC_Ubuntu22.04_llvm14_clang-" + igcdevver
+        )
+        return config
+
     compute_runtime = get_latest_release('intel/compute-runtime')
 
     config[platform_tag]['compute_runtime']['github_tag'] = compute_runtime['tag_name']
@@ -46,15 +59,6 @@ def uplift_linux_igfx_driver(config, platform_tag):
                 config[platform_tag]['igc']['url'] = 'https://github.com/intel/intel-graphics-compiler/releases/tag/igc-' + ver
                 break
 
-    igc_dev = get_latest_workflow_runs("intel/intel-graphics-compiler", "build-IGC")
-    igcdevver = igc_dev["head_sha"][:7]
-    config[platform_tag]["igc_dev"]["github_tag"] = "igc-dev-" + igcdevver
-    config[platform_tag]["igc_dev"]["version"] = igcdevver
-    config[platform_tag]["igc_dev"]["updated_at"] = igc_dev["updated_at"]
-    config[platform_tag]["igc_dev"]["url"] = get_artifacts_download_url(
-        "intel/intel-graphics-compiler", "IGC_Ubuntu22.04_llvm14_clang-" + igcdevver
-    )
-
     cm = get_latest_release('intel/cm-compiler')
     config[platform_tag]['cm']['github_tag'] = cm['tag_name']
     config[platform_tag]['cm']['version'] = cm['tag_name'].replace('cmclang-', '')
@@ -68,22 +72,30 @@ def uplift_linux_igfx_driver(config, platform_tag):
     return config
 
 
-def main(platform_tag):
+def main(platform_tag, igc_dev_only):
     script = os.path.dirname(os.path.realpath(__file__))
     config_name = os.path.join(script, '..', 'dependencies.json')
+    if igc_dev_only:
+        config_name = os.path.join(script, "..", "dependencies-igc-dev.json")
     config = {}
 
     with open(config_name, "r") as f:
         config = json.loads(f.read())
-        config = uplift_linux_igfx_driver(config, platform_tag)
+        config = uplift_linux_igfx_driver(config, platform_tag, igc_dev_only)
 
     with open(config_name, "w") as f:
         json.dump(config, f, indent=2)
         f.write('\n')
 
+    if igc_dev_only:
+        return config[platform_tag]["igc_dev"]["github_tag"]
+
     return config[platform_tag]['compute_runtime']['version']
 
 
 if __name__ == '__main__':
-    platform_tag = sys.argv[1] if len(sys.argv) > 1 else "ERROR_PLATFORM"
-    sys.stdout.write(main(platform_tag) + '\n')
+    parser = argparse.ArgumentParser()
+    parser.add_argument("platform_tag")
+    parser.add_argument("--igc-dev-only", action="store_true")
+    args = parser.parse_args()
+    sys.stdout.write(main(args.platform_tag, args.igc_dev_only) + "\n")
