[benchmarks] use manifest to build compute-runtime dependencies

Instead of hardcoding compute runtime dependencies, scripts will now
fetch its manifest file to see what are the correct versions of
dependencies to build them.
This patch also adds an option to build IGC from source.

Author: pbalcer

.github/workflows/benchmarks-reusable.yml

@@ -205,6 +205,7 @@ jobs:
         --umf ${{ github.workspace }}/umf_build
         --adapter ${{ matrix.adapter.str_name }}
         --compute-runtime ${{ inputs.compute_runtime_commit }}
+        --build-igc
         ${{ inputs.upload_report && '--output-html' || '' }}
         ${{ inputs.bench_script_params }}
 

scripts/benchmarks/README.md

@@ -9,11 +9,11 @@ Scripts for running performance tests on SYCL and Unified Runtime.
 
 ## Running
 
-`$ ./main.py ~/benchmarks_workdir/ ~/llvm/build/ ~/ur adapter_name`
+`$ ./main.py ~/benchmarks_workdir/ --sycl ~/llvm/build/ --ur ~/ur --adapter adapter_name`
 
 This will download and build everything in `~/benchmarks_workdir/` using the compiler in `~/llvm/build/`, UR source from `~/ur` and then run the benchmarks for `adapter_name` adapter. The results will be stored in `benchmark_results.md`.
 
-The scripts will try to reuse the files stored in `~/benchmarks_workdir/`, but the benchmarks will be rebuilt every time. To avoid that, use `-no-rebuild` option.
+The scripts will try to reuse the files stored in `~/benchmarks_workdir/`, but the benchmarks will be rebuilt every time. To avoid that, use `--no-rebuild` option.
 
 ## Running in CI
 
@@ -47,7 +47,27 @@ are stored [here](https://oneapi-src.github.io/unified-runtime/benchmark_results
 ### Python
 
 dataclasses-json==0.6.7
+PyYAML==6.0.2
+Mako==1.3.0
 
 ### System
 
-libopencv-dev
+Sobel Filter benchmark:
+
+`$ sudo apt-get install libopencv-dev`
+
+### Compute-runtime and IGC
+
+The scripts have an option to build compute-runtime and all related components from source:
+
+`$ ./main.py ~/benchmarks_workdir/ --compute-runtime [tag] --build-igc`
+
+For this to work, the system needs to have the appropriate dependencies installed.
+
+compute-runtime (Ubuntu):
+
+`$ sudo apt-get install cmake g++ git pkg-config`
+
+IGC (Ubuntu):
+
+`$ sudo apt-get install flex bison libz-dev cmake libc6 libstdc++6 python3-pip`

scripts/benchmarks/main.py

@@ -244,7 +244,7 @@ def validate_and_parse_env_args(env_args):
     parser.add_argument('--ur', type=str, help='UR install prefix path', default=None)
     parser.add_argument('--umf', type=str, help='UMF install prefix path', default=None)
     parser.add_argument('--adapter', type=str, help='Options to build the Unified Runtime as part of the benchmark', default="level_zero")
-    parser.add_argument("--no-rebuild", help='Rebuild the benchmarks from scratch.', action="store_true")
+    parser.add_argument("--no-rebuild", help='Do not rebuild the benchmarks from scratch.', action="store_true")
     parser.add_argument("--env", type=str, help='Use env variable for a benchmark run.', action="append", default=[])
     parser.add_argument("--save", type=str, help='Save the results for comparison under a specified name.')
     parser.add_argument("--compare", type=str, help='Compare results against previously saved data.', action="append", default=["baseline"])
@@ -262,6 +262,7 @@ def validate_and_parse_env_args(env_args):
     parser.add_argument("--dry-run", help='Do not run any actual benchmarks', action="store_true", default=False)
     parser.add_argument("--compute-runtime", nargs='?', const=options.compute_runtime_tag, help="Fetch and build compute runtime")
     parser.add_argument("--iterations-stddev", type=int, help="Max number of iterations of the loop calculating stddev after completed benchmark runs", default=options.iterations_stddev)
+    parser.add_argument("--build-igc", help="Build IGC from source instead of using the OS-installed version", action="store_true", default=options.build_igc)
 
     args = parser.parse_args()
     additional_env_vars = validate_and_parse_env_args(args.env)
@@ -283,6 +284,10 @@ def validate_and_parse_env_args(env_args):
     options.dry_run = args.dry_run
     options.umf = args.umf
     options.iterations_stddev = args.iterations_stddev
+    options.build_igc = args.build_igc
+
+    if args.build_igc and args.compute_runtime is None:
+        parser.error("--build-igc requires --compute-runtime to be set")
     if args.compute_runtime is not None:
         options.build_compute_runtime = True
         options.compute_runtime_tag = args.compute_runtime

scripts/benchmarks/options.py

@@ -30,7 +30,8 @@ class Options:
     build_compute_runtime: bool = False
     extra_ld_libraries: list[str] = field(default_factory=list)
     extra_env_vars: dict = field(default_factory=dict)
-    compute_runtime_tag: str = 'c1ed0334d65f6ce86d7273fe4137d1d4a5b5fa7c'
+    compute_runtime_tag: str = '24.52.32224.8'
+    build_igc: bool = False
 
 options = Options()
 

scripts/benchmarks/utils/compute_runtime.py

@@ -5,6 +5,7 @@
 
 import os
 import re
+import yaml
 
 from pathlib import Path
 from .utils import *
@@ -21,9 +22,7 @@ def replace_in_file(file_path, search_pattern, replacement):
 
 class ComputeRuntime:
     def __init__(self):
-        self.gmmlib = self.build_gmmlib()
-        self.level_zero = self.build_level_zero()
-        self.compute_runtime = self.build_compute_runtime(self.gmmlib, self.level_zero)
+        self.compute_runtime = self.build_compute_runtime()
 
         return
 
@@ -32,14 +31,15 @@ def ld_libraries(self) -> list[str]:
             os.path.join(self.gmmlib, "lib64"),
             os.path.join(self.level_zero, "lib64"),
             os.path.join(self.compute_runtime, "bin"),
+            os.path.join(self.igc, "lib"),
         ]
 
     def env_vars(self) -> dict:
         return {"ZE_ENABLE_ALT_DRIVERS" : os.path.join(self.compute_runtime, "bin", "libze_intel_gpu.so"),
                 "OCL_ICD_FILENAMES" : os.path.join(self.compute_runtime, "bin", "libigdrcl.so")}
 
-    def build_gmmlib(self):
-        self.gmmlib_repo = git_clone(options.workdir, "gmmlib-repo", "https://github.com/intel/gmmlib.git", "9104c2090158b35d440afdf8ec940d89cc7b3c6a")
+    def build_gmmlib(self, repo, commit):
+        self.gmmlib_repo = git_clone(options.workdir, "gmmlib-repo", repo, commit)
         self.gmmlib_build = os.path.join(options.workdir, "gmmlib-build")
         self.gmmlib_install = os.path.join(options.workdir, "gmmlib-install")
         configure_command = [
@@ -54,8 +54,8 @@ def build_gmmlib(self):
         run(f"cmake --install {self.gmmlib_build}")
         return self.gmmlib_install
 
-    def build_level_zero(self):
-        self.level_zero_repo = git_clone(options.workdir, "level-zero-repo", "https://github.com/oneapi-src/level-zero.git", "3969f34c16a843b943b948f8fe7081ef87deb369")
+    def build_level_zero(self, repo, commit):
+        self.level_zero_repo = git_clone(options.workdir, "level-zero-repo", repo, commit)
         self.level_zero_build = os.path.join(options.workdir, "level-zero-build")
         self.level_zero_install = os.path.join(options.workdir, "level-zero-install")
 
@@ -75,10 +75,64 @@ def build_level_zero(self):
         run(f"cmake --install {self.level_zero_build}")
         return self.level_zero_install
 
-    def build_compute_runtime(self, gmmlib, level_zero):
+
+    def build_igc(self, repo, commit):
+        self.igc_repo = git_clone(options.workdir, "igc", repo, commit)
+        self.vc_intr = git_clone(options.workdir, "vc-intrinsics", "https://github.com/intel/vc-intrinsics", "facb2076a2ce6cd6527c1e16570ba0fbaa2f1dba")
+        self.llvm_project = git_clone(options.workdir, "llvm-project", "https://github.com/llvm/llvm-project", "llvmorg-14.0.5")
+        llvm_projects = os.path.join(self.llvm_project, "llvm", "projects")
+        self.ocl = git_clone(llvm_projects, "opencl-clang", "https://github.com/intel/opencl-clang", "ocl-open-140")
+        self.translator = git_clone(llvm_projects, "llvm-spirv", "https://github.com/KhronosGroup/SPIRV-LLVM-Translator", "llvm_release_140")
+        self.spirv_tools = git_clone(options.workdir, "SPIRV-Tools", "https://github.com/KhronosGroup/SPIRV-Tools.git", "173fe3c60a8d9c7d35d7842ae267bb9df267a127")
+        self.spirv_headers = git_clone(options.workdir, "SPIRV-Headers", "https://github.com/KhronosGroup/SPIRV-Headers.git", "2b2e05e088841c63c0b6fd4c9fb380d8688738d3")
+
+        self.igc_build = os.path.join(options.workdir, "igc-build")
+        self.igc_install = os.path.join(options.workdir, "igc-install")
+        configure_command = [
+            "cmake",
+            f"-B {self.igc_build}",
+            f"-S {self.igc_repo}",
+            f"-DCMAKE_INSTALL_PREFIX={self.igc_install}",
+            f"-DCMAKE_BUILD_TYPE=Release",
+        ]
+        run(configure_command)
+
+        # set timeout to 30min. IGC takes A LONG time to build if building from scratch.
+        run(f"cmake --build {self.igc_build} -j", timeout=600 * 3)
+        # cmake --install doesn't work...
+        run("make install", cwd=self.igc_build)
+        return self.igc_install
+
+    def read_manifest(self, manifest_path):
+        with open(manifest_path, 'r') as file:
+            manifest = yaml.safe_load(file)
+        return manifest
+
+    def get_repo_info(self, manifest, component_name):
+        component = manifest['components'].get(component_name)
+        if component:
+            repo = component.get('repository')
+            revision = component.get('revision')
+            return repo, revision
+        return None, None
+
+    def build_compute_runtime(self):
         self.compute_runtime_repo = git_clone(options.workdir, "compute-runtime-repo", "https://github.com/intel/compute-runtime.git", options.compute_runtime_tag)
         self.compute_runtime_build = os.path.join(options.workdir, "compute-runtime-build")
 
+        manifest_path = os.path.join(self.compute_runtime_repo, "manifests", "manifest.yml")
+        manifest = self.read_manifest(manifest_path)
+
+        level_zero_repo, level_zero_commit = self.get_repo_info(manifest, 'level_zero')
+        self.level_zero = self.build_level_zero(level_zero_repo, level_zero_commit)
+
+        gmmlib_repo, gmmlib_commit = self.get_repo_info(manifest, 'gmmlib')
+        self.gmmlib = self.build_gmmlib(gmmlib_repo, gmmlib_commit)
+
+        if options.build_igc:
+            igc_repo, igc_commit = self.get_repo_info(manifest, 'igc')
+            self.igc = self.build_igc(igc_repo, igc_commit)
+
         cmakelists_path = os.path.join(self.compute_runtime_repo, "level_zero", "cmake", "FindLevelZero.cmake")
         # specifying custom L0 is problematic...
         replace_in_file(cmakelists_path, r'(\$\{LEVEL_ZERO_ROOT\}\s*)', r'\1NO_DEFAULT_PATH\n')
@@ -95,9 +149,12 @@ def build_compute_runtime(self, gmmlib, level_zero):
             "-DNEO_ENABLE_i915_PRELIM_DETECTION=1",
             "-DNEO_ENABLE_I915_PRELIM_DETECTION=1",
             "-DNEO_SKIP_UNIT_TESTS=1",
-            f"-DGMM_DIR={gmmlib}",
-            f"-DLEVEL_ZERO_ROOT={level_zero}"
+            f"-DGMM_DIR={self.gmmlib}",
+            f"-DLEVEL_ZERO_ROOT={self.level_zero}"
         ]
+        if options.build_igc:
+            configure_command.append(f"-DIGC_DIR={self.igc}")
+
         run(configure_command)
         run(f"cmake --build {self.compute_runtime_build} -j")
         return self.compute_runtime_build

scripts/benchmarks/utils/utils.py

@@ -13,7 +13,7 @@
 from options import options
 from pathlib import Path
 
-def run(command, env_vars={}, cwd=None, add_sycl=False, ld_library=[]):
+def run(command, env_vars={}, cwd=None, add_sycl=False, ld_library=[], timeout=options.timeout):
     try:
         if isinstance(command, str):
             command = command.split()
@@ -32,7 +32,7 @@ def run(command, env_vars={}, cwd=None, add_sycl=False, ld_library=[]):
 
         env.update(env_vars)
 
-        result = subprocess.run(command, cwd=cwd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, timeout=options.timeout) # nosec B603
+        result = subprocess.run(command, cwd=cwd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, timeout=timeout) # nosec B603
 
         if options.verbose:
             print(result.stdout.decode())

third_party/benchmark_requirements.txt

@@ -41,3 +41,4 @@ sphinxcontrib-websupport==1.2.4
 sphinx-rtd-theme==1.0.0
 urllib3==2.2.2
 dataclasses-json==0.6.7
+PyYAML==6.0.1