add 3 more benchmarks

pbalcer · pbalcer · commit c0a71915b47c · 2024-12-02T13:06:03.000+01:00
This patch implements support for:
 - dl-cifar
 - dl-mnist
 - svm

These are all workloads from Velocity Bench that require oneMKL.
diff --git a/scripts/benchmarks/benches/base.py b/scripts/benchmarks/benches/base.py
@@ -40,19 +40,21 @@ def run_bench(self, command, env_vars, ld_library=[]):
             ld_library=ld_library
         ).stdout.decode()
 
-    def create_data_path(self, name):
-        data_path = os.path.join(self.directory, "data", name)
-
-        if options.rebuild and Path(data_path).exists():
-           shutil.rmtree(data_path)
+    def create_data_path(self, name, skip_data_dir = False):
+        if skip_data_dir:
+            data_path = os.path.join(self.directory, name)
+        else:
+            data_path = os.path.join(self.directory, 'data', name)
+            if options.rebuild and Path(data_path).exists():
+                shutil.rmtree(data_path)
 
         Path(data_path).mkdir(parents=True, exist_ok=True)
 
         return data_path
 
-    def download(self, name, url, file, untar = False):
-        self.data_path = self.create_data_path(name)
-        return download(self.data_path, url, file, True)
+    def download(self, name, url, file, untar = False, unzip = False, skip_data_dir = False):
+        self.data_path = self.create_data_path(name, skip_data_dir)
+        return download(self.data_path, url, file, untar, unzip)
 
     def name(self):
         raise NotImplementedError()
diff --git a/scripts/benchmarks/benches/llamacpp.py b/scripts/benchmarks/benches/llamacpp.py
@@ -6,85 +6,14 @@
 import csv
 import io
 from pathlib import Path
-import re
-import shutil
 from utils.utils import download, git_clone
 from .base import Benchmark, Suite
 from .result import Result
 from utils.utils import run, create_build_path
 from .options import options
+from .oneapi import get_oneapi
 import os
 
-class OneAPI:
-    # random unique number for benchmark oneAPI installation
-    ONEAPI_BENCHMARK_INSTANCE_ID = 98765
-    def __init__(self, directory):
-        self.oneapi_dir = os.path.join(directory, 'oneapi')
-        Path(self.oneapi_dir).mkdir(parents=True, exist_ok=True)
-        # delete if some option is set?
-
-        # can we just hardcode these links?
-        self.install_package('dnnl', 'https://registrationcenter-download.intel.com/akdlm/IRC_NAS/87e117ab-039b-437d-9c80-dcd5c9e675d5/intel-onednn-2025.0.0.862_offline.sh')
-        self.install_package('mkl', 'https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940_offline.sh')
-        return
-
-    def install_package(self, name, url):
-        package_path = os.path.join(self.oneapi_dir, name)
-        if Path(package_path).exists():
-            print(f"{package_path} exists, skipping installing oneAPI package {name}...")
-            return
-
-        package = download(self.oneapi_dir, url, f'package_{name}.sh')
-        try:
-            print(f"installing f{name}")
-            run(f"sh {package} -a -s --eula accept --install-dir {self.oneapi_dir} --instance f{self.ONEAPI_BENCHMARK_INSTANCE_ID}")
-        except:
-            print("oneAPI installation likely exists already")
-            return
-        print(f"f{name} installation complete")
-
-    def package_dir(self, package, dir):
-        return os.path.join(self.oneapi_dir, package, 'latest', dir)
-
-    def package_cmake(self, package):
-        package_lib = self.package_dir(package, 'lib')
-        return os.path.join(package_lib, 'cmake', package)
-
-    def mkl_lib(self):
-        return self.package_dir('mkl', 'lib')
-
-    def mkl_include(self):
-        return self.package_dir('mkl', 'include')
-
-    def mkl_cmake(self):
-        return self.package_cmake('mkl')
-
-    def dnn_lib(self):
-        return self.package_dir('dnnl', 'lib')
-
-    def dnn_include(self):
-        return self.package_dir('dnnl', 'include')
-
-    def dnn_cmake(self):
-        return self.package_cmake('dnnl')
-
-    def tbb_lib(self):
-        return self.package_dir('tbb', 'lib')
-
-    def tbb_cmake(self):
-        return self.package_cmake('tbb')
-
-    def compiler_lib(self):
-        return self.package_dir('compiler', 'lib')
-
-    def ld_libraries(self):
-        return [
-            self.compiler_lib(),
-            self.mkl_lib(),
-            self.tbb_lib(),
-            self.dnn_lib()
-        ]
-
 class LlamaCppBench(Suite):
     def __init__(self, directory):
         if options.sycl is None:
@@ -103,7 +32,7 @@ def setup(self):
 
         self.model = download(self.models_dir, "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi-3-mini-4k-instruct-q4.gguf", "Phi-3-mini-4k-instruct-q4.gguf")
 
-        self.oneapi = OneAPI(self.directory)
+        self.oneapi = get_oneapi()
 
         self.build_path = create_build_path(self.directory, 'llamacpp-build')
 
diff --git a/scripts/benchmarks/benches/oneapi.py b/scripts/benchmarks/benches/oneapi.py
@@ -0,0 +1,86 @@
+# Copyright (C) 2024 Intel Corporation
+# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
+# See LICENSE.TXT
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+from pathlib import Path
+from utils.utils import download, run
+from .options import options
+import os
+
+class OneAPI:
+    # random unique number for benchmark oneAPI installation
+    ONEAPI_BENCHMARK_INSTANCE_ID = 98765
+    def __init__(self):
+        self.oneapi_dir = os.path.join(options.workdir, 'oneapi')
+        Path(self.oneapi_dir).mkdir(parents=True, exist_ok=True)
+        # delete if some option is set?
+
+        # can we just hardcode these links?
+        self.install_package('dnnl', 'https://registrationcenter-download.intel.com/akdlm/IRC_NAS/87e117ab-039b-437d-9c80-dcd5c9e675d5/intel-onednn-2025.0.0.862_offline.sh')
+        self.install_package('mkl', 'https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940_offline.sh')
+        return
+
+    def install_package(self, name, url):
+        package_path = os.path.join(self.oneapi_dir, name)
+        if Path(package_path).exists():
+            print(f"{package_path} exists, skipping installing oneAPI package {name}...")
+            return
+
+        package = download(self.oneapi_dir, url, f'package_{name}.sh')
+        try:
+            print(f"installing f{name}")
+            run(f"sh {package} -a -s --eula accept --install-dir {self.oneapi_dir} --instance f{self.ONEAPI_BENCHMARK_INSTANCE_ID}")
+        except:
+            print("oneAPI installation likely exists already")
+            return
+        print(f"f{name} installation complete")
+
+    def package_dir(self, package, dir):
+        return os.path.join(self.oneapi_dir, package, 'latest', dir)
+
+    def package_cmake(self, package):
+        package_lib = self.package_dir(package, 'lib')
+        return os.path.join(package_lib, 'cmake', package)
+
+    def mkl_lib(self):
+        return self.package_dir('mkl', 'lib')
+
+    def mkl_include(self):
+        return self.package_dir('mkl', 'include')
+
+    def mkl_cmake(self):
+        return self.package_cmake('mkl')
+
+    def dnn_lib(self):
+        return self.package_dir('dnnl', 'lib')
+
+    def dnn_include(self):
+        return self.package_dir('dnnl', 'include')
+
+    def dnn_cmake(self):
+        return self.package_cmake('dnnl')
+
+    def tbb_lib(self):
+        return self.package_dir('tbb', 'lib')
+
+    def tbb_cmake(self):
+        return self.package_cmake('tbb')
+
+    def compiler_lib(self):
+        return self.package_dir('compiler', 'lib')
+
+    def ld_libraries(self):
+        return [
+            self.compiler_lib(),
+            self.mkl_lib(),
+            self.tbb_lib(),
+            self.dnn_lib()
+        ]
+
+oneapi_instance = None
+
+def get_oneapi() -> OneAPI: # oneAPI singleton
+    if not hasattr(get_oneapi, "instance"):
+        get_oneapi.instance = OneAPI()
+    return get_oneapi.instance
diff --git a/scripts/benchmarks/benches/options.py b/scripts/benchmarks/benches/options.py
@@ -8,6 +8,7 @@ class Compare(Enum):
 
 @dataclass
 class Options:
+    workdir: str = None
     sycl: str = None
     ur: str = None
     ur_adapter: str = None
diff --git a/scripts/benchmarks/benches/velocity.py b/scripts/benchmarks/benches/velocity.py
@@ -10,6 +10,9 @@
 from .result import Result
 from utils.utils import run, create_build_path
 from .options import options
+from .oneapi import get_oneapi
+import shutil
+
 import os
 
 class VelocityBench(Suite):
@@ -35,7 +38,10 @@ def benchmarks(self) -> list[Benchmark]:
             CudaSift(self),
             Easywave(self),
             QuickSilver(self),
-            SobelFilter(self)
+            SobelFilter(self),
+            DLCifar(self),
+            DLMnist(self),
+            SVM(self)
         ]
 
 class VelocityBase(Benchmark):
@@ -50,6 +56,12 @@ def __init__(self, name: str, bin_name: str, vb: VelocityBench, unit: str):
     def download_deps(self):
         return
 
+    def extra_cmake_args(self) -> list[str]:
+        return []
+
+    def ld_libraries(self) -> list[str]:
+        return []
+
     def setup(self):
         self.download_deps()
         self.benchmark_bin = os.path.join(self.directory, self.bench_name, self.bin_name)
@@ -62,8 +74,10 @@ def setup(self):
             f"-S {self.code_path}",
             f"-DCMAKE_BUILD_TYPE=Release"
         ]
+        configure_command += self.extra_cmake_args()
+
         run(configure_command, {'CC': 'clang', 'CXX':'clang++'}, add_sycl=True)
-        run(f"cmake --build {build_path} -j", add_sycl=True)
+        run(f"cmake --build {build_path} -j", add_sycl=True, ld_library=self.ld_libraries())
 
     def bin_args(self) -> list[str]:
         return []
@@ -82,7 +96,7 @@ def run(self, env_vars) -> list[Result]:
         ]
         command += self.bin_args()
 
-        result = self.run_bench(command, env_vars)
+        result = self.run_bench(command, env_vars, ld_library=self.ld_libraries())
 
         return [ Result(label=self.name(), value=self.parse_output(result), command=command, env=env_vars, stdout=result, unit=self.unit) ]
 
@@ -136,7 +150,6 @@ def __init__(self, vb: VelocityBench):
 
     def download_deps(self):
         self.download("sobel_filter", "https://github.com/oneapi-src/Velocity-Bench/raw/main/sobel_filter/res/sobel_filter_data.tgz?download=", "sobel_filter_data.tgz", untar=True)
-        return
 
     def name(self):
         return "Velocity-Bench Sobel Filter"
@@ -228,7 +241,6 @@ def get_last_elapsed_time(self, log_file_path) -> float:
     def parse_output(self, stdout: str) -> float:
         return self.get_last_elapsed_time(os.path.join(options.benchmark_cwd, "easywave.log"))
 
-
 class CudaSift(VelocityBase):
     def __init__(self, vb: VelocityBench):
         super().__init__("cudaSift", "cudaSift", vb, "ms")
@@ -248,3 +260,103 @@ def parse_output(self, stdout: str) -> float:
             return float(match.group(1))
         else:
             raise ValueError("Failed to parse benchmark output.")
+
+class DLCifar(VelocityBase):
+    def __init__(self, vb: VelocityBench):
+        self.oneapi = get_oneapi()
+        super().__init__("dl-cifar", "dl-cifar_sycl", vb, "s")
+
+    def ld_libraries(self):
+        return self.oneapi.ld_libraries()
+
+    def download_deps(self):
+        # TODO: dl-cifar hardcodes the path to this dataset as "../../datasets/cifar-10-binary"...
+        self.download("datasets", "https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz", "cifar-10-binary.tar.gz", untar=True, skip_data_dir=True)
+        return
+
+    def extra_cmake_args(self):
+        return [
+            f"-DCMAKE_CXX_FLAGS=-O3 -fsycl -ffast-math -I{self.oneapi.dnn_include()} -I{self.oneapi.mkl_include()} -L{self.oneapi.dnn_lib()} -L{self.oneapi.mkl_lib()}"
+        ]
+
+    def name(self):
+        return "Velocity-Bench dl-cifar"
+
+    def parse_output(self, stdout: str) -> float:
+        match = re.search(r'dl-cifar - total time for whole calculation: (\d+\.\d+) s', stdout)
+        if match:
+            return float(match.group(1))
+        else:
+            raise ValueError("Failed to parse benchmark output.")
+
+class DLMnist(VelocityBase):
+    def __init__(self, vb: VelocityBench):
+        self.oneapi = get_oneapi()
+        super().__init__("dl-mnist", "dl-mnist-sycl", vb, "s")
+
+    def ld_libraries(self):
+        return self.oneapi.ld_libraries()
+
+    def download_deps(self):
+        # TODO: dl-mnist hardcodes the path to this dataset as "../../datasets/"...
+        self.download("datasets", "https://raw.githubusercontent.com/fgnt/mnist/master/train-images-idx3-ubyte.gz", "train-images.idx3-ubyte.gz", unzip=True, skip_data_dir=True)
+        self.download("datasets", "https://raw.githubusercontent.com/fgnt/mnist/master/train-labels-idx1-ubyte.gz", "train-labels.idx1-ubyte.gz", unzip=True, skip_data_dir=True)
+        self.download("datasets", "https://raw.githubusercontent.com/fgnt/mnist/master/t10k-images-idx3-ubyte.gz", "t10k-images.idx3-ubyte.gz", unzip=True, skip_data_dir=True)
+        self.download("datasets", "https://raw.githubusercontent.com/fgnt/mnist/master/t10k-labels-idx1-ubyte.gz", "t10k-labels.idx1-ubyte.gz", unzip=True, skip_data_dir=True)
+
+    def extra_cmake_args(self):
+        return [
+            f"-DCMAKE_CXX_FLAGS=-O3 -fsycl -ffast-math -I{self.oneapi.dnn_include()} -I{self.oneapi.mkl_include()} -L{self.oneapi.dnn_lib()} -L{self.oneapi.mkl_lib()}"
+        ]
+
+    def name(self):
+        return "Velocity-Bench dl-mnist"
+
+    def bin_args(self):
+        return [
+            "-conv_algo", "ONEDNN_AUTO"
+        ]
+
+    # TODO: This shouldn't be required.
+    # The application crashes with a segfault without it.
+    def extra_env_vars(self):
+        return {
+            "NEOReadDebugKeys":"1",
+            "DisableScratchPages":"0",
+        }
+
+    def parse_output(self, stdout: str) -> float:
+        match = re.search(r'dl-mnist - total time for whole calculation: (\d+\.\d+) s', stdout)
+        if match:
+            return float(match.group(1))
+        else:
+            raise ValueError("Failed to parse benchmark output.")
+
+class SVM(VelocityBase):
+    def __init__(self, vb: VelocityBench):
+        self.oneapi = get_oneapi()
+        super().__init__("svm", "svm_sycl", vb, "s")
+
+    def ld_libraries(self):
+        return self.oneapi.ld_libraries()
+
+    def extra_cmake_args(self):
+        return [
+            f"-DCMAKE_CXX_FLAGS=-O3 -fsycl -ffast-math -I{self.oneapi.dnn_include()} -I{self.oneapi.mkl_include()} -L{self.oneapi.dnn_lib()} -L{self.oneapi.mkl_lib()}"
+        ]
+
+    def name(self):
+        return "Velocity-Bench svm"
+
+    def bin_args(self):
+        return [
+            f"{self.code_path}/a9a",
+            f"{self.code_path}/a.m",
+        ]
+
+    def parse_output(self, stdout: str) -> float:
+        match = re.search(r'Total      elapsed time : (\d+\.\d+) s', stdout)
+        if match:
+            return float(match.group(1))
+        else:
+            raise ValueError("Failed to parse benchmark output.")
diff --git a/scripts/benchmarks/main.py b/scripts/benchmarks/main.py
diff --git a/scripts/benchmarks/utils/utils.py b/scripts/benchmarks/utils/utils.py
