[UR][Benchmarks] GROMACS/Grappa benchmarks added to the suite (#17934)

Gromacs package and Grappa benchmarking datasets added to the benchmark
suite

Author: mateuszpn

devops/scripts/benchmarks/.gitignore

@@ -0,0 +1 @@
+/html/data.js

devops/scripts/benchmarks/benches/base.py

@@ -6,6 +6,7 @@
 from dataclasses import dataclass
 import os
 import shutil
+import subprocess
 from pathlib import Path
 from utils.result import BenchmarkMetadata, BenchmarkTag, Result
 from options import options
@@ -68,7 +69,9 @@ def get_adapter_full_path():
             False
         ), f"could not find adapter file {adapter_path} (and in similar lib paths)"
 
-    def run_bench(self, command, env_vars, ld_library=[], add_sycl=True):
+    def run_bench(
+        self, command, env_vars, ld_library=[], add_sycl=True, use_stdout=True
+    ):
         env_vars = env_vars.copy()
         if options.ur is not None:
             env_vars.update(
@@ -80,13 +83,18 @@ def run_bench(self, command, env_vars, ld_library=[], add_sycl=True):
         ld_libraries = options.extra_ld_libraries.copy()
         ld_libraries.extend(ld_library)
 
-        return run(
+        result = run(
             command=command,
             env_vars=env_vars,
             add_sycl=add_sycl,
             cwd=options.benchmark_cwd,
             ld_library=ld_libraries,
-        ).stdout.decode()
+        )
+
+        if use_stdout:
+            return result.stdout.decode()
+        else:
+            return result.stderr.decode()
 
     def create_data_path(self, name, skip_data_dir=False):
         if skip_data_dir:

devops/scripts/benchmarks/benches/gromacs.py

@@ -0,0 +1,268 @@
+# Copyright (C) 2025 Intel Corporation
+# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
+# See LICENSE.TXT
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+import os
+import subprocess
+from pathlib import Path
+from .base import Suite, Benchmark
+from options import options
+from utils.utils import git_clone, download, run, create_build_path
+from utils.result import Result
+import re
+
+
+class GromacsBench(Suite):
+
+    def git_url(self):
+        return "https://gitlab.com/gromacs/gromacs.git"
+
+    def git_tag(self):
+        return "v2025.1"
+
+    def grappa_url(self):
+        return "https://zenodo.org/record/11234002/files/grappa-1.5k-6.1M_rc0.9.tar.gz"
+
+    def grappa_file(self):
+        return Path(os.path.basename(self.grappa_url()))
+
+    def __init__(self, directory):
+        self.directory = Path(directory).resolve()
+        model_path = str(self.directory / self.grappa_file()).replace(".tar.gz", "")
+        self.grappa_dir = Path(model_path)
+        build_path = create_build_path(self.directory, "gromacs-build")
+        self.gromacs_build_path = Path(build_path)
+        self.gromacs_src = self.directory / "gromacs-repo"
+
+    def name(self):
+        return "Gromacs Bench"
+
+    def benchmarks(self) -> list[Benchmark]:
+        return [
+            GromacsBenchmark(self, "0006", "pme", "graphs"),
+            GromacsBenchmark(self, "0006", "pme", "eager"),
+            GromacsBenchmark(self, "0006", "rf", "graphs"),
+            GromacsBenchmark(self, "0006", "rf", "eager"),
+            # some people may need it
+            # GromacsBenchmark(self, "0192", "pme", "eager"),
+            # GromacsBenchmark(self, "0192", "rf", "eager"),
+        ]
+
+    def setup(self):
+        self.gromacs_src = git_clone(
+            self.directory,
+            "gromacs-repo",
+            self.git_url(),
+            self.git_tag(),
+        )
+
+        # TODO: Detect the GPU architecture and set the appropriate flags
+
+        # Build GROMACS
+        run(
+            [
+                "cmake",
+                f"-S {str(self.directory)}/gromacs-repo",
+                f"-B {self.gromacs_build_path}",
+                f"-DCMAKE_BUILD_TYPE=Release",
+                f"-DCMAKE_CXX_COMPILER=clang++",
+                f"-DCMAKE_C_COMPILER=clang",
+                f"-DGMX_GPU=SYCL",
+                f"-DGMX_SYCL_ENABLE_GRAPHS=ON",
+                f"-DGMX_SYCL_ENABLE_EXPERIMENTAL_SUBMIT_API=ON",
+                f"-DGMX_FFT_LIBRARY=MKL",
+                f"-DGMX_GPU_FFT_LIBRARY=MKL",
+                f"-DGMX_GPU_NB_CLUSTER_SIZE=8",
+                f"-DGMX_GPU_NB_NUM_CLUSTER_PER_CELL_X=1",
+                f"-DGMX_OPENMP=OFF",
+            ],
+            add_sycl=True,
+        )
+        run(
+            f"cmake --build {self.gromacs_build_path} -j {options.build_jobs}",
+            add_sycl=True,
+        )
+        download(
+            self.directory,
+            self.grappa_url(),
+            self.directory / self.grappa_file(),
+            checksum="cc02be35ba85c8b044e47d097661dffa8bea57cdb3db8b5da5d01cdbc94fe6c8902652cfe05fb9da7f2af0698be283a2",
+            untar=True,
+        )
+
+    def teardown(self):
+        pass
+
+
+class GromacsBenchmark(Benchmark):
+    def __init__(self, suite, model, type, option):
+        self.suite = suite
+        self.model = model  # The model name (e.g., "0001.5")
+        self.type = type  # The type of benchmark ("pme" or "rf")
+        self.option = option  # "graphs" or "eager"
+
+        self.gromacs_src = suite.gromacs_src
+        self.grappa_dir = suite.grappa_dir
+        self.gmx_path = suite.gromacs_build_path / "bin" / "gmx"
+
+        if self.type == "pme":
+            self.extra_args = [
+                "-pme",
+                "gpu",
+                "-pmefft",
+                "gpu",
+                "-notunepme",
+            ]
+        else:
+            self.extra_args = []
+
+    def name(self):
+        return f"gromacs-{self.model}-{self.type}-{self.option}"
+
+    def setup(self):
+        if self.type != "rf" and self.type != "pme":
+            raise ValueError(f"Unknown benchmark type: {self.type}")
+
+        if self.option != "graphs" and self.option != "eager":
+            raise ValueError(f"Unknown option: {self.option}")
+
+        if not self.gmx_path.exists():
+            raise FileNotFoundError(f"gmx executable not found at {self.gmx_path}")
+
+        model_dir = self.grappa_dir / self.model
+
+        if not model_dir.exists():
+            raise FileNotFoundError(f"Model directory not found: {model_dir}")
+
+        cmd_list = [
+            str(self.gmx_path),
+            "grompp",
+            "-f",
+            f"{str(self.grappa_dir)}/{self.type}.mdp",
+            "-c",
+            str(model_dir / "conf.gro"),
+            "-p",
+            str(model_dir / "topol.top"),
+            "-o",
+            f"{str(model_dir)}/{self.type}.tpr",
+        ]
+
+        # Generate configuration files
+        self.conf_result = run(
+            cmd_list,
+            add_sycl=True,
+        )
+
+    def run(self, env_vars):
+        model_dir = self.grappa_dir / self.model
+
+        env_vars.update({"SYCL_CACHE_PERSISTENT": "1"})
+
+        if self.option == "graphs":
+            env_vars.update({"GMX_CUDA_GRAPH": "1"})
+
+        # Run benchmark
+        command = [
+            str(self.gmx_path),
+            "mdrun",
+            "-s",
+            f"{str(model_dir)}/{self.type}.tpr",
+            "-nb",
+            "gpu",
+            "-update",
+            "gpu",
+            "-bonded",
+            "gpu",
+            "-ntmpi",
+            "1",
+            "-ntomp",
+            "1",
+            "-nobackup",
+            "-noconfout",
+            "-nstlist",
+            "100",
+            "-pin",
+            "on",
+            "-resethway",
+        ] + self.extra_args
+
+        mdrun_output = self.run_bench(
+            command,
+            env_vars,
+            add_sycl=True,
+            use_stdout=False,
+        )
+
+        if self.type == "pme" and not self._validate_correctness(
+            options.benchmark_cwd + "/md.log"
+        ):
+            raise ValueError(
+                f"Validation failed: Conserved energy drift exceeds threshold in {model_dir / 'md.log'}"
+            )
+
+        time = self._extract_execution_time(mdrun_output)
+
+        if options.verbose:
+            print(f"[{self.name()}] Time: {time:.3f} seconds")
+
+        return [
+            Result(
+                label=f"{self.name()}",
+                value=time,
+                unit="s",
+                command=command,
+                env=env_vars,
+                stdout=mdrun_output,
+                git_url=self.suite.git_url(),
+                git_hash=self.suite.git_tag(),
+            )
+        ]
+
+    def _extract_execution_time(self, log_content):
+        # Look for the line containing "Time:"
+        # and extract the first numeric value after it
+        time_lines = [line for line in log_content.splitlines() if "Time:" in line]
+
+        if len(time_lines) != 1:
+            raise ValueError(
+                f"Expected exactly 1 line containing 'Time:' in the log content, "
+                f"but found {len(time_lines)}."
+            )
+
+        for part in time_lines[0].split():
+            if part.replace(".", "", 1).isdigit():
+                return float(part)
+
+        raise ValueError(f"No numeric value found in the 'Time:' line.")
+
+    def _validate_correctness(self, log_file):
+        threshold = 1e-3  # Define an acceptable energy drift threshold
+
+        log_file = Path(log_file)
+        if not log_file.exists():
+            raise FileNotFoundError(f"Log file not found: {log_file}")
+
+        sci_pattern = r"([-+]?\d*\.\d+(?:e[-+]?\d+)?)"
+        with open(log_file, "r") as file:
+            for line in file:
+                if "Conserved energy drift:" in line:
+                    match = re.search(sci_pattern, line, re.IGNORECASE)
+                    if match:
+                        try:
+                            drift_value = float(match.group(1))
+                            return abs(drift_value) <= threshold
+                        except ValueError:
+                            print(
+                                f"Parsed drift value: {drift_value} exceeds threshold"
+                            )
+                            return False
+                    else:
+                        raise ValueError(
+                            f"No valid numerical value found in line: {line}"
+                        )
+
+        raise ValueError(f"Conserved Energy Drift not found in log file: {log_file}")
+
+    def teardown(self):
+        pass

devops/scripts/benchmarks/main.py

@@ -6,6 +6,7 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 from benches.compute import *
+from benches.gromacs import GromacsBench
 from benches.velocity import VelocityBench
 from benches.syclbench import *
 from benches.llamacpp import *
@@ -166,6 +167,7 @@ def main(directory, additional_env_vars, save_name, compare_names, filter):
         SyclBench(directory),
         LlamaCppBench(directory),
         UMFSuite(directory),
+        GromacsBench(directory),
         TestSuite(),
     ]
 
@@ -198,6 +200,7 @@ def main(directory, additional_env_vars, save_name, compare_names, filter):
             except Exception as e:
                 failures[s.name()] = f"Suite setup failure: {e}"
                 print(f"{type(s).__name__} setup failed. Benchmarks won't be added.")
+                print(f"failed: {e}")
             else:
                 print(f"{type(s).__name__} setup complete.")
                 benchmarks += suite_benchmarks
@@ -250,7 +253,7 @@ def main(directory, additional_env_vars, save_name, compare_names, filter):
             print(f"tearing down {benchmark.name()}... ", flush=True)
         benchmark.teardown()
         if options.verbose:
-            print("{benchmark.name()} teardown complete.")
+            print(f"{benchmark.name()} teardown complete.")
 
     this_name = options.current_run_name
     chart_data = {}

devops/scripts/benchmarks/presets.py

@@ -8,6 +8,7 @@
 presets: dict[str, list[str]] = {
     "Full": [
         "Compute Benchmarks",
+        "Gromacs Bench",
         "llama.cpp bench",
         "SYCL-Bench",
         "Velocity Bench",
@@ -24,12 +25,16 @@
     ],
     "Normal": [
         "Compute Benchmarks",
+        "Gromacs Bench",
         "llama.cpp bench",
         "Velocity Bench",
     ],
     "Test": [
         "Test Suite",
     ],
+    "Gromacs": [
+        "Gromacs Bench",
+    ],
 }