[SCYL][ROCM] Support for ROCm backend (intel#348)

* ROCm flags added to test suite, readme updated with docs, basic test update

Author: AidanBeltonS

SYCL/FilterSelector/select.cpp

@@ -39,6 +39,7 @@ int main() {
   bool HasLevelZeroDevices = false;
   bool HasOpenCLDevices = false;
   bool HasCUDADevices = false;
+  bool HasROCmDevices = false;
   bool HasOpenCLGPU = false;
   bool HasLevelZeroGPU = false;
 
@@ -50,6 +51,8 @@ int main() {
       HasOpenCLDevices = true;
     } else if (Backend == backend::cuda) {
       HasCUDADevices = true;
+    } else if (Backend == backend::rocm) {
+      HasROCmDevices = true;
     }
   }
 
@@ -64,6 +67,7 @@ int main() {
   std::cout << "HasLevelZeroDevices = " << HasLevelZeroDevices << std::endl;
   std::cout << "HasOpenCLDevices    = " << HasOpenCLDevices << std::endl;
   std::cout << "HasCUDADevices      = " << HasCUDADevices << std::endl;
+  std::cout << "HasROCmDevices      = " << HasROCmDevices << std::endl;
   std::cout << "HasOpenCLGPU        = " << HasOpenCLGPU << std::endl;
   std::cout << "HasLevelZeroGPU     = " << HasLevelZeroGPU << std::endl;
 
@@ -198,5 +202,17 @@ int main() {
     std::cout << "...PASS" << std::endl;
   }
 
+  if (HasROCmDevices) {
+    std::cout << "Test 'rocm'";
+    device d19(ONEAPI::filter_selector("rocm"));
+    assert(d19.get_platform().get_backend() == backend::rocm);
+    std::cout << "...PASS" << std::endl;
+
+    std::cout << "test 'rocm:gpu'";
+    device d20(ONEAPI::filter_selector("rocm:gpu"));
+    assert(d20.is_gpu() && d19.get_platform().get_backend() == backend::rocm);
+    std::cout << "...PASS" << std::endl;
+  }
+
   return 0;
 }

SYCL/Plugin/sycl-ls-gpu-rocm.cpp

@@ -0,0 +1,15 @@
+// REQUIRES: gpu, rocm, sycl-ls
+
+// RUN: env SYCL_DEVICE_FILTER=rocm sycl-ls --verbose >%t.rocm.out
+// RUN: FileCheck %s --check-prefixes=CHECK-BUILTIN-GPU-ROCM,CHECK-CUSTOM-GPU-ROCM --input-file %t.rocm.out
+
+// CHECK-BUILTIN-GPU-ROCM: gpu_selector(){{.*}}GPU :{{.*}}ROCM
+// CHECK-CUSTOM-GPU-ROCM: custom_selector(gpu){{.*}}GPU :{{.*}}ROCM
+
+//==---- sycl-ls-gpu-rocm.cpp - SYCL test for discovered/selected devices --==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//

SYCL/Plugin/sycl-ls-gpu-sycl-be.cpp

@@ -1,4 +1,4 @@
-// REQUIRES: gpu, cuda, opencl
+// REQUIRES: gpu, cuda, rocm, opencl, sycl-ls
 
 // RUN: sycl-ls --verbose >%t.default.out
 // RUN: FileCheck %s --check-prefixes=CHECK-BUILTIN-GPU-OPENCL,CHECK-CUSTOM-GPU-OPENCL --input-file %t.default.out
@@ -15,6 +15,12 @@
 // CHECK-BUILTIN-GPU-CUDA: gpu_selector(){{.*}}GPU : CUDA
 // CHECK-CUSTOM-GPU-CUDA: custom_selector(gpu){{.*}}GPU : CUDA
 
+// RUN: env SYCL_DEVICE_FILTER=rocm sycl-ls --verbose >%t.rocm.out
+// RUN: FileCheck %s --check-prefixes=CHECK-BUILTIN-GPU-ROCM,CHECK-CUSTOM-GPU-ROCM --input-file %t.rocm.out
+
+// CHECK-BUILTIN-GPU-ROCM: gpu_selector(){{.*}}GPU : ROCm
+// CHECK-CUSTOM-GPU-ROCM: custom_selector(gpu){{.*}}GPU : ROCm
+
 //==---- sycl-ls-gpu-sycl-be.cpp - SYCL test for discovered/selected devices
 //--==//
 //

SYCL/README.md

@@ -81,18 +81,19 @@ list of configurations. Each configuration includes backend separated
 from comma-separated list of target devices with colon. Example:
 
 ```
--DCHECK_SYCL_ALL="opencl:cpu,host;level_zero:gpu,host;cuda:gpu"
+-DCHECK_SYCL_ALL="opencl:cpu,host;level_zero:gpu,host;cuda:gpu;rocm:gpu"
 ```
 
 ***SYCL_BE*** SYCL backend to be used for testing. Supported values are:
  - **opencl** - for OpenCL backend;
  - **cuda** - for CUDA backend;
+ - **rocm** - for ROCm backend;
  - **level_zero** - Level Zero backend.
 
 ***SYCL_TARGET_DEVICES*** comma separated list of target devices for testing.
 Default value is cpu,gpu,acc,host. Supported values are:
  - **cpu**  - CPU device available in OpenCL backend only;
- - **gpu**  - GPU device available in OpenCL, Level Zero and CUDA backends;
+ - **gpu**  - GPU device available in OpenCL, Level Zero, CUDA, and ROCm backends;
  - **acc**  - FPGA emulator device available in OpenCL backend only;
  - **host** - SYCL Host device available with all backends.
 
@@ -105,6 +106,15 @@ specified by this variable.
 
 ***LEVEL_ZERO_LIBS_DIR*** path to Level Zero libraries.
 
+***ROCM_PLATFORM*** platform selection for ROCm targeted devices.
+Defaults to AMD if no value is given. Supported values are:
+ - **AMD**    - for ROCm to target AMD GPUs
+ - **NVIDIA** - for ROCm to target NVIDIA GPUs
+ 
+ ***MCPU*** Flag must be set for when using ROCm triple. 
+ For example it may be set to "gfx906".
+
+
 # Special test categories
 
 There are two special directories for extended testing. See documentation at:
@@ -124,7 +134,7 @@ unavailable.
 
  * **windows**, **linux** - host OS;
  * **cpu**, **gpu**, **host**, **accelerator** - target device;
- * **cuda**, **opencl**, **level_zero** - target backend;
+ * **cuda**, **rocm**, **opencl**, **level_zero** - target backend;
  * **sycl-ls** - sycl-ls tool availability;
  * **cl_options** - CL command line options recognized (or not) by compiler;
  * **opencl_icd** - OpenCL ICD loader availability;
@@ -142,7 +152,7 @@ configure specific single test execution in the command line:
  * **dpcpp_compiler** - full path to dpcpp compiler;
  * **target_device** - comma-separated list of target devices (cpu, gpu, acc,
    host);
- * **sycl_be** - SYCL backend to be used (opencl, level_zero, cuda);
+ * **sycl_be** - SYCL backend to be used (opencl, level_zero, cuda, rocm);
  * **dump_ir** - if IR dumping is supported for compiler (True, False);
  * **gpu-intel-dg1** - tells LIT infra that Intel GPU DG1 is present in the
    system. It is developer / CI infra responsibility to make sure that the

SYCL/lit.cfg.py

@@ -154,7 +154,7 @@
 # Mapping from SYCL_BE backend definition style to SYCL_DEVICE_FILTER used
 # for backward compatibility
 try:
-  config.sycl_be = { 'PI_OPENCL': 'opencl',  'PI_CUDA': 'cuda', 'PI_LEVEL_ZERO': 'level_zero'}[config.sycl_be]
+  config.sycl_be = { 'PI_OPENCL': 'opencl',  'PI_CUDA': 'cuda', 'PI_ROCM': 'rocm', 'PI_LEVEL_ZERO': 'level_zero'}[config.sycl_be]
 except:
   # do nothing a we expect that new format of plugin values are used
   pass
@@ -168,12 +168,24 @@
 if config.dump_ir_supported:
    config.available_features.add('dump_ir')
 
-if config.sycl_be not in ['host', 'opencl','cuda', 'level_zero']:
-    lit_config.error("Unknown SYCL BE specified '" +
-                     config.sycl_be +
-                     "' supported values are opencl, cuda, level_zero")
+if config.sycl_be not in ['host', 'opencl', 'cuda', 'rocm', 'level_zero']:
+   lit_config.error("Unknown SYCL BE specified '" +
+                    config.sycl_be +
+                    "' supported values are opencl, cuda, rocm, level_zero")
 
-config.substitutions.append( ('%clangxx', ' '+ config.dpcpp_compiler + ' ' + config.cxx_flags ) )
+# If ROCM_PLATFORM flag is not set, default to AMD, and check if ROCM platform is supported
+supported_rocm_platforms=["AMD", "NVIDIA"]
+if config.rocm_platform == "":
+    config.rocm_platform = "AMD"
+if config.rocm_platform not in supported_rocm_platforms:
+    lit_config.error("Unknown ROCm platform '" + config.rocm_platform + "' supported platforms are " + ', '.join(supported_rocm_platforms))
+
+if config.sycl_be == "rocm" and config.rocm_platform == "AMD":
+    mcpu_flag = '-mcpu=' + config.mcpu
+else:
+    mcpu_flag = ""
+
+config.substitutions.append( ('%clangxx', ' '+ config.dpcpp_compiler + ' ' + config.cxx_flags + ' ' + mcpu_flag) )
 config.substitutions.append( ('%clang', ' ' + config.dpcpp_compiler + ' ' + config.c_flags ) )
 config.substitutions.append( ('%threads_lib', config.sycl_threads_lib) )
 
@@ -275,8 +287,10 @@
 config.substitutions.append( ('%ACC_RUN_PLACEHOLDER',  acc_run_substitute) )
 config.substitutions.append( ('%ACC_CHECK_PLACEHOLDER',  acc_check_substitute) )
 
-if config.sycl_be == 'cuda':
+if config.sycl_be == 'cuda' or (config.sycl_be == 'rocm' and config.rocm_platform == 'NVIDIA'):
     config.substitutions.append( ('%sycl_triple',  "nvptx64-nvidia-cuda-sycldevice" ) )
+elif config.sycl_be == 'rocm' and config.rocm_platform == 'AMD':
+    config.substitutions.append( ('%sycl_triple',  "amdgcn-amd-amdhsa-sycldevice" ) )
 else:
     config.substitutions.append( ('%sycl_triple',  "spir64-unknown-unknown-sycldevice" ) )
 

SYCL/lit.site.cfg.py.in

@@ -22,6 +22,8 @@ config.level_zero_include = "@LEVEL_ZERO_INCLUDE@"
 config.opencl_include_dir = os.path.join(config.sycl_include, 'sycl')
 config.target_devices = lit_config.params.get("target_devices", "@SYCL_TARGET_DEVICES@")
 config.sycl_be = lit_config.params.get("sycl_be", "@SYCL_BE@")
+config.rocm_platform = "@ROCM_PLATFORM@"
+config.mcpu = "@MCPU@"
 config.sycl_threads_lib = '@SYCL_THREADS_LIB@'
 config.extra_environment = lit_config.params.get("extra_environment", "@LIT_EXTRA_ENVIRONMENT@")
 config.cxx_flags = "@CMAKE_CXX_FLAGS@"