[SYCL] Don't select devices with no available images (#6203)

This patch solves:

* [SYCL] Default selector should filter devices based on available device images #2004
* [SYCL][CUDA] Default selector behaviour #1665

In some cases the current selector may select a device for which we don't have an AOT or SPIR-V binary for, this patch ensures that such devices get skipped.

Author: npmiller

sycl/source/detail/program_manager/program_manager.cpp

@@ -1460,6 +1460,16 @@ kernel_id ProgramManager::getSYCLKernelID(const std::string &KernelName) {
   return KernelID->second;
 }
 
+bool ProgramManager::hasCompatibleImage(const device &Dev) {
+  std::lock_guard<std::mutex> Guard(m_KernelIDsMutex);
+
+  return std::any_of(
+      m_BinImg2KernelIDs.cbegin(), m_BinImg2KernelIDs.cend(),
+      [&](std::pair<RTDeviceBinaryImage *,
+                    std::shared_ptr<std::vector<kernel_id>>>
+              Elem) { return compatibleWithDevice(Elem.first, Dev); });
+}
+
 std::vector<kernel_id> ProgramManager::getAllSYCLKernelIDs() {
   std::lock_guard<std::mutex> KernelIDsGuard(m_KernelIDsMutex);
 

sycl/source/detail/program_manager/program_manager.hpp

@@ -188,6 +188,9 @@ class ProgramManager {
   void addOrInitDeviceGlobalEntry(const void *DeviceGlobalPtr,
                                   const char *UniqueId);
 
+  // Returns true if any available image is compatible with the device Dev.
+  bool hasCompatibleImage(const device &Dev);
+
   // The function returns a vector of SYCL device images that are compiled with
   // the required state and at least one device from the passed list of devices.
   std::vector<device_image_plain> getSYCLDeviceImagesWithCompatibleState(

sycl/source/device_selector.cpp

@@ -27,14 +27,27 @@
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
 
-// Utility function to check if device is of the preferred backend.
-// Currently preference is given to the level_zero backend.
-static bool isDeviceOfPreferredSyclBe(const device &Device) {
+// SYCL_DEVICE_FILTER doesn't need to be considered in the device preferences
+// as it filters the device list returned by device::get_devices itself, so
+// only matching devices will be scored.
+static int getDevicePreference(const device &Device) {
+  int Score = 0;
+
+  // No preferences for host devices.
   if (Device.is_host())
-    return false;
+    return Score;
+
+  // Strongly prefer devices with available images.
+  auto &program_manager = cl::sycl::detail::ProgramManager::getInstance();
+  if (program_manager.hasCompatibleImage(Device))
+    Score += 1000;
+
+  // Prefer level_zero backend devices.
+  if (detail::getSyclObjImpl(Device)->getPlugin().getBackend() ==
+      backend::ext_oneapi_level_zero)
+    Score += 50;
 
-  return detail::getSyclObjImpl(Device)->getPlugin().getBackend() ==
-         backend::ext_oneapi_level_zero;
+  return Score;
 }
 
 device device_selector::select_device() const {
@@ -64,11 +77,12 @@ device device_selector::select_device() const {
 
     // SYCL spec says: "If more than one device receives the high score then
     // one of those tied devices will be returned, but which of the devices
-    // from the tied set is to be returned is not defined". Here we give a
-    // preference to the device of the preferred BE.
-    //
+    // from the tied set is to be returned is not defined". So use the device
+    // preference score to resolve ties, this is necessary for custom_selectors
+    // that may not already include device preference in their scoring.
     if ((score < dev_score) ||
-        (score == dev_score && isDeviceOfPreferredSyclBe(dev))) {
+        ((score == dev_score) &&
+         (getDevicePreference(*res) < getDevicePreference(dev)))) {
       res = &dev;
       score = dev_score;
     }
@@ -97,25 +111,13 @@ device device_selector::select_device() const {
 /// 1. GPU
 /// 2. CPU
 /// 3. Host
+/// 4. Accelerator
 int default_selector::operator()(const device &dev) const {
-
-  int Score = REJECT_DEVICE_SCORE;
-
-  // Give preference to device of SYCL BE.
-  if (isDeviceOfPreferredSyclBe(dev))
-    Score = 50;
-
-  // If SYCL_DEVICE_FILTER is set, filter device gets a high point.
-  // All unmatched devices should never be selected.
-  detail::device_filter_list *FilterList =
-      detail::SYCLConfig<detail::SYCL_DEVICE_FILTER>::get();
-  // device::get_devices returns filtered list of devices.
-  // Keep 1000 for default score when filters were applied.
-  if (FilterList)
-    Score = 1000;
+  // The default selector doesn't reject any devices.
+  int Score = 0;
 
   if (dev.get_info<info::device::device_type>() == detail::get_forced_type())
-    Score += 1000;
+    Score += 2000;
 
   if (dev.is_gpu())
     Score += 500;
@@ -132,18 +134,18 @@ int default_selector::operator()(const device &dev) const {
   if (dev.is_accelerator())
     Score += 75;
 
+  // Add preference score.
+  Score += getDevicePreference(dev);
+
   return Score;
 }
 
 int gpu_selector::operator()(const device &dev) const {
   int Score = REJECT_DEVICE_SCORE;
 
   if (dev.is_gpu()) {
-    // device::get_devices returns filtered list of devices.
     Score = 1000;
-    // Give preference to device of SYCL BE.
-    if (isDeviceOfPreferredSyclBe(dev))
-      Score += 50;
+    Score += getDevicePreference(dev);
   }
   return Score;
 }
@@ -152,12 +154,8 @@ int cpu_selector::operator()(const device &dev) const {
   int Score = REJECT_DEVICE_SCORE;
 
   if (dev.is_cpu()) {
-    // device::get_devices returns filtered list of devices.
     Score = 1000;
-
-    // Give preference to device of SYCL BE.
-    if (isDeviceOfPreferredSyclBe(dev))
-      Score += 50;
+    Score += getDevicePreference(dev);
   }
   return Score;
 }
@@ -166,12 +164,8 @@ int accelerator_selector::operator()(const device &dev) const {
   int Score = REJECT_DEVICE_SCORE;
 
   if (dev.is_accelerator()) {
-    // device::get_devices returns filtered list of devices.
     Score = 1000;
-
-    // Give preference to device of SYCL BE.
-    if (isDeviceOfPreferredSyclBe(dev))
-      Score += 50;
+    Score += getDevicePreference(dev);
   }
   return Score;
 }
@@ -181,9 +175,7 @@ int host_selector::operator()(const device &dev) const {
 
   if (dev.is_host()) {
     Score = 1000;
-    // Give preference to device of SYCL BE.
-    if (isDeviceOfPreferredSyclBe(dev))
-      Score += 50;
+    Score += getDevicePreference(dev);
   }
   return Score;
 }