Add memspace "highest bandwidth"

This memspace contains memory targets ordered by the highest bandwidth
value (targets higher in the ordering are prioritised).
Querying the bandwidth value requires HMAT support on the platform,
calling umfMemspaceHighestBandwidthGet() will return NULL if it's not
supported.

Author: kswiecicki

include/umf/memspace.h

@@ -56,6 +56,11 @@ umf_memspace_handle_t umfMemspaceHostAllGet(void);
 ///
 umf_memspace_handle_t umfMemspaceHighestCapacityGet(void);
 
+/// \brief Retrieves predefined highest bandwidth memspace.
+/// \return highest bandwidth memspace handle on success or NULL on failure.
+///
+umf_memspace_handle_t umfMemspaceHighestBandwidthGet(void);
+
 #ifdef __cplusplus
 }
 #endif

src/CMakeLists.txt

@@ -79,7 +79,8 @@ set(UMF_SOURCES_LINUX
     memory_targets/memory_target_numa.c
     memspaces/memspace_numa.c
     memspaces/memspace_host_all.c
-    memspaces/memspace_highest_capacity.c)
+    memspaces/memspace_highest_capacity.c
+    memspaces/memspace_highest_bandwidth.c)
 
 set(UMF_SOURCES_WINDOWS ${UMF_SOURCES_WINDOWS} provider/provider_os_memory.c
                         provider/provider_os_memory_windows.c)

src/libumf_linux.c

@@ -29,6 +29,7 @@ void __attribute__((destructor)) umfDestroy(void) {
     umfMemoryTrackerDestroy(t);
     umfMemspaceHostAllDestroy();
     umfMemspaceHighestCapacityDestroy();
+    umfMemspaceHighestBandwidthDestroy();
     umfDestroyTopology();
 }
 

src/memory_target.c

@@ -84,3 +84,11 @@ umf_result_t umfMemoryTargetGetCapacity(umf_memory_target_handle_t memoryTarget,
     assert(capacity);
     return memoryTarget->ops->get_capacity(memoryTarget->priv, capacity);
 }
+
+umf_result_t
+umfMemoryTargetGetBandwidth(umf_memory_target_handle_t memoryTarget,
+                            size_t *capacity) {
+    assert(memoryTarget);
+    assert(capacity);
+    return memoryTarget->ops->get_bandwidth(memoryTarget->priv, capacity);
+}

src/memory_target.h

@@ -37,6 +37,9 @@ umf_result_t umfMemoryTargetClone(umf_memory_target_handle_t memoryTarget,
                                   umf_memory_target_handle_t *outHandle);
 umf_result_t umfMemoryTargetGetCapacity(umf_memory_target_handle_t memoryTarget,
                                         size_t *capacity);
+umf_result_t
+umfMemoryTargetGetBandwidth(umf_memory_target_handle_t memoryTarget,
+                            size_t *bandwidth);
 
 #ifdef __cplusplus
 }

src/memory_target_ops.h

@@ -41,6 +41,7 @@ typedef struct umf_memory_target_ops_t {
         umf_memory_provider_handle_t *provider);
 
     umf_result_t (*get_capacity)(void *memoryTarget, size_t *capacity);
+    umf_result_t (*get_bandwidth)(void *memoryTarget, size_t *bandwidth);
 } umf_memory_target_ops_t;
 
 #ifdef __cplusplus

src/memory_targets/memory_target_numa.c

@@ -195,12 +195,47 @@ static umf_result_t numa_get_capacity(void *memTarget, size_t *capacity) {
     return UMF_RESULT_SUCCESS;
 }
 
+static umf_result_t numa_get_bandwidth(void *memTarget, size_t *bandwidth) {
+    hwloc_topology_t topology = umfGetTopology();
+    if (!topology) {
+        return UMF_RESULT_ERROR_NOT_SUPPORTED;
+    }
+
+    hwloc_obj_t numaNode = hwloc_get_obj_by_type(
+        topology, HWLOC_OBJ_NUMANODE,
+        ((struct numa_memory_target_t *)memTarget)->physical_id);
+    if (!numaNode) {
+        return UMF_RESULT_ERROR_INVALID_ARGUMENT;
+    }
+
+    umf_result_t umf_ret = UMF_RESULT_SUCCESS;
+
+    hwloc_const_cpuset_t cCpuset = hwloc_topology_get_allowed_cpuset(topology);
+    hwloc_cpuset_t cpuset = hwloc_bitmap_dup(cCpuset);
+    struct hwloc_location initiator = {.location.cpuset = cpuset,
+                                       .type = HWLOC_LOCATION_TYPE_CPUSET};
+    hwloc_uint64_t value = 0;
+    int ret = hwloc_memattr_get_value(topology, HWLOC_MEMATTR_ID_BANDWIDTH,
+                                      numaNode, &initiator, 0, &value);
+    if (ret) {
+        umf_ret = (errno == EINVAL) ? UMF_RESULT_ERROR_NOT_SUPPORTED
+                                    : UMF_RESULT_ERROR_UNKNOWN;
+        goto err;
+    }
+
+    *bandwidth = value;
+err:
+    hwloc_bitmap_free(cpuset);
+    return umf_ret;
+}
+
 struct umf_memory_target_ops_t UMF_MEMORY_TARGET_NUMA_OPS = {
     .version = UMF_VERSION_CURRENT,
     .initialize = numa_initialize,
     .finalize = numa_finalize,
     .pool_create_from_memspace = numa_pool_create_from_memspace,
     .clone = numa_clone,
     .get_capacity = numa_get_capacity,
+    .get_bandwidth = numa_get_bandwidth,
     .memory_provider_create_from_memspace =
         numa_memory_provider_create_from_memspace};

src/memspace_internal.h

@@ -47,6 +47,7 @@ void umfMemspaceDestroy(umf_memspace_handle_t hMemspace);
 
 void umfMemspaceHostAllDestroy(void);
 void umfMemspaceHighestCapacityDestroy(void);
+void umfMemspaceHighestBandwidthDestroy(void);
 
 #ifdef __cplusplus
 }

src/memspaces/memspace_highest_bandwidth.c

@@ -0,0 +1,84 @@
+/*
+ *
+ * Copyright (C) 2024 Intel Corporation
+ *
+ * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ */
+
+#include <assert.h>
+#include <hwloc.h>
+#include <stdlib.h>
+
+#include "base_alloc_global.h"
+#include "memory_target_numa.h"
+#include "memspace_internal.h"
+#include "memspace_numa.h"
+#include "topology.h"
+#include "utils_concurrency.h"
+
+static umf_result_t
+umfMemspaceHighestBandwidthCreate(umf_memspace_handle_t *hMemspace) {
+    if (!hMemspace) {
+        return UMF_RESULT_ERROR_INVALID_ARGUMENT;
+    }
+
+    umf_memspace_handle_t hostAllMemspace = umfMemspaceHostAllGet();
+    if (!hostAllMemspace) {
+        return UMF_RESULT_ERROR_UNKNOWN;
+    }
+
+    umf_memspace_handle_t highBandwidthMemspace = NULL;
+    umf_result_t ret =
+        umfMemspaceClone(hostAllMemspace, &highBandwidthMemspace);
+    if (ret != UMF_RESULT_SUCCESS) {
+        return ret;
+    }
+
+    ret = umfMemspaceSortDesc(highBandwidthMemspace,
+                              &umfMemoryTargetGetBandwidth);
+    if (ret != UMF_RESULT_SUCCESS) {
+        // TODO: Should we have an alternative way to query 'bandwidth' value
+        //       of each NUMA node?
+        // HWLOC could possibly return an 'EINVAL' error, which in this context
+        // means that the HMAT is unavailable and we can't obtain the
+        // 'bandwidth' value of any NUMA node.
+        goto err_destroy_memspace;
+    }
+
+    *hMemspace = highBandwidthMemspace;
+    return UMF_RESULT_SUCCESS;
+
+err_destroy_memspace:
+    umfMemspaceDestroy(highBandwidthMemspace);
+    return ret;
+}
+
+static umf_memspace_handle_t UMF_MEMSPACE_HIGHEST_BANDWIDTH = NULL;
+static UTIL_ONCE_FLAG UMF_MEMSPACE_HBW_INITIALIZED = UTIL_ONCE_FLAG_INIT;
+
+void umfMemspaceHighestBandwidthDestroy(void) {
+    if (UMF_MEMSPACE_HIGHEST_BANDWIDTH) {
+        umfMemspaceDestroy(UMF_MEMSPACE_HIGHEST_BANDWIDTH);
+        UMF_MEMSPACE_HIGHEST_BANDWIDTH = NULL;
+    }
+}
+
+static void umfMemspaceHighestBandwidthInit(void) {
+    umf_result_t ret =
+        umfMemspaceHighestBandwidthCreate(&UMF_MEMSPACE_HIGHEST_BANDWIDTH);
+    if (ret != UMF_RESULT_SUCCESS) {
+        assert(ret == UMF_RESULT_ERROR_NOT_SUPPORTED);
+    }
+
+    // TODO: Setup appropriate cleanup when HBW memspace becomes available
+    // on Windows. HBW memspace depends on OS provider, which currently
+    // doesn't support Windows.
+}
+
+umf_memspace_handle_t umfMemspaceHighestBandwidthGet(void) {
+    util_init_once(&UMF_MEMSPACE_HBW_INITIALIZED,
+                   umfMemspaceHighestBandwidthInit);
+    return UMF_MEMSPACE_HIGHEST_BANDWIDTH;
+}

test/CMakeLists.txt

@@ -180,6 +180,10 @@ if(LINUX) # OS-specific functions are implemented only for Linux now
         NAME memspace_highest_capacity
         SRCS memspaces/memspace_highest_capacity.cpp
         LIBS ${UMF_UTILS_FOR_TEST} ${LIBNUMA_LIBRARIES})
+    add_umf_test(
+        NAME memspace_highest_bandwidth
+        SRCS memspaces/memspace_highest_bandwidth.cpp
+        LIBS ${UMF_UTILS_FOR_TEST} ${LIBNUMA_LIBRARIES})
 endif()
 
 # TODO add support for Windows

test/memspaces/memspace_highest_bandwidth.cpp

@@ -0,0 +1,19 @@
+// Copyright (C) 2024 Intel Corporation
+// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "memory_target_numa.h"
+#include "memspace_helpers.hpp"
+#include "memspace_internal.h"
+#include "test_helpers.h"
+
+#include <numa.h>
+#include <numaif.h>
+#include <umf/memspace.h>
+
+using umf_test::test;
+
+TEST_F(numaNodesTest, memspaceGet) {
+    umf_memspace_handle_t hMemspace = umfMemspaceHighestBandwidthGet();
+    UT_ASSERTne(hMemspace, nullptr);
+}