Add memspace "highest bandwidth"

kswiecicki · kswiecicki · commit 5c838736f6c6 · 2024-05-22T15:00:15.000Z
This memspace contains an aggregated list of NUMA nodes identified as best
targets after selecting each NUMA node as the initiator.
Querying the bandwidth value requires HMAT support on the platform,
calling umfMemspaceHighestBandwidthGet() will return NULL if it's not
supported.
diff --git a/include/umf/memspace.h b/include/umf/memspace.h
@@ -56,6 +56,11 @@ umf_memspace_handle_t umfMemspaceHostAllGet(void);
 ///
 umf_memspace_handle_t umfMemspaceHighestCapacityGet(void);
 
+/// \brief Retrieves predefined highest bandwidth memspace.
+/// \return highest bandwidth memspace handle on success or NULL on failure.
+///
+umf_memspace_handle_t umfMemspaceHighestBandwidthGet(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -81,7 +81,8 @@ set(UMF_SOURCES_COMMON_LINUX_MACOSX
     memory_targets/memory_target_numa.c
     memspaces/memspace_numa.c
     memspaces/memspace_host_all.c
-    memspaces/memspace_highest_capacity.c)
+    memspaces/memspace_highest_capacity.c
+    memspaces/memspace_highest_bandwidth.c)
 
 set(UMF_SOURCES_LINUX ${UMF_SOURCES_LINUX} ${UMF_SOURCES_COMMON_LINUX_MACOSX}
                       provider/provider_os_memory_linux.c)
diff --git a/src/libumf.map b/src/libumf.map
@@ -30,6 +30,7 @@ UMF_1.0 {
         umfMemoryProviderPutIPCHandle;
         umfMemspaceCreateFromNumaArray;
         umfMemspaceDestroy;
+        umfMemspaceHighestBandwidthGet;
         umfMemspaceHighestCapacityGet;
         umfMemspaceHostAllGet;
         umfOpenIPCHandle;
diff --git a/src/libumf_linux.c b/src/libumf_linux.c
@@ -29,6 +29,7 @@ void __attribute__((destructor)) umfDestroy(void) {
     umfMemoryTrackerDestroy(t);
     umfMemspaceHostAllDestroy();
     umfMemspaceHighestCapacityDestroy();
+    umfMemspaceHighestBandwidthDestroy();
     umfDestroyTopology();
 }
 
diff --git a/src/memory_target.c b/src/memory_target.c
@@ -83,3 +83,14 @@ umf_result_t umfMemoryTargetGetCapacity(umf_memory_target_handle_t memoryTarget,
     assert(capacity);
     return memoryTarget->ops->get_capacity(memoryTarget->priv, capacity);
 }
+
+umf_result_t
+umfMemoryTargetGetBandwidth(umf_memory_target_handle_t srcMemoryTarget,
+                            umf_memory_target_handle_t dstMemoryTarget,
+                            size_t *bandwidth) {
+    assert(srcMemoryTarget);
+    assert(dstMemoryTarget);
+    assert(bandwidth);
+    return srcMemoryTarget->ops->get_bandwidth(
+        srcMemoryTarget->priv, dstMemoryTarget->priv, bandwidth);
+}
diff --git a/src/memory_target.h b/src/memory_target.h
@@ -37,6 +37,10 @@ umf_result_t umfMemoryTargetClone(umf_memory_target_handle_t memoryTarget,
                                   umf_memory_target_handle_t *outHandle);
 umf_result_t umfMemoryTargetGetCapacity(umf_memory_target_handle_t memoryTarget,
                                         size_t *capacity);
+umf_result_t
+umfMemoryTargetGetBandwidth(umf_memory_target_handle_t srcMemoryTarget,
+                            umf_memory_target_handle_t dstMemoryTarget,
+                            size_t *bandwidth);
 
 #ifdef __cplusplus
 }
diff --git a/src/memory_target_ops.h b/src/memory_target_ops.h
@@ -41,6 +41,8 @@ typedef struct umf_memory_target_ops_t {
         umf_memory_provider_handle_t *provider);
 
     umf_result_t (*get_capacity)(void *memoryTarget, size_t *capacity);
+    umf_result_t (*get_bandwidth)(void *srcMemoryTarget, void *dstMemoryTarget,
+                                  size_t *bandwidth);
 } umf_memory_target_ops_t;
 
 #ifdef __cplusplus
diff --git a/src/memory_targets/memory_target_numa.c b/src/memory_targets/memory_target_numa.c
@@ -19,6 +19,7 @@
 #include "base_alloc_global.h"
 #include "memory_target_numa.h"
 #include "topology.h"
+#include "utils_log.h"
 
 struct numa_memory_target_t {
     unsigned physical_id;
@@ -143,12 +144,57 @@ static umf_result_t numa_get_capacity(void *memTarget, size_t *capacity) {
     return UMF_RESULT_SUCCESS;
 }
 
+static umf_result_t numa_get_bandwidth(void *srcMemoryTarget,
+                                       void *dstMemoryTarget,
+                                       size_t *bandwidth) {
+    hwloc_topology_t topology = umfGetTopology();
+    if (!topology) {
+        return UMF_RESULT_ERROR_NOT_SUPPORTED;
+    }
+
+    hwloc_obj_t srcNumaNode = hwloc_get_obj_by_type(
+        topology, HWLOC_OBJ_NUMANODE,
+        ((struct numa_memory_target_t *)srcMemoryTarget)->physical_id);
+    if (!srcNumaNode) {
+        return UMF_RESULT_ERROR_INVALID_ARGUMENT;
+    }
+
+    hwloc_obj_t dstNumaNode = hwloc_get_obj_by_type(
+        topology, HWLOC_OBJ_NUMANODE,
+        ((struct numa_memory_target_t *)dstMemoryTarget)->physical_id);
+    if (!dstNumaNode) {
+        return UMF_RESULT_ERROR_INVALID_ARGUMENT;
+    }
+
+    // Given NUMA nodes aren't local, HWLOC returns an error in such case.
+    if (!hwloc_bitmap_intersects(srcNumaNode->cpuset, dstNumaNode->cpuset)) {
+        *bandwidth = 0;
+        return UMF_RESULT_SUCCESS;
+    }
+
+    struct hwloc_location initiator = {.location.cpuset = srcNumaNode->cpuset,
+                                       .type = HWLOC_LOCATION_TYPE_CPUSET};
+    hwloc_uint64_t value = 0;
+    int ret = hwloc_memattr_get_value(topology, HWLOC_MEMATTR_ID_BANDWIDTH,
+                                      dstNumaNode, &initiator, 0, &value);
+    if (ret) {
+        LOG_ERR("Retrieving bandwidth for initiator node %u to node %u failed.",
+                srcNumaNode->os_index, dstNumaNode->os_index);
+        return (errno == EINVAL) ? UMF_RESULT_ERROR_NOT_SUPPORTED
+                                 : UMF_RESULT_ERROR_UNKNOWN;
+    }
+
+    *bandwidth = value;
+    return UMF_RESULT_SUCCESS;
+}
+
 struct umf_memory_target_ops_t UMF_MEMORY_TARGET_NUMA_OPS = {
     .version = UMF_VERSION_CURRENT,
     .initialize = numa_initialize,
     .finalize = numa_finalize,
     .pool_create_from_memspace = numa_pool_create_from_memspace,
     .clone = numa_clone,
     .get_capacity = numa_get_capacity,
+    .get_bandwidth = numa_get_bandwidth,
     .memory_provider_create_from_memspace =
         numa_memory_provider_create_from_memspace};
diff --git a/src/memspace.c b/src/memspace.c
@@ -208,3 +208,85 @@ umfMemspaceSortDesc(umf_memspace_handle_t hMemspace,
 
     return UMF_RESULT_SUCCESS;
 }
+
+umf_result_t umfMemspaceFilter(umf_memspace_handle_t hMemspace,
+                               umfGetTargetFn getTarget,
+                               umf_memspace_handle_t *filteredMemspace) {
+    if (!hMemspace || !getTarget) {
+        return UMF_RESULT_ERROR_INVALID_ARGUMENT;
+    }
+
+    umf_memory_target_handle_t *uniqueBestNodes =
+        umf_ba_global_alloc(hMemspace->size * sizeof(*uniqueBestNodes));
+    if (!uniqueBestNodes) {
+        return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY;
+    }
+
+    umf_result_t ret = UMF_RESULT_SUCCESS;
+
+    size_t numUniqueBestNodes = 0;
+    for (size_t nodeIdx = 0; nodeIdx < hMemspace->size; nodeIdx++) {
+        umf_memory_target_handle_t target = NULL;
+        ret = getTarget(hMemspace->nodes[nodeIdx], hMemspace->nodes,
+                        hMemspace->size, &target);
+        if (ret != UMF_RESULT_SUCCESS) {
+            goto err_free_best_targets;
+        }
+
+        // check if the target is already present in the best nodes
+        size_t bestTargetIdx;
+        for (bestTargetIdx = 0; bestTargetIdx < numUniqueBestNodes;
+             bestTargetIdx++) {
+            if (uniqueBestNodes[bestTargetIdx] == target) {
+                break;
+            }
+        }
+
+        // if the target is not present, add it to the best nodes
+        if (bestTargetIdx == numUniqueBestNodes) {
+            uniqueBestNodes[numUniqueBestNodes++] = target;
+        }
+    }
+
+    // copy the unique best nodes into a new memspace
+    umf_memspace_handle_t newMemspace =
+        umf_ba_global_alloc(sizeof(*newMemspace));
+    if (!newMemspace) {
+        ret = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY;
+        goto err_free_best_targets;
+    }
+
+    newMemspace->size = numUniqueBestNodes;
+    newMemspace->nodes =
+        umf_ba_global_alloc(sizeof(*newMemspace->nodes) * newMemspace->size);
+    if (!newMemspace->nodes) {
+        ret = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY;
+        goto err_free_new_memspace;
+    }
+
+    size_t cloneIdx = 0;
+    for (size_t cloneIdx = 0; cloneIdx < newMemspace->size; cloneIdx++) {
+        ret = umfMemoryTargetClone(uniqueBestNodes[cloneIdx],
+                                   &newMemspace->nodes[cloneIdx]);
+        if (ret != UMF_RESULT_SUCCESS) {
+            goto err_free_cloned_nodes;
+        }
+    }
+
+    *filteredMemspace = newMemspace;
+    umf_ba_global_free(uniqueBestNodes);
+
+    return UMF_RESULT_SUCCESS;
+
+err_free_cloned_nodes:
+    while (cloneIdx != 0) {
+        cloneIdx--;
+        umfMemoryTargetDestroy(newMemspace->nodes[cloneIdx]);
+    }
+    umf_ba_global_free(newMemspace->nodes);
+err_free_new_memspace:
+    umf_ba_global_free(newMemspace);
+err_free_best_targets:
+    umf_ba_global_free(uniqueBestNodes);
+    return ret;
+}
diff --git a/src/memspace_internal.h b/src/memspace_internal.h
@@ -39,6 +39,18 @@ typedef umf_result_t (*umfGetPropertyFn)(umf_memory_target_handle_t,
 umf_result_t umfMemspaceSortDesc(umf_memspace_handle_t hMemspace,
                                  umfGetPropertyFn getProperty);
 
+typedef umf_result_t (*umfGetTargetFn)(umf_memory_target_handle_t initiator,
+                                       umf_memory_target_handle_t *nodes,
+                                       size_t numNodes,
+                                       umf_memory_target_handle_t *target);
+
+///
+/// \brief Filters the targets using getTarget() to create a new memspace
+///
+umf_result_t umfMemspaceFilter(umf_memspace_handle_t hMemspace,
+                               umfGetTargetFn getTarget,
+                               umf_memspace_handle_t *filteredMemspace);
+
 ///
 /// \brief Destroys memspace
 /// \param hMemspace handle to memspace
@@ -47,6 +59,7 @@ void umfMemspaceDestroy(umf_memspace_handle_t hMemspace);
 
 void umfMemspaceHostAllDestroy(void);
 void umfMemspaceHighestCapacityDestroy(void);
+void umfMemspaceHighestBandwidthDestroy(void);
 
 #ifdef __cplusplus
 }
diff --git a/src/memspaces/memspace_highest_bandwidth.c b/src/memspaces/memspace_highest_bandwidth.c
@@ -0,0 +1,103 @@
+/*
+ *
+ * Copyright (C) 2024 Intel Corporation
+ *
+ * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ */
+
+#include <assert.h>
+#include <ctype.h>
+#include <hwloc.h>
+#include <stdlib.h>
+
+#include "base_alloc_global.h"
+#include "memory_target_numa.h"
+#include "memspace_internal.h"
+#include "memspace_numa.h"
+#include "topology.h"
+#include "utils_common.h"
+#include "utils_concurrency.h"
+#include "utils_log.h"
+
+static umf_result_t getBestBandwidthTarget(umf_memory_target_handle_t initiator,
+                                           umf_memory_target_handle_t *nodes,
+                                           size_t numNodes,
+                                           umf_memory_target_handle_t *target) {
+    size_t bestNodeIdx = 0;
+    size_t bestBandwidth = 0;
+    for (size_t nodeIdx = 0; nodeIdx < numNodes; nodeIdx++) {
+        size_t bandwidth = 0;
+        umf_result_t ret =
+            umfMemoryTargetGetBandwidth(initiator, nodes[nodeIdx], &bandwidth);
+        if (ret) {
+            return ret;
+        }
+
+        if (bandwidth > bestBandwidth) {
+            bestNodeIdx = nodeIdx;
+            bestBandwidth = bandwidth;
+        }
+    }
+
+    *target = nodes[bestNodeIdx];
+
+    return UMF_RESULT_SUCCESS;
+}
+
+static umf_result_t
+umfMemspaceHighestBandwidthCreate(umf_memspace_handle_t *hMemspace) {
+    if (!hMemspace) {
+        return UMF_RESULT_ERROR_INVALID_ARGUMENT;
+    }
+
+    umf_memspace_handle_t hostAllMemspace = umfMemspaceHostAllGet();
+    if (!hostAllMemspace) {
+        return UMF_RESULT_ERROR_UNKNOWN;
+    }
+
+    umf_memspace_handle_t highBandwidthMemspace = NULL;
+    umf_result_t ret = umfMemspaceFilter(
+        hostAllMemspace, getBestBandwidthTarget, &highBandwidthMemspace);
+    if (ret != UMF_RESULT_SUCCESS) {
+        // HWLOC could possibly return an 'EINVAL' error, which in this context
+        // means that the HMAT is unavailable and we can't obtain the
+        // 'bandwidth' value of any NUMA node.
+        return ret;
+    }
+
+    *hMemspace = highBandwidthMemspace;
+    return UMF_RESULT_SUCCESS;
+}
+
+static umf_memspace_handle_t UMF_MEMSPACE_HIGHEST_BANDWIDTH = NULL;
+static UTIL_ONCE_FLAG UMF_MEMSPACE_HBW_INITIALIZED = UTIL_ONCE_FLAG_INIT;
+
+void umfMemspaceHighestBandwidthDestroy(void) {
+    if (UMF_MEMSPACE_HIGHEST_BANDWIDTH) {
+        umfMemspaceDestroy(UMF_MEMSPACE_HIGHEST_BANDWIDTH);
+        UMF_MEMSPACE_HIGHEST_BANDWIDTH = NULL;
+    }
+}
+
+static void umfMemspaceHighestBandwidthInit(void) {
+    umf_result_t ret =
+        umfMemspaceHighestBandwidthCreate(&UMF_MEMSPACE_HIGHEST_BANDWIDTH);
+    if (ret != UMF_RESULT_SUCCESS) {
+        LOG_ERR(
+            "Creating the highest bandwidth memspace failed with a %u error\n",
+            ret);
+        assert(ret == UMF_RESULT_ERROR_NOT_SUPPORTED);
+    }
+
+#if defined(_WIN32) && !defined(UMF_SHARED_LIBRARY)
+    atexit(umfMemspaceHighestBandwidthDestroy);
+#endif
+}
+
+umf_memspace_handle_t umfMemspaceHighestBandwidthGet(void) {
+    util_init_once(&UMF_MEMSPACE_HBW_INITIALIZED,
+                   umfMemspaceHighestBandwidthInit);
+    return UMF_MEMSPACE_HIGHEST_BANDWIDTH;
+}
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -173,6 +173,10 @@ if(LINUX) # OS-specific functions are implemented only for Linux now
         NAME memspace_highest_capacity
         SRCS memspaces/memspace_highest_capacity.cpp
         LIBS ${UMF_UTILS_FOR_TEST} ${LIBNUMA_LIBRARIES})
+    add_umf_test(
+        NAME memspace_highest_bandwidth
+        SRCS memspaces/memspace_highest_bandwidth.cpp
+        LIBS ${UMF_UTILS_FOR_TEST} ${LIBNUMA_LIBRARIES})
 endif()
 
 if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_LEVEL_ZERO_PROVIDER)
diff --git a/test/memspaces/memspace_highest_bandwidth.cpp b/test/memspaces/memspace_highest_bandwidth.cpp
@@ -0,0 +1,19 @@
+// Copyright (C) 2024 Intel Corporation
+// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "memory_target_numa.h"
+#include "memspace_helpers.hpp"
+#include "memspace_internal.h"
+#include "test_helpers.h"
+
+#include <numa.h>
+#include <numaif.h>
+#include <umf/memspace.h>
+
+using umf_test::test;
+
+TEST_F(numaNodesTest, memspaceGet) {
+    umf_memspace_handle_t hMemspace = umfMemspaceHighestBandwidthGet();
+    UT_ASSERTne(hMemspace, nullptr);
+}

Original file line number	Diff line number	Diff line change
`@@ -29,6 +29,7 @@ void __attribute__((destructor)) umfDestroy(void) {`
`29`	`29`	`umfMemoryTrackerDestroy(t);`
`30`	`30`	`umfMemspaceHostAllDestroy();`
`31`	`31`	`umfMemspaceHighestCapacityDestroy();`
	`32`	`+ umfMemspaceHighestBandwidthDestroy();`
`32`	`33`	`umfDestroyTopology();`
`33`	`34`	`}`
`34`	`35`