Add IPC example with Level 0 provider

CMakeLists.txt

@@ -436,9 +436,10 @@ endif()
 install(FILES ${CMAKE_SOURCE_DIR}/LICENSE.TXT
         DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/doc/${PROJECT_NAME}/")
 
-install(FILES examples/basic/gpu_shared_memory.c
-              examples/basic/utils_level_zero.h examples/basic/basic.c
-        DESTINATION "${CMAKE_INSTALL_DOCDIR}/examples")
+install(
+    FILES examples/basic/gpu_shared_memory.c examples/basic/utils_level_zero.h
+          examples/basic/basic.c examples/basic/ipc_level_zero.c
+    DESTINATION "${CMAKE_INSTALL_DOCDIR}/examples")
 
 # Add the include directory and the headers target to the install.
 install(DIRECTORY "${PROJECT_SOURCE_DIR}/include/"

examples/CMakeLists.txt

@@ -70,3 +70,41 @@ else()
             "UMF_BUILD_LEVEL_ZERO_PROVIDER and UMF_BUILD_LIBUMF_POOL_DISJOINT "
             "to be turned ON - skipping")
 endif()
+
+if(UMF_BUILD_GPU_EXAMPLES
+   AND UMF_BUILD_LIBUMF_POOL_DISJOINT
+   AND UMF_ENABLE_POOL_TRACKING
+   AND UMF_BUILD_LEVEL_ZERO_PROVIDER)
+    set(EXAMPLE_NAME umf_example_ipc_level_zero)
+
+    add_umf_executable(
+        NAME ${EXAMPLE_NAME}
+        SRCS basic/ipc_level_zero.c
+        LIBS umf disjoint_pool ze_loader)
+
+    target_include_directories(
+        ${EXAMPLE_NAME}
+        PRIVATE ${LEVEL_ZERO_INCLUDE_DIRS} ${UMF_CMAKE_SOURCE_DIR}/src/utils
+                ${UMF_CMAKE_SOURCE_DIR}/include)
+
+    target_link_directories(${EXAMPLE_NAME} PRIVATE ${LIBHWLOC_LIBRARY_DIRS})
+
+    add_test(
+        NAME ${EXAMPLE_NAME}
+        COMMAND ${EXAMPLE_NAME}
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+
+    set_tests_properties(${EXAMPLE_NAME} PROPERTIES LABELS "example")
+
+    if(WINDOWS)
+        # append PATH to DLLs
+        set_property(TEST ${EXAMPLE_NAME} PROPERTY ENVIRONMENT_MODIFICATION
+                                                   "${DLL_PATH_LIST}")
+    endif()
+else()
+    message(
+        STATUS
+            "IPC Level 0 example requires UMF_BUILD_GPU_EXAMPLES, "
+            "UMF_BUILD_LEVEL_ZERO_PROVIDER, UMF_BUILD_LIBUMF_POOL_DISJOINT and "
+            "UMF_ENABLE_POOL_TRACKING to be turned ON - skipping")
+endif()

examples/README.md

@@ -27,3 +27,14 @@ cleans up and exits with an error status.
 * Level Zero headers and libraries
 * compatible GPU with installed driver
 * set UMF_BUILD_GPU_EXAMPLES, UMF_BUILD_LIBUMF_POOL_DISJOINT and UMF_BUILD_LEVEL_ZERO_PROVIDER CMake configuration flags to ON
+
+## IPC example with Level Zero memory provider
+This example demonstrates how to use UMF IPC API. The example creates two
+memory pools of Level Zero device memory: the producer pool (where the buffer
+is allocated) and the consumer pool (where the IPC handle is mapped). To run
+and build this example Level Zero development package should be installed.
+
+### Requirements
+* Level Zero headers and libraries
+* compatible GPU with installed driver
+* set UMF_BUILD_GPU_EXAMPLES, UMF_BUILD_LIBUMF_POOL_DISJOINT, UMF_BUILD_LEVEL_ZERO_PROVIDER and UMF_ENABLE_POOL_TRACKING CMake configuration flags to ON

examples/basic/ipc_level_zero.c

@@ -0,0 +1,170 @@
+/*
+ *
+ * Copyright (C) 2024 Intel Corporation
+ *
+ * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ */
+
+#include <stdio.h>
+
+#include "umf/ipc.h"
+#include "umf/memory_pool.h"
+#include "umf/pools/pool_disjoint.h"
+#include "umf/providers/provider_level_zero.h"
+
+#include "utils_level_zero.h"
+
+int create_level_zero_pool(ze_context_handle_t context,
+                           ze_device_handle_t device,
+                           umf_memory_pool_handle_t *pool) {
+    // setup params
+    level_zero_memory_provider_params_t params = {0};
+    params.level_zero_context_handle = context;
+    params.level_zero_device_handle = device;
+    params.memory_type = UMF_MEMORY_TYPE_DEVICE;
+    // create Level Zero provider
+    umf_memory_provider_handle_t provider = 0;
+    umf_result_t umf_result = umfMemoryProviderCreate(
+        umfLevelZeroMemoryProviderOps(), &params, &provider);
+    if (umf_result != UMF_RESULT_SUCCESS) {
+        fprintf(stderr, "Failed to create Level Zero memory provider!\n");
+        return -1;
+    }
+
+    // create pool
+    umf_pool_create_flags_t flags = UMF_POOL_CREATE_FLAG_OWN_PROVIDER;
+    umf_disjoint_pool_params_t disjoint_params = umfDisjointPoolParamsDefault();
+    umf_result = umfPoolCreate(umfDisjointPoolOps(), provider, &disjoint_params,
+                               flags, pool);
+    if (umf_result != UMF_RESULT_SUCCESS) {
+        fprintf(stderr, "Failed to create pool!\n");
+        return -1;
+    }
+
+    return 0;
+}
+
+int main(void) {
+    uint32_t driver_idx = 0;
+    ze_driver_handle_t driver = NULL;
+    ze_device_handle_t device = NULL;
+    ze_context_handle_t producer_context = NULL;
+    ze_context_handle_t consumer_context = NULL;
+    int ret = init_level_zero();
+    if (ret != 0) {
+        fprintf(stderr, "Failed to init Level 0!\n");
+        return ret;
+    }
+
+    ret = find_driver_with_gpu(&driver_idx, &driver);
+    if (ret || driver == NULL) {
+        fprintf(stderr, "Cannot find L0 driver with GPU device!\n");
+        return ret;
+    }
+
+    ret = create_context(driver, &producer_context);
+    if (ret != 0) {
+        fprintf(stderr, "Failed to create L0 context!\n");
+        return ret;
+    }
+
+    ret = create_context(driver, &consumer_context);
+    if (ret != 0) {
+        fprintf(stderr, "Failed to create L0 context!\n");
+        return ret;
+    }
+
+    ret = find_gpu_device(driver, &device);
+    if (ret || device == NULL) {
+        fprintf(stderr, "Cannot find GPU device!\n");
+        return ret;
+    }
+
+    // create producer pool
+    umf_memory_pool_handle_t producer_pool = 0;
+    ret = create_level_zero_pool(producer_context, device, &producer_pool);
+    if (ret != 0) {
+        fprintf(stderr, "Failed to create producer pool!\n");
+        return ret;
+    }
+
+    fprintf(stdout, "Producer pool created.\n");
+
+    void *initial_buf = umfPoolMalloc(producer_pool, 1024);
+    if (!initial_buf) {
+        fprintf(stderr, "Failed to allocate buffer from UMF pool!\n");
+        return -1;
+    }
+
+    fprintf(stdout, "Buffer allocated from the producer pool.\n");
+
+    umf_ipc_handle_t ipc_handle = NULL;
+    size_t handle_size = 0;
+    umf_result_t umf_result =
+        umfGetIPCHandle(initial_buf, &ipc_handle, &handle_size);
+    if (umf_result != UMF_RESULT_SUCCESS) {
+        fprintf(stderr, "Failed to get IPC handle!\n");
+        return -1;
+    }
+
+    fprintf(stdout, "IPC handle obtained.\n");
+
+    // create consumer pool
+    umf_memory_pool_handle_t consumer_pool = 0;
+    ret = create_level_zero_pool(consumer_context, device, &consumer_pool);
+    if (ret != 0) {
+        fprintf(stderr, "Failed to create consumer pool!\n");
+        return ret;
+    }
+
+    fprintf(stdout, "Consumer pool created.\n");
+
+    void *mapped_buf = NULL;
+    umf_result = umfOpenIPCHandle(consumer_pool, ipc_handle, &mapped_buf);
+    if (umf_result != UMF_RESULT_SUCCESS) {
+        fprintf(stderr, "Failed to open IPC handle!\n");
+        return -1;
+    }
+
+    fprintf(stdout, "IPC handle opened in the consumer pool.\n");
+
+    // Now we have two mappings (belongs to different Level Zero contexts) to the same physical memory region:
+    // * the initial mapping, pointed by initial_buf, we get by allocating memory from the producer pool.
+    // * the second mapping, pointed by mapped_buf, we get by opening the IPC handle in the consumer pool.
+
+    umf_result = umfPutIPCHandle(ipc_handle);
+    if (umf_result != UMF_RESULT_SUCCESS) {
+        fprintf(stderr, "Failed to put IPC handle!\n");
+        return -1;
+    }
+
+    fprintf(stdout, "IPC handle released in the producer pool.\n");
+
+    umf_result = umfCloseIPCHandle(mapped_buf);
+    if (umf_result != UMF_RESULT_SUCCESS) {
+        fprintf(stderr, "Failed to close IPC handle!\n");
+        return -1;
+    }
+
+    fprintf(stdout, "IPC handle closed in the consumer pool.\n");
+
+    umfFree(initial_buf);
+
+    umfPoolDestroy(producer_pool);
+    umfPoolDestroy(consumer_pool);
+
+    ret = destroy_context(producer_context);
+    if (ret != 0) {
+        fprintf(stderr, "Failed to destroy L0 context!\n");
+        return ret;
+    }
+
+    ret = destroy_context(consumer_context);
+    if (ret != 0) {
+        fprintf(stderr, "Failed to destroy L0 context!\n");
+        return ret;
+    }
+    return 0;
+}

scripts/docs_config/examples.rst

@@ -127,8 +127,86 @@ in the UMF repository.
 
 TODO
 
+IPC example with Level Zero Memory Provider
+==============================================================================
+The full code of the example is in the `examples/basic/ipc_level_zero.c`_ file in the UMF repository.
+The example demonstrates how to use UMF :ref:`IPC API <ipc-api>`. For demonstration purpose the example uses
+Level Zero memory provider to instantiate a pool. But the same flow will work with any memory provider that
+supports IPC capabilities.
+
+Here we omit describing how memory pools are created as its orthogonal to the IPC API usage. For more information
+on how to create memory pools refer to the previous examples. Also for simplification, our example is single process
+while :ref:`IPC API <ipc-api>` targeted for interprocess communication when IPC handle is created by one process
+to be used in another process.
+
+To use :ref:`IPC API <ipc-api>` the `umf/ipc.h`_ header should be included.
+
+.. code-block:: c
+
+   #include <umf/ipc.h>
+
+To get IPC handle for the memory allocated by UMF the :any:`umfGetIPCHandle` function should be used.
+
+.. code-block:: c
+
+    umf_ipc_handle_t ipc_handle = NULL;
+    size_t handle_size = 0;
+    umf_result_t umf_result = umfGetIPCHandle(initial_buf, &ipc_handle, &handle_size);
+
+The :any:`umfGetIPCHandle` function requires only the memory pointer as an input parameter and internally determines
+the memory pool to which the memory region belongs. While in our example the :any:`umfPoolMalloc` function is called
+a few lines before the :any:`umfGetIPCHandle` function is called, in a real application, memory might be allocated even
+by a different library and the caller of the :any:`umfGetIPCHandle` function may not know the corresponding memory pool.
+
+The :any:`umfGetIPCHandle` function returns the IPC handle and its size. The IPC handle is a byte-copyable opaque
+data structure. The :any:`umf_ipc_handle_t` type is defined as a pointer to a byte array. The size of the handle
+might be different for different memory provider types. The code snippet below demonstrates how the IPC handle can
+be serialized for marshalling purposes.
+
+.. code-block:: c
+
+    // Serialize IPC handle
+    void *serialized_ipc_handle = malloc(handle_size);
+    memcpy(serialized_ipc_handle, (void*)ipc_handle, handle_size);
+
+.. note::
+    The method of sending the IPC handle between processes is not defined by the UMF.
+
+When the IPC handle is transferred
+to another process it can be opened by the :any:`umfOpenIPCHandle` function.
+
+.. code-block:: c
+
+    void *mapped_buf = NULL;
+    umf_result = umfOpenIPCHandle(consumer_pool, ipc_handle, &mapped_buf);
+
+The :any:`umfOpenIPCHandle` function requires the memory pool handle and the IPC handle as input parameters. It mapps
+the handle to the current process address space and returns the pointer to the same memory region that was allocated
+in the producer process.
+
+.. note::
+    The virtual addresses of the memory region referred to by the IPC handle may not be the same in the producer and consumer processes.
+
+To release IPC handle on the producer side the :any:`umfPutIPCHandle` function should be used.
+
+.. code-block:: c
+
+    umf_result = umfPutIPCHandle(ipc_handle);
+
+To close IPC handle on the consumer side the :any:`umfCloseIPCHandle` function should be used.
+
+.. code-block:: c
+
+    umf_result = umfCloseIPCHandle(mapped_buf);
+
+The :any:`umfPutIPCHandle` function on the producer side might be called even before the :any:`umfCloseIPCHandle`
+function is called on the consumer side. The memory mappings on the consumer side remains valid until
+the :any:`umfCloseIPCHandle` function is called.
+
 .. _examples/basic/basic.c: https://github.com/oneapi-src/unified-memory-framework/blob/main/examples/basic/basic.c
 .. _examples/basic/gpu_shared_memory.c: https://github.com/oneapi-src/unified-memory-framework/blob/main/examples/basic/gpu_shared_memory.c
+.. _examples/basic/ipc_level_zero.c: https://github.com/oneapi-src/unified-memory-framework/blob/main/examples/basic/ipc_level_zero.c
 .. _README: https://github.com/oneapi-src/unified-memory-framework/blob/main/README.md#memory-pool-managers
+.. _umf/ipc.h: https://github.com/oneapi-src/unified-memory-framework/blob/main/include/umf/ipc.h
 .. _provider_os_memory.h: https://github.com/oneapi-src/unified-memory-framework/blob/main/include/umf/providers/provider_os_memory.h
 .. _pool_scalable.h: https://github.com/oneapi-src/unified-memory-framework/blob/main/include/umf/pools/pool_scalable.h