Merge pull request #671 from ldorau/Add_devdax_memory_provider

Add devdax memory provider

Author: bratpiorka

.github/workflows/devdax.yml

@@ -0,0 +1,66 @@
+# This workflow builds and tests the devdax memory provider.
+# It requires a DAX device (e.g. /dev/dax0.0) configured in the OS.
+# This DAX device should be specified using UMF_TESTS_DEVDAX_PATH and UMF_TESTS_DEVDAX_SIZE
+# CI environment variables.
+
+name: DevDax
+
+on: [workflow_call]
+
+permissions:
+  contents: read
+
+env:
+  UMF_TESTS_DEVDAX_PATH : "/dev/dax0.0"
+  UMF_TESTS_DEVDAX_SIZE : 1054867456
+  BUILD_DIR : "${{github.workspace}}/build"
+  INSTL_DIR : "${{github.workspace}}/../install-dir"
+
+jobs:
+  devdax:
+    name: Build
+    # run only on upstream; forks may not have a DAX device
+    if: github.repository == 'oneapi-src/unified-memory-framework'
+    strategy:
+      matrix:
+        build_type: [Debug, Release]
+        shared_library: ['ON', 'OFF']
+
+    runs-on: ["DSS-DEVDAX", "DSS-Ubuntu"]
+    steps:
+      - name: Check if the devdax exists
+        run: |
+          ndctl list -N --device-dax
+          ls -al ${{env.UMF_TESTS_DEVDAX_PATH}}
+
+      - name: Checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          fetch-depth: 0
+
+      - name: Configure build
+        run: >
+          cmake
+          -B ${{env.BUILD_DIR}}
+          -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}"
+          -DCMAKE_BUILD_TYPE=${{matrix.build_type}}
+          -DCMAKE_C_COMPILER=gcc
+          -DCMAKE_CXX_COMPILER=g++
+          -DUMF_BUILD_SHARED_LIBRARY=${{matrix.shared_library}}
+          -DUMF_BUILD_BENCHMARKS=OFF
+          -DUMF_BUILD_TESTS=ON
+          -DUMF_BUILD_GPU_TESTS=OFF
+          -DUMF_BUILD_GPU_EXAMPLES=OFF
+          -DUMF_FORMAT_CODE_STYLE=OFF
+          -DUMF_DEVELOPER_MODE=ON
+          -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
+          -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
+          -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
+          -DUMF_TESTS_FAIL_ON_SKIP=ON
+
+      - name: Build UMF
+        run: cmake --build ${{env.BUILD_DIR}} --config ${{matrix.build_type}} -j $(nproc)
+
+      - name: Run only devdax tests
+        working-directory: ${{env.BUILD_DIR}}
+        run: ctest -C ${{matrix.build_type}} -R devdax -V

.github/workflows/pr_push.yml

@@ -85,6 +85,9 @@ jobs:
     name: Basic builds
     needs: [FastBuild]
     uses: ./.github/workflows/basic.yml
+  DevDax:
+    needs: [FastBuild]
+    uses: ./.github/workflows/devdax.yml
   Sanitizers:
     needs: [FastBuild]
     uses: ./.github/workflows/sanitizers.yml

CMakeLists.txt

@@ -419,6 +419,9 @@ endif()
 
 if(NOT UMF_DISABLE_HWLOC)
     add_optional_symbol(umfOsMemoryProviderOps)
+    if(LINUX)
+        add_optional_symbol(umfDevDaxMemoryProviderOps)
+    endif()
 endif()
 
 add_subdirectory(src)

README.md

@@ -172,6 +172,22 @@ Additionally, required for tests:
 5) Required packages:
    - liblevel-zero-dev (Linux) or level-zero-sdk (Windows)
 
+#### DevDax memory provider (Linux only)
+
+A memory provider that provides memory from a device DAX (a character device file /dev/daxX.Y).
+It can be used when large memory mappings are needed.
+
+The DevDax memory provider does not support the free operation
+(`umfMemoryProviderFree()` always returns `UMF_RESULT_ERROR_NOT_SUPPORTED`),
+so it should be used with a pool manager that will take over
+the managing of the provided memory - for example the jemalloc pool
+with the `disable_provider_free` parameter set to true.
+
+##### Requirements
+
+1) Linux OS
+2) A character device file /dev/daxX.Y created in the OS.
+
 ### Memory pool managers
 
 #### Proxy pool (part of libumf)

include/umf/providers/provider_devdax_memory.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2024 Intel Corporation
+ *
+ * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+*/
+
+#ifndef UMF_DEVDAX_MEMORY_PROVIDER_H
+#define UMF_DEVDAX_MEMORY_PROVIDER_H
+
+#include <umf/providers/provider_os_memory.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/// @cond
+#define UMF_DEVDAX_RESULTS_START_FROM 2000
+/// @endcond
+
+/// @brief Memory provider settings struct
+typedef struct umf_devdax_memory_provider_params_t {
+    /// path of the device DAX
+    char *path;
+    /// size of the device DAX in bytes
+    size_t size;
+    /// combination of 'umf_mem_protection_flags_t' flags
+    unsigned protection;
+} umf_devdax_memory_provider_params_t;
+
+/// @brief Devdax Memory Provider operation results
+typedef enum umf_devdax_memory_provider_native_error {
+    UMF_DEVDAX_RESULT_SUCCESS = UMF_DEVDAX_RESULTS_START_FROM, ///< Success
+    UMF_DEVDAX_RESULT_ERROR_ALLOC_FAILED,        ///< Memory allocation failed
+    UMF_DEVDAX_RESULT_ERROR_ADDRESS_NOT_ALIGNED, ///< Allocated address is not aligned
+    UMF_DEVDAX_RESULT_ERROR_FREE_FAILED,         ///< Memory deallocation failed
+    UMF_DEVDAX_RESULT_ERROR_PURGE_FORCE_FAILED, ///< Force purging failed
+} umf_devdax_memory_provider_native_error_t;
+
+umf_memory_provider_ops_t *umfDevDaxMemoryProviderOps(void);
+
+/// @brief Create default params for the devdax memory provider
+static inline umf_devdax_memory_provider_params_t
+umfDevDaxMemoryProviderParamsDefault(char *path, size_t size) {
+    umf_devdax_memory_provider_params_t params = {
+        path, /* path of the device DAX */
+        size, /* size of the device DAX in bytes */
+        UMF_PROTECTION_READ | UMF_PROTECTION_WRITE, /* protection */
+    };
+
+    return params;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* UMF_DEVDAX_MEMORY_PROVIDER_H */

scripts/docs_config/api.rst

@@ -96,6 +96,14 @@ A memory provider that provides memory from L0 device.
 .. doxygenfile:: provider_level_zero.h
     :sections: define enum typedef func var
 
+DevDax Memory Provider
+------------------------------------------
+
+A memory provider that provides memory from a device DAX (a character device file /dev/daxX.Y).
+
+.. doxygenfile:: provider_devdax_memory.h
+    :sections: define enum typedef func var
+
 Memspace
 ==========================================
 

src/CMakeLists.txt

@@ -101,6 +101,7 @@ set(UMF_SOURCES_MACOSX libumf_linux.c)
 set(UMF_SOURCES_WINDOWS libumf_windows.c)
 
 set(UMF_SOURCES_COMMON_LINUX_MACOSX
+    provider/provider_devdax_memory.c
     provider/provider_os_memory.c
     provider/provider_os_memory_posix.c
     memtargets/memtarget_numa.c