Merge pull request #1820 from pbalcer/static-linking

Add support for static linking of the L0 adapter

Author: pbalcer

.github/workflows/build-hw-reusable.yml

@@ -18,6 +18,10 @@ on:
         required: false
         type: string
         default: OFF
+      static_adapter:
+        required: false
+        type: string
+        default: OFF
 
 permissions:
   contents: read
@@ -36,7 +40,7 @@ jobs:
     strategy:
       matrix:
         adapter: [
-          {name: "${{inputs.adapter_name}}", platform: "${{inputs.platform}}", static_Loader: "${{inputs.static_loader}}"},
+          {name: "${{inputs.adapter_name}}", platform: "${{inputs.platform}}", static_Loader: "${{inputs.static_loader}}", static_adapter: "${{inputs.static_loader}}"},
         ]
         build_type: [Debug, Release]
         compiler: [{c: gcc, cxx: g++}, {c: clang, cxx: clang++}]
@@ -49,6 +53,10 @@ jobs:
            build_type: Release
          - adapter: {static_Loader: ON}
            compiler: {c: clang, cxx: clang++}
+         - adapter: {static_adapter: ON}
+           build_type: Release
+         - adapter: {static_adapter: ON}
+           compiler: {c: clang, cxx: clang++}
 
     runs-on: ${{inputs.runner_name}}
 
@@ -76,6 +84,7 @@ jobs:
         -DUR_BUILD_TESTS=ON
         -DUR_BUILD_ADAPTER_${{matrix.adapter.name}}=ON
         -DUR_STATIC_LOADER=${{matrix.adapter.static_Loader}}
+        -DUR_STATIC_ADAPTER_${{matrix.adapter.name}}=${{matrix.adapter.static_adapter}}
         -DUR_DPCXX=${{github.workspace}}/dpcpp_compiler/bin/clang++
         -DUR_SYCL_LIBRARY_DIR=${{github.workspace}}/dpcpp_compiler/lib
         ${{ matrix.adapter.name == 'HIP' && '-DUR_CONFORMANCE_AMD_ARCH=gfx1030' || '' }}

.github/workflows/cmake.yml

@@ -155,6 +155,7 @@ jobs:
       adapter_name: L0
       runner_name: L0
       static_loader: ON
+      static_adapter: ON
 
   opencl:
     name: OpenCL
@@ -216,7 +217,8 @@ jobs:
         os: ['windows-2019', 'windows-2022']
         adapter: [
           {name: None, var: ''}, {name: L0, var: '-DUR_BUILD_ADAPTER_L0=ON'},
-          {name: None, var: ''}, {name: L0_V2, var: '-DUR_BUILD_ADAPTER_L0_V2=ON'}
+          {name: None, var: ''}, {name: L0_V2, var: '-DUR_BUILD_ADAPTER_L0_V2=ON'},
+          {name: L0, var: '-DUR_BUILD_ADAPTER_L0=ON -DUR_STATIC_ADAPTER_L0=ON'}
         ]
 
         # TODO: building level zero loader on windows-2019 and clang-cl is currently broken
@@ -225,16 +227,25 @@ jobs:
            adapter: {name: L0, var: '-DUR_BUILD_ADAPTER_L0=ON'}
          - os: 'windows-2019'
            adapter: {name: L0_V2, var: '-DUR_BUILD_ADAPTER_L0_V2=ON'}
+         - os: 'windows-2019'
+           adapter: {name: L0, var: '-DUR_BUILD_ADAPTER_L0=ON -DUR_STATIC_ADAPTER_L0=ON'}
          - adapter: {name: L0, var: '-DUR_BUILD_ADAPTER_L0=ON'}
            compiler: {c: clang-cl, cxx: clang-cl}
          - adapter: {name: L0_V2, var: '-DUR_BUILD_ADAPTER_L0_V2=ON'}
            compiler: {c: clang-cl, cxx: clang-cl}
+         - adapter: {name: L0, var: '-DUR_BUILD_ADAPTER_L0=ON -DUR_STATIC_ADAPTER_L0=ON'}
+           compiler: {c: clang-cl, cxx: clang-cl}
 
         build_type: [Debug, Release]
         compiler: [{c: cl, cxx: cl}, {c: clang-cl, cxx: clang-cl}]
         include:
           - compiler: {c: clang-cl, cxx: clang-cl}
             toolset: "-T ClangCL"
+          - os: 'windows-2022'
+            adapter: {name: L0, var: '-DUR_BUILD_ADAPTER_L0=ON -DUR_STATIC_ADAPTER_L0=ON'}
+            build_type: 'Release'
+            compiler: {c: cl, cxx: cl}
+
     runs-on: ${{matrix.os}}
 
     steps:

CMakeLists.txt

@@ -52,6 +52,7 @@ option(UR_BUILD_ADAPTER_HIP "Build the HIP adapter" OFF)
 option(UR_BUILD_ADAPTER_NATIVE_CPU "Build the Native-CPU adapter" OFF)
 option(UR_BUILD_ADAPTER_ALL "Build all currently supported adapters" OFF)
 option(UR_BUILD_ADAPTER_L0_V2 "Build the (experimental) Level-Zero v2 adapter" OFF)
+option(UR_STATIC_ADAPTER_L0 "Build the Level-Zero adapter as static and embed in the loader" OFF)
 option(UR_BUILD_EXAMPLE_CODEGEN "Build the codegen example." OFF)
 option(VAL_USE_LIBBACKTRACE_BACKTRACE "enable libbacktrace validation backtrace for linux" OFF)
 option(UR_ENABLE_ASSERTIONS "Enable assertions for all build types" OFF)

scripts/generate_code.py

@@ -400,6 +400,32 @@ def generate_loader(path, section, namespace, tags, version, specs, meta):
     )
     print("Generated %s lines of code.\n"%loc)
 
+"""
+    generates c/c++ files from the specification documents
+"""
+def _mako_interface_loader_api(path, adapter, ext, namespace, tags, version, specs, meta):
+    dstpath = os.path.join(path, adapter)
+    os.makedirs(dstpath, exist_ok=True)
+
+    template = f"ur_interface_loader.{ext}.mako"
+    fin = os.path.join(templates_dir, template)
+
+    name = f"ur_interface_loader"
+
+    filename = f"{name}.{ext}"
+    fout = os.path.join(dstpath, filename)
+
+    print("Generating %s..."%fout)
+    return util.makoWrite(
+        fin, fout,
+        name=name,
+        adapter=adapter,
+        ver=version,
+        namespace=namespace,
+        tags=tags,
+        specs=specs,
+        meta=meta,)
+
 """
 Entry-point:
     generates adapter for unified_runtime
@@ -416,6 +442,10 @@ def generate_adapters(path, section, namespace, tags, version, specs, meta):
     loc += _mako_linker_scripts(
         dstpath, "adapter", "def", namespace, tags, version, specs, meta
     )
+
+    loc += _mako_interface_loader_api(dstpath, "level_zero", "cpp", namespace, tags, version, specs, meta)
+    loc += _mako_interface_loader_api(dstpath, "level_zero", "hpp", namespace, tags, version, specs, meta)
+
     print("Generated %s lines of code.\n"%loc)
 
 """

scripts/templates/ldrddi.cpp.mako

@@ -365,6 +365,10 @@ ${tbl['export']['name']}(
     // Load the device-platform DDI tables
     for( auto& platform : ur_loader::getContext()->platforms )
     {
+        // statically linked adapter inside of the loader
+        if (platform.handle == nullptr)
+            continue;
+
         if(platform.initStatus != ${X}_RESULT_SUCCESS)
             continue;
         auto getTable = reinterpret_cast<${tbl['pfn']}>(

scripts/templates/queue_api.cpp.mako

@@ -24,8 +24,9 @@ from templates import helper as th
 ur_queue_handle_t_::~ur_queue_handle_t_() {}
 
 ## FUNCTION ###################################################################
+namespace ${x}::level_zero {
 %for obj in th.get_queue_related_functions(specs, n, tags):
-${X}_APIEXPORT ${x}_result_t ${X}_APICALL
+${x}_result_t
 ${th.make_func_name(n, tags, obj)}(
     %for line in th.make_param_lines(n, tags, obj, format=["name", "type", "delim"]):
     ${line}
@@ -35,3 +36,4 @@ ${th.make_func_name(n, tags, obj)}(
     return ${obj['params'][0]['name']}->${th.transform_queue_related_function_name(n, tags, obj, format=["name"])};
 }
 %endfor
+}

scripts/templates/ur_interface_loader.cpp.mako

@@ -0,0 +1,88 @@
+<%!
+import re
+from templates import helper as th
+%><%
+    n=namespace
+    N=n.upper()
+
+    x=tags['$x']
+    X=x.upper()
+    Adapter=adapter.upper()
+%>//===--------- ${n}_interface_loader.cpp - Level Zero Adapter ------------===//
+//
+// Copyright (C) 2024 Intel Corporation
+//
+// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM
+// Exceptions. See LICENSE.TXT
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include <${n}_api.h>
+#include <${n}_ddi.h>
+
+#include "ur_interface_loader.hpp"
+
+static ur_result_t validateProcInputs(ur_api_version_t version, void *pDdiTable) {
+  if (nullptr == pDdiTable) {
+    return UR_RESULT_ERROR_INVALID_NULL_POINTER;
+  }
+  // Pre 1.0 we enforce loader and adapter must have same version.
+  // Post 1.0 only major version match should be required.
+  if (version != UR_API_VERSION_CURRENT) {
+    return UR_RESULT_ERROR_UNSUPPORTED_VERSION;
+  }
+  return UR_RESULT_SUCCESS;
+}
+
+#ifdef UR_STATIC_ADAPTER_${Adapter}
+namespace ${n}::${adapter} {
+#elif defined(__cplusplus)
+extern "C" {
+#endif
+
+%for tbl in th.get_pfntables(specs, meta, n, tags):
+${X}_APIEXPORT ${x}_result_t ${X}_APICALL ${tbl['export']['name']}(
+    %for line in th.make_param_lines(n, tags, tbl['export'], format=["type", "name", "delim"]):
+    ${line}
+    %endfor
+    )
+{
+    auto result = validateProcInputs(version, pDdiTable);
+    if (UR_RESULT_SUCCESS != result) {
+        return result;
+    }
+
+    %for obj in tbl['functions']:
+    pDdiTable->${th.append_ws(th.make_pfn_name(n, tags, obj), 43)} = ${n}::${adapter}::${th.make_func_name(n, tags, obj)};
+    %endfor
+
+    return result;
+}
+
+%endfor
+
+#ifdef UR_STATIC_ADAPTER_${Adapter}
+} // namespace ur::${adapter}
+#elif defined(__cplusplus)
+} // extern "C"
+#endif
+
+#ifdef UR_STATIC_ADAPTER_${Adapter}
+namespace ur::${adapter} {
+ur_result_t urAdapterGetDdiTables(ur_dditable_t *ddi) {
+  if (ddi == nullptr) {
+    return UR_RESULT_ERROR_INVALID_NULL_POINTER;
+  }
+
+  ur_result_t result;
+
+%for tbl in th.get_pfntables(specs, meta, n, tags):
+  result = ${n}::${adapter}::${tbl['export']['name']}( ${X}_API_VERSION_CURRENT, &ddi->${tbl['name']} );
+  if (result != UR_RESULT_SUCCESS)
+    return result;
+%endfor
+
+  return result;
+}
+}
+#endif

scripts/templates/ur_interface_loader.hpp.mako

@@ -0,0 +1,38 @@
+<%!
+import re
+from templates import helper as th
+%><%
+    n=namespace
+    N=n.upper()
+
+    x=tags['$x']
+    X=x.upper()
+    Adapter=adapter.upper()
+%>//===--------- ${n}_interface_loader.hpp - Level Zero Adapter ------------===//
+//
+// Copyright (C) 2024 Intel Corporation
+//
+// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM
+// Exceptions. See LICENSE.TXT
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include <${n}_api.h>
+#include <${n}_ddi.h>
+
+namespace ${n}::${adapter} {
+%for s in specs:
+%for obj in th.filter_items(s['objects'], 'type', 'function'):
+%if not th.obj_traits.is_loader_only(obj):
+${x}_result_t ${th.make_func_name(n, tags, obj)}(
+    %for line in th.make_param_lines(n, tags, obj, format=["type", "name", "delim"]):
+    ${line}
+    %endfor
+    );
+%endif
+%endfor
+%endfor
+#ifdef UR_STATIC_ADAPTER_LEVEL_ZERO
+ur_result_t urAdapterGetDdiTables(ur_dditable_t *ddi);
+#endif
+}

source/adapters/level_zero/CMakeLists.txt

@@ -73,27 +73,33 @@ if (NOT DEFINED LEVEL_ZERO_LIBRARY OR NOT DEFINED LEVEL_ZERO_INCLUDE_DIR)
         ${level-zero-loader_SOURCE_DIR}/include CACHE PATH "Path to Level Zero Headers")
 endif()
 
-add_library (LevelZeroLoader INTERFACE)
+add_library(LevelZeroLoader INTERFACE)
 # The MSVC linker does not like / at the start of a path, so to work around this
 # we split it into a link library and a library path, where the path is allowed
 # to have leading /.
 get_filename_component(LEVEL_ZERO_LIBRARY_SRC "${LEVEL_ZERO_LIBRARY}" DIRECTORY)
 get_filename_component(LEVEL_ZERO_LIB_NAME "${LEVEL_ZERO_LIBRARY}" NAME)
 target_link_directories(LevelZeroLoader
-    INTERFACE "${LEVEL_ZERO_LIBRARY_SRC}"
+    INTERFACE "$<BUILD_INTERFACE:${LEVEL_ZERO_LIBRARY_SRC}>"
+              "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>"
 )
 target_link_libraries(LevelZeroLoader
     INTERFACE "${LEVEL_ZERO_LIB_NAME}"
 )
 
-add_library (LevelZeroLoader-Headers INTERFACE)
+add_library(LevelZeroLoader-Headers INTERFACE)
 target_include_directories(LevelZeroLoader-Headers
-    INTERFACE "${LEVEL_ZERO_INCLUDE_DIR}"
+    INTERFACE "$<BUILD_INTERFACE:${LEVEL_ZERO_INCLUDE_DIR}>"
+              "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>"
 )
 
 if(UR_BUILD_ADAPTER_L0)
-    add_ur_adapter(ur_adapter_level_zero
-        SHARED
+    set(ADAPTER_LIB_TYPE SHARED)
+    if(UR_STATIC_ADAPTER_L0)
+        set(ADAPTER_LIB_TYPE STATIC)
+    endif()
+
+    add_ur_adapter(ur_adapter_level_zero ${ADAPTER_LIB_TYPE}
         ${CMAKE_CURRENT_SOURCE_DIR}/ur_interface_loader.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/adapter.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/adapter.cpp
@@ -133,6 +139,20 @@ if(UR_BUILD_ADAPTER_L0)
         ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.cpp
     )
 
+    if(UR_STATIC_ADAPTER_L0)
+        target_compile_definitions(ur_adapter_level_zero PUBLIC UR_STATIC_ADAPTER_LEVEL_ZERO)
+
+        # 'utils' target from 'level-zero-loader' includes path which is prefixed
+        # in the source directory, this breaks the installation of 'utils' target.
+        set_target_properties(utils PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "")
+        install(TARGETS ur_adapter_level_zero ur_umf LevelZeroLoader LevelZeroLoader-Headers ze_loader utils
+            EXPORT ${PROJECT_NAME}-targets
+            ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+            RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+            LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        )
+    endif()
+
     if(NOT WIN32)
         target_sources(ur_adapter_level_zero
             PRIVATE
@@ -152,7 +172,7 @@ if(UR_BUILD_ADAPTER_L0)
 
     if (WIN32)
     # 0x800: Search for the DLL only in the System32 folder
-    target_link_options(ur_adapter_level_zero PUBLIC /DEPENDENTLOADFLAG:0x800)
+    target_link_options(ur_adapter_level_zero PRIVATE /DEPENDENTLOADFLAG:0x800)
     endif()
 
     target_link_libraries(ur_adapter_level_zero PRIVATE