[SYCL] Support building sycl runtime with libcxx in configure script (#2731)

Currently, we claim to support building sycl runtime with llvm libc++ as an experimental feature but building with libcxx will fail in latest sources. This patch includes:

Adding support in configure script to build sycl runtime with llvm libc++. We need to manually construct the cmake commands previously.
Fix some building failures:
disabling xpti module when building sycl runtime with libcxx temporarily
use libcxx to compile object files required by sycl runtime library during link time.

Signed-off-by: gejin <[email protected]>

Author: jinge90

buildbot/configure.py

@@ -13,6 +13,10 @@ def do_configure(args):
     if not os.path.isdir(abs_obj_dir):
       os.makedirs(abs_obj_dir)
 
+    llvm_external_projects = 'sycl;llvm-spirv;opencl-aot;libdevice'
+    if not args.use_libcxx:
+        llvm_external_projects += ';xpti;xptifw'
+
     llvm_dir = os.path.join(abs_src_dir, "llvm")
     sycl_dir = os.path.join(abs_src_dir, "sycl")
     spirv_dir = os.path.join(abs_src_dir, "llvm-spirv")
@@ -22,14 +26,15 @@ def do_configure(args):
     ocl_header_dir = os.path.join(abs_obj_dir, "OpenCL-Headers")
     icd_loader_lib = os.path.join(abs_obj_dir, "OpenCL-ICD-Loader", "build")
     llvm_targets_to_build = 'X86'
-    llvm_enable_projects = 'clang;llvm-spirv;sycl;opencl-aot;xpti;xptifw;libdevice'
+    llvm_enable_projects = 'clang;' + llvm_external_projects
     libclc_targets_to_build = ''
     sycl_build_pi_cuda = 'OFF'
     sycl_werror = 'ON'
     llvm_enable_assertions = 'ON'
     llvm_enable_doxygen = 'OFF'
     llvm_enable_sphinx = 'OFF'
     llvm_build_shared_libs = 'OFF'
+    sycl_enable_xpti_tracing = 'OFF' if args.use_libcxx else 'ON'
 
     icd_loader_lib = os.path.join(icd_loader_lib, "libOpenCL.so" if platform.system() == 'Linux' else "OpenCL.lib")
 
@@ -64,7 +69,7 @@ def do_configure(args):
         "-DCMAKE_BUILD_TYPE={}".format(args.build_type),
         "-DLLVM_ENABLE_ASSERTIONS={}".format(llvm_enable_assertions),
         "-DLLVM_TARGETS_TO_BUILD={}".format(llvm_targets_to_build),
-        "-DLLVM_EXTERNAL_PROJECTS=sycl;llvm-spirv;opencl-aot;xpti;xptifw;libdevice",
+        "-DLLVM_EXTERNAL_PROJECTS={}".format(llvm_external_projects),
         "-DLLVM_EXTERNAL_SYCL_SOURCE_DIR={}".format(sycl_dir),
         "-DLLVM_EXTERNAL_LLVM_SPIRV_SOURCE_DIR={}".format(spirv_dir),
         "-DLLVM_EXTERNAL_XPTI_SOURCE_DIR={}".format(xpti_dir),
@@ -81,7 +86,7 @@ def do_configure(args):
         "-DLLVM_ENABLE_DOXYGEN={}".format(llvm_enable_doxygen),
         "-DLLVM_ENABLE_SPHINX={}".format(llvm_enable_sphinx),
         "-DBUILD_SHARED_LIBS={}".format(llvm_build_shared_libs),
-        "-DSYCL_ENABLE_XPTI_TRACING=ON" # Explicitly turn on XPTI tracing
+        "-DSYCL_ENABLE_XPTI_TRACING={}".format(sycl_enable_xpti_tracing)
     ]
 
     if args.system_ocl:
@@ -104,6 +109,15 @@ def do_configure(args):
     # Add path to root CMakeLists.txt
     cmake_cmd.append(llvm_dir)
 
+    if args.use_libcxx:
+      if not (args.libcxx_include and args.libcxx_library):
+        sys.exit("Please specify include and library path of libc++ when building sycl "
+                 "runtime with it")
+      cmake_cmd.extend([
+            "-DSYCL_USE_LIBCXX=ON",
+            "-DSYCL_LIBCXX_INCLUDE_PATH={}".format(args.libcxx_include),
+            "-DSYCL_LIBCXX_LIBRARY_PATH={}".format(args.libcxx_library)])
+
     print("[Cmake Command]: {}".format(" ".join(cmake_cmd)))
 
     try:
@@ -143,7 +157,9 @@ def main():
     parser.add_argument("--shared-libs", action='store_true', help="Build shared libraries")
     parser.add_argument("--cmake-opt", action='append', help="Additional CMake option not configured via script parameters")
     parser.add_argument("--cmake-gen", default="Ninja", help="CMake generator")
-
+    parser.add_argument("--use-libcxx", action="store_true", help="build sycl runtime with libcxx")
+    parser.add_argument("--libcxx-include", metavar="LIBCXX_INCLUDE_PATH", help="libcxx include path")
+    parser.add_argument("--libcxx-library", metavar="LIBCXX_LIBRARY_PATH", help="libcxx library path")
     args = parser.parse_args()
 
     print("args:{}".format(args))

sycl/CMakeLists.txt

@@ -227,13 +227,17 @@ function( add_common_options LIB_NAME)
   if (SYCL_USE_LIBCXX)
       if ((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") OR
           (CMAKE_CXX_COMPILER_ID STREQUAL "Clang"))
-          target_compile_options(${LIB_NAME} PRIVATE -nostdinc++)
           if ((NOT (DEFINED SYCL_LIBCXX_INCLUDE_PATH)) OR (NOT (DEFINED SYCL_LIBCXX_LIBRARY_PATH)))
               message(FATAL_ERROR "When building with libc++ SYCL_LIBCXX_INCLUDE_PATHS and"
                                   "SYCL_LIBCXX_LIBRARY_PATH should be set")
           endif()
+          target_link_libraries(${LIB_NAME} PRIVATE "-L${SYCL_LIBCXX_LIBRARY_PATH}" -Wl,-rpath,${SYCL_LIBCXX_LIBRARY_PATH} -nodefaultlibs -lc++ -lc++abi -lm -lc -lgcc_s -lgcc)
+          target_compile_options(${LIB_NAME} PRIVATE -nostdinc++)
           target_include_directories(${LIB_NAME} PRIVATE "${SYCL_LIBCXX_INCLUDE_PATH}")
-          target_link_libraries(${LIB_NAME} PRIVATE "-L${SYCL_LIBCXX_LIBRARY_PATH}" -nodefaultlibs -lc++ -lc++abi -lm -lc -lgcc_s -lgcc)
+          if (ARGC EQUAL 2)
+            target_compile_options(${ARGV1} PRIVATE -nostdinc++)
+            target_include_directories(${ARGV1} PRIVATE "${SYCL_LIBCXX_INCLUDE_PATH}")
+          endif()
       else()
           message(FATAL_ERROR "Build with libc++ is not yet supported for this compiler")
       endif()
@@ -243,7 +247,7 @@ function( add_common_options LIB_NAME)
 # More information https://bugs.launchpad.net/ubuntu/+source/gcc-5/+bug/1568899
     if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND
         CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 5.0)
-    target_link_libraries(${LIB_NAME} PRIVATE gcc_s gcc)
+        target_link_libraries(${ARGV0} PRIVATE gcc_s gcc)
     endif()
 
   endif()

sycl/source/CMakeLists.txt

@@ -84,7 +84,7 @@ function(add_sycl_rt_library LIB_NAME)
     PUBLIC
       $<$<BOOL:${SYCL_BUILD_PI_CUDA}>:USE_PI_CUDA>)
 
-  add_common_options(${LIB_NAME})
+  add_common_options(${LIB_NAME} ${LIB_OBJ_NAME})
 
   set_target_properties(${LIB_NAME} PROPERTIES
                         VERSION ${SYCL_VERSION_STRING}

sycl/test/abi/layout_accessors_device.cpp

@@ -1,5 +1,6 @@
 // RUN: %clangxx -fsycl -fsycl-device-only -c -fno-color-diagnostics -Xclang -fdump-record-layouts %s | FileCheck %s
 // REQUIRES: linux
+// UNSUPPORTED: libcxx
 
 // clang-format off
 

sycl/test/abi/layout_accessors_host.cpp

@@ -1,5 +1,6 @@
 // RUN: %clangxx -fsycl -c -fno-color-diagnostics -Xclang -fdump-record-layouts %s | FileCheck %s
 // REQUIRES: linux
+// UNSUPPORTED: libcxx
 
 // clang-format off
 

sycl/test/abi/layout_array.cpp

@@ -1,6 +1,7 @@
 // RUN: %clangxx -fsycl -c -fno-color-diagnostics -Xclang -fdump-record-layouts %s | FileCheck %s
 // RUN: %clangxx -fsycl -fsycl-device-only -c -fno-color-diagnostics -Xclang -fdump-record-layouts %s | FileCheck %s
 // REQUIRES: linux
+// UNSUPPORTED: libcxx
 
 // clang-format off
 

sycl/test/abi/layout_buffer.cpp

@@ -1,5 +1,6 @@
 // RUN: %clangxx -fsycl -c -fno-color-diagnostics -Xclang -fdump-record-layouts %s | FileCheck %s
 // REQUIRES: linux
+// UNSUPPORTED: libcxx
 
 // clang-format off
 

sycl/test/abi/layout_handler.cpp

@@ -1,5 +1,6 @@
 // RUN: %clangxx -fsycl -c -fno-color-diagnostics -Xclang -fdump-record-layouts %s | FileCheck %s
 // REQUIRES: linux
+// UNSUPPORTED: libcxx
 
 #include <CL/sycl/handler.hpp>
 #include <CL/sycl/queue.hpp>

sycl/test/abi/layout_image.cpp

@@ -1,5 +1,6 @@
 // RUN: %clangxx -fsycl -c -fno-color-diagnostics -Xclang -fdump-record-layouts %s | FileCheck %s
 // REQUIRES: linux
+// UNSUPPORTED: libcxx
 
 // clang-format off
 

sycl/test/abi/layout_vec.cpp

@@ -1,6 +1,7 @@
 // RUN: %clangxx -fsycl -c -fno-color-diagnostics -Xclang -fdump-record-layouts %s | FileCheck %s
 // RUN: %clangxx -fsycl -fsycl-device-only -c -fno-color-diagnostics -Xclang -fdump-record-layouts %s | FileCheck %s
 // REQUIRES: linux
+// UNSUPPORTED: libcxx
 
 // clang-format off
 

sycl/test/abi/pi_level_zero_symbol_check.dump

@@ -5,6 +5,7 @@
 
 # RUN: env LLVM_BIN_PATH=%llvm_build_bin_dir python %sycl_tools_src_dir/abi_check.py --mode check_symbols --reference %s %sycl_libs_dir/libpi_level_zero.so
 # REQUIRES: linux
+# UNSUPPORTED: libcxx
 
 piContextCreate
 piContextGetInfo

sycl/test/abi/pi_opencl_symbol_check.dump

@@ -5,6 +5,7 @@
 
 # RUN: env LLVM_BIN_PATH=%llvm_build_bin_dir python %sycl_tools_src_dir/abi_check.py --mode check_symbols --reference %s %sycl_libs_dir/libpi_opencl.so
 # REQUIRES: linux
+# UNSUPPORTED: libcxx
 
 piContextCreate
 piDeviceGetInfo

sycl/test/abi/sycl_symbols_linux.dump

@@ -5,6 +5,7 @@
 
 # RUN: env LLVM_BIN_PATH=%llvm_build_bin_dir python %sycl_tools_src_dir/abi_check.py --mode check_symbols --reference %s %sycl_libs_dir/libsycl.so
 # REQUIRES: linux
+# UNSUPPORTED: libcxx
 
 _Z20__spirv_ocl_prefetchPKcm
 _Z21__spirv_MemoryBarrierN5__spv5ScopeEj

sycl/test/abi/symbol_size_alignment.cpp

@@ -1,4 +1,5 @@
 // RUN: %clangxx -fsycl %s -o %t
+// UNSUPPORTED: libcxx
 
 // Changing symbol size or alignment is a breaking change. If it happens, refer
 // to the ABI Policy Guide for further instructions on breaking ABI.

sycl/test/abi/user_mangling.cpp

@@ -2,6 +2,7 @@
 // RUN: %clangxx -fsycl -c -emit-llvm -S -o - %s | FileCheck %s --check-prefix CHK-HOST
 // RUN: %clangxx -fsycl -fsycl-device-only -O0 -c -emit-llvm -S -o - %s | FileCheck %s --check-prefix CHK-DEVICE
 // REQUIRES: linux
+// UNSUPPORTED: libcxx
 
 #include <CL/sycl.hpp>
 

sycl/test/lit.cfg.py

@@ -39,8 +39,13 @@
 llvm_config.with_system_environment(['PATH', 'OCL_ICD_FILENAMES', 'SYCL_DEVICE_ALLOWLIST', 'SYCL_CONFIG_FILE_NAME'])
 
 # Configure LD_LIBRARY_PATH or corresponding os-specific alternatives
+# Add 'libcxx' feature to filter out all SYCL abi tests when SYCL runtime
+# is built with llvm libcxx. This feature is added for Linux only since MSVC
+# CL compiler doesn't support to use llvm libcxx instead of MSVC STL.
 if platform.system() == "Linux":
     config.available_features.add('linux')
+    if config.sycl_use_libcxx == "ON":
+        config.available_features.add('libcxx')
     llvm_config.with_system_environment('LD_LIBRARY_PATH')
     llvm_config.with_environment('LD_LIBRARY_PATH', config.sycl_libs_dir, append_path=True)
     llvm_config.with_system_environment('CFLAGS')

sycl/test/lit.site.cfg.py.in

@@ -22,6 +22,7 @@ config.sycl_clang_extra_flags = "@SYCL_CLANG_EXTRA_FLAGS@"
 config.llvm_enable_projects = "@LLVM_ENABLE_PROJECTS@"
 
 config.sycl_threads_lib = '@SYCL_THREADS_LIB@'
+config.sycl_use_libcxx = '@SYCL_USE_LIBCXX@'
 
 import lit.llvm
 lit.llvm.initialize(lit_config, config)