[libclc] Refactor build system to allow in-tree builds

The previous build system was adding custom "OpenCL" and "LLVM IR"
languages in CMake to build the builtin libraries. This was making it
harder to build in-tree because the tool binaries needed to be present
at configure time.

This commit refactors the build system to use custom commands to build
the bytecode files one by one, and link them all together into the final
bytecode library. It also enables in-tree builds by aliasing the
clang/llvm-link/etc. tool targets to internal targets, which are
imported from the LLVM installation directory when building out of tree.

Diffing (with llvm-diff) all of the final bytecode libraries in an
out-of-tree configuration against those built using the current tip
system shows no changes. Note that there are textual changes to metadata
IDs which confuse regular diff, and that llvm-diff 14 and below may show
false-positives.

This commit also removes a file listed in one of the SOURCEs which
didn't exist and which was preventing the use of ENABLE_RUNTIME_SUBORMAL
when configuring CMake.

Author: frasercrmck

libclc/CMakeLists.txt

@@ -4,6 +4,11 @@ project( libclc VERSION 0.2.0 LANGUAGES CXX C)
 
 set(CMAKE_CXX_STANDARD 17)
 
+# Add path for custom modules
+list( INSERT CMAKE_MODULE_PATH 0 "${PROJECT_SOURCE_DIR}/cmake/modules" )
+
+include( AddLibclc )
+
 include( GNUInstallDirs )
 set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS
   amdgcn-amdhsa/lib/SOURCES;
@@ -27,31 +32,47 @@ set( LIBCLC_TARGETS_TO_BUILD "all"
 
 option( ENABLE_RUNTIME_SUBNORMAL "Enable runtime linking of subnormal support." OFF )
 
-find_package(LLVM REQUIRED HINTS "${LLVM_CMAKE_DIR}")
-include(AddLLVM)
+if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
+  # Out-of-tree configuration
+  find_package(LLVM REQUIRED HINTS "${LLVM_CMAKE_DIR}")
+  include(AddLLVM)
 
-message( STATUS "libclc LLVM version: ${LLVM_PACKAGE_VERSION}" )
+  message( STATUS "libclc LLVM version: ${LLVM_PACKAGE_VERSION}" )
 
-if( ${LLVM_PACKAGE_VERSION} VERSION_LESS ${LIBCLC_MIN_LLVM} )
-  message( FATAL_ERROR "libclc needs at least LLVM ${LIBCLC_MIN_LLVM}" )
-endif()
+  if( ${LLVM_PACKAGE_VERSION} VERSION_LESS ${LIBCLC_MIN_LLVM} )
+    message( FATAL_ERROR "libclc needs at least LLVM ${LIBCLC_MIN_LLVM}" )
+  endif()
 
-find_program( LLVM_CLANG clang PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH )
-find_program( LLVM_AS llvm-as PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH )
-find_program( LLVM_LINK llvm-link PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH )
-find_program( LLVM_OPT opt PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH )
-find_program( LLVM_SPIRV llvm-spirv PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH )
+  # Import required tools as targets
+  foreach( tool clang llvm-as llvm-link opt )
+    find_program( LLVM_TOOL_${tool} ${tool} PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH )
+    add_executable( libclc::${tool} IMPORTED GLOBAL )
+    set_target_properties( libclc::${tool} PROPERTIES IMPORTED_LOCATION ${LLVM_TOOL_${tool}} )
+  endforeach()
+else()
+  # In-tree configuration
+  set( LLVM_PACKAGE_VERSION ${LLVM_VERSION} )
+
+  # Note that we check this later (for both build types) but we can provide a
+  # more useful error message when built in-tree. We assume that LLVM tools are
+  # always available so don't warn here.
+  if( NOT clang IN_LIST LLVM_ENABLE_PROJECTS )
+    message(FATAL_ERROR "Clang is not enabled, but is required to build libclc in-tree")
+  endif()
 
-# Print toolchain
-message( STATUS "libclc toolchain - clang: ${LLVM_CLANG}" )
-message( STATUS "libclc toolchain - llvm-as: ${LLVM_AS}" )
-message( STATUS "libclc toolchain - llvm-link: ${LLVM_LINK}" )
-message( STATUS "libclc toolchain - opt: ${LLVM_OPT}" )
-message( STATUS "libclc toolchain - llvm-spirv: ${LLVM_SPIRV}" )
-if( NOT LLVM_CLANG OR NOT LLVM_OPT OR NOT LLVM_AS OR NOT LLVM_LINK )
+  foreach( tool clang llvm-as llvm-link opt )
+    add_executable(libclc::${tool} ALIAS ${tool})
+  endforeach()
+endif()
+
+if( NOT TARGET libclc::clang OR NOT TARGET libclc::opt
+    OR NOT TARGET libclc::llvm-as OR NOT TARGET libclc::llvm-link )
   message( FATAL_ERROR "libclc toolchain incomplete!" )
 endif()
 
+# llvm-spirv is an optional dependency, used to build spirv-* targets.
+find_program( LLVM_SPIRV llvm-spirv PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH )
+
 # List of all targets. Note that some are added dynamically below.
 set( LIBCLC_TARGETS_ALL
   amdgcn--
@@ -66,7 +87,7 @@ set( LIBCLC_TARGETS_ALL
 )
 
 # mesa3d environment is only available since LLVM 4.0
-if( ${LLVM_PACKAGE_VERSION} VERSION_GREATER "3.9.0" )
+if( ${LLVM_PACKAGE_VERSION} VERSION_GREATER_EQUAL 4.0.0 )
   list( APPEND LIBCLC_TARGETS_ALL amdgcn-mesa-mesa3d )
 endif()
 
@@ -90,24 +111,9 @@ if( "spirv-mesa3d-" IN_LIST LIBCLC_TARGETS_TO_BUILD OR "spirv64-mesa3d-" IN_LIST
   endif()
 endif()
 
-set( CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake )
-set( CMAKE_CLC_COMPILER ${LLVM_CLANG} )
-set( CMAKE_CLC_ARCHIVE ${LLVM_LINK} )
-set( CMAKE_LLAsm_PREPROCESSOR ${LLVM_CLANG} )
-set( CMAKE_LLAsm_COMPILER ${LLVM_AS} )
-set( CMAKE_LLAsm_ARCHIVE ${LLVM_LINK} )
-
 # Construct LLVM version define
 set( LLVM_VERSION_DEFINE "-DHAVE_LLVM=0x${LLVM_VERSION_MAJOR}0${LLVM_VERSION_MINOR}" )
 
-
-# LLVM 13 enables standard includes by default
-if( ${LLVM_PACKAGE_VERSION} VERSION_GREATER "12.99.99" )
-  set( CMAKE_LLAsm_FLAGS "${CMAKE_LLAsm_FLAGS} -cl-no-stdinc" )
-  set( CMAKE_CLC_FLAGS "${CMAKE_CLC_FLAGS} -cl-no-stdinc" )
-endif()
-
-enable_language( CLC LLAsm )
 # This needs to be set before any target that needs it
 # We need to use LLVM_INCLUDE_DIRS here, because if we are linking to an
 # llvm build directory, this includes $src/llvm/include which is where all the
@@ -162,13 +168,18 @@ configure_file( libclc.pc.in libclc.pc @ONLY )
 install( FILES ${CMAKE_CURRENT_BINARY_DIR}/libclc.pc DESTINATION "${CMAKE_INSTALL_DATADIR}/pkgconfig" )
 install( DIRECTORY generic/include/clc DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" )
 
+set( LIBCLC_OBJFILE_DIR "${CMAKE_CURRENT_BINARY_DIR}/obj.libclc.dir" )
+
 if( ENABLE_RUNTIME_SUBNORMAL )
-  add_library( subnormal_use_default STATIC
-    generic/lib/subnormal_use_default.ll )
-  add_library( subnormal_disable STATIC
-    generic/lib/subnormal_disable.ll )
-  install( TARGETS subnormal_use_default subnormal_disable ARCHIVE
-    DESTINATION "${CMAKE_INSTALL_DATADIR}/clc" )
+  foreach( file subnormal_use_default subnormal_disable )
+    link_bc(
+       OUTPUT ${LIBCLC_OBJFILE_DIR}/${file}.bc
+       INPUTS ${PROJECT_SOURCE_DIR}/generic/lib/${file}.ll
+    )
+    add_custom_target( ${file}.bc ALL DEPENDS ${LIBCLC_OBJFILE_DIR}/${file}.bc )
+    install( FILES ${LIBCLC_OBJFILE_DIR}/${file}.bc ARCHIVE
+      DESTINATION "${CMAKE_INSTALL_DATADIR}/clc" )
+  endforeach()
 endif()
 
 find_package( Python3 REQUIRED COMPONENTS Interpreter )
@@ -205,7 +216,7 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
     list( APPEND dirs amdgpu )
   endif()
 
-  #nvptx is special
+  # nvptx is special
   if( ${ARCH} STREQUAL nvptx OR ${ARCH} STREQUAL nvptx64 )
     set( DARCH ptx )
   else()
@@ -226,38 +237,32 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
     endforeach()
   endforeach()
 
-  # Add the generated convert.cl here to prevent adding
-  # the one listed in SOURCES
+  # Add the generated convert.cl here to prevent adding the one listed in
+  # SOURCES
+  set( objects )   # A "set" of already-added input files
+  set( rel_files ) # Source directory input files, relative to the root dir
+  set( gen_files ) # Generated binary input files, relative to the binary dir
   if( NOT ${ARCH} STREQUAL "spirv" AND NOT ${ARCH} STREQUAL "spirv64" )
     if( NOT ENABLE_RUNTIME_SUBNORMAL AND NOT ${ARCH} STREQUAL "clspv" AND
         NOT ${ARCH} STREQUAL "clspv64" )
-      set( rel_files convert.cl )
-      set( objects convert.cl )
+      list( APPEND gen_files convert.cl )
+      list( APPEND objects convert.cl )
       list( APPEND rel_files generic/lib/subnormal_use_default.ll )
     elseif(${ARCH} STREQUAL "clspv" OR ${ARCH} STREQUAL "clspv64")
-      set( rel_files clspv-convert.cl )
-      set( objects clspv-convert.cl )
+      list( APPEND gen_files clspv-convert.cl )
+      list( APPEND objects clspv-convert.cl )
     endif()
-  else()
-    set( rel_files )
-    set( objects )
   endif()
 
   foreach( l ${source_list} )
     file( READ ${l} file_list )
     string( REPLACE "\n" ";" file_list ${file_list} )
     get_filename_component( dir ${l} DIRECTORY )
     foreach( f ${file_list} )
-      list( FIND objects ${f} found )
-      if( found EQUAL  -1 )
+      # Only add each file once, so that targets can 'specialize' builtins
+      if( NOT ${f} IN_LIST objects )
         list( APPEND objects ${f} )
         list( APPEND rel_files ${dir}/${f} )
-        # FIXME: This should really go away
-        file( TO_CMAKE_PATH ${PROJECT_SOURCE_DIR}/${dir}/${f} src_loc )
-        get_filename_component( fdir ${src_loc} DIRECTORY )
-
-        set_source_files_properties( ${dir}/${f}
-          PROPERTIES COMPILE_FLAGS "-I ${fdir}" )
       endif()
     endforeach()
   endforeach()
@@ -295,32 +300,67 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
       set( opt_flags -O3 )
     endif()
 
-    set( builtins_link_lib_tgt builtins.link.${arch_suffix} )
-    add_library( ${builtins_link_lib_tgt} STATIC ${rel_files} )
-    # Make sure we depend on the pseudo target to prevent
-    # multiple invocations
-    add_dependencies( ${builtins_link_lib_tgt} generate_convert.cl )
-    add_dependencies( ${builtins_link_lib_tgt} clspv-generate_convert.cl )
-    # CMake will turn this include into absolute path
-    target_include_directories( ${builtins_link_lib_tgt} PRIVATE
-      "generic/include" )
-    target_compile_definitions( ${builtins_link_lib_tgt} PRIVATE
-      "__CLC_INTERNAL" )
-    string( TOUPPER "-DCLC_${ARCH}" CLC_TARGET_DEFINE )
-    target_compile_definitions( ${builtins_link_lib_tgt} PRIVATE
-      ${CLC_TARGET_DEFINE} )
-    target_compile_options( ${builtins_link_lib_tgt} PRIVATE  -target
-      ${t} ${mcpu} -fno-builtin -nostdlib ${build_flags} )
-    set_target_properties( ${builtins_link_lib_tgt} PROPERTIES
-      LINKER_LANGUAGE CLC )
+    set( LIBCLC_ARCH_OBJFILE_DIR "${LIBCLC_OBJFILE_DIR}/${arch_suffix}" )
+    file( MAKE_DIRECTORY ${LIBCLC_ARCH_OBJFILE_DIR} )
+
+    string( TOUPPER "CLC_${ARCH}" CLC_TARGET_DEFINE )
+
+    list( APPEND build_flags
+      -D__CLC_INTERNAL
+      -D${CLC_TARGET_DEFINE}
+      -I${PROJECT_SOURCE_DIR}/generic/include
+      # FIXME: Fix libclc to not require disabling this noisy warnings
+      -Wno-bitwise-conditional-parentheses
+    )
+
+    set( bytecode_files "" )
+    foreach( file IN LISTS gen_files rel_files )
+      # We need to take each file and produce an absolute input file, as well
+      # as a unique architecture-specific output file. We deal with a mix of
+      # different input files, which makes this trickier.
+      if( ${file} IN_LIST gen_files )
+        # Generated files are given just as file names, which we must make
+        # absolute to the binary directory.
+        set( input_file ${CMAKE_CURRENT_BINARY_DIR}/${file} )
+        set( output_file "${LIBCLC_ARCH_OBJFILE_DIR}/${file}.o" )
+      else()
+        # Other files are originally relative to each SOURCE file, which are
+        # then make relative to the libclc root directory. We must normalize
+        # the path (e.g., ironing out any ".."), then make it relative to the
+        # root directory again, and use that relative path component for the
+        # binary path.
+        get_filename_component( abs_path ${file} ABSOLUTE BASE_DIR ${PROJECT_SOURCE_DIR} )
+        file( RELATIVE_PATH root_rel_path ${PROJECT_SOURCE_DIR} ${abs_path} )
+        set( input_file ${PROJECT_SOURCE_DIR}/${file} )
+        set( output_file "${LIBCLC_ARCH_OBJFILE_DIR}/${root_rel_path}.o" )
+      endif()
+
+      get_filename_component( file_dir ${file} DIRECTORY )
+
+      compile_to_bc(
+        TRIPLE ${t}
+        INPUT ${input_file}
+        OUTPUT ${output_file}
+        EXTRA_OPTS "${mcpu}" -fno-builtin -nostdlib
+                   "${build_flags}" -I${PROJECT_SOURCE_DIR}/${file_dir}
+      )
+      list(APPEND bytecode_files ${output_file})
+    endforeach()
+
+    set( builtins_link_lib_tgt builtins.link.${arch_suffix}.bc )
+
+    link_bc(
+      OUTPUT ${builtins_link_lib_tgt}
+      INPUTS ${bytecode_files}
+    )
 
     set( obj_suffix ${arch_suffix}.bc )
     set( builtins_opt_lib_tgt builtins.opt.${obj_suffix} )
 
     # Add opt target
     add_custom_command( OUTPUT ${builtins_opt_lib_tgt}
-      COMMAND ${LLVM_OPT} ${opt_flags} -o ${builtins_opt_lib_tgt}
-        $<TARGET_FILE:${builtins_link_lib_tgt}>
+      COMMAND libclc::opt ${opt_flags} -o ${builtins_opt_lib_tgt}
+        ${builtins_link_lib_tgt}
       DEPENDS ${builtins_link_lib_tgt} )
     add_custom_target( "opt.${obj_suffix}" ALL
       DEPENDS ${builtins_opt_lib_tgt} )