[MLIR] Add support for libMLIR.so

Putting this up mainly for discussion on
how this should be done. I am interested in MLIR from
the Julia side and we currently have a strong preference
to dynamically linking against the LLVM shared library,
and would like to have a MLIR shared library.

This patch adds a new cmake function add_mlir_library()
which accumulates a list of targets to be compiled into
libMLIR.so.  Note that not all libraries make sense to
be compiled into libMLIR.so.  In particular, we want
to avoid libraries which primarily exist to support
certain tools (such as mlir-opt and mlir-cpu-runner).

Note that the resulting libMLIR.so depends on LLVM, but
does not contain any LLVM components.  As a result, it
is necessary to link with libLLVM.so to avoid linkage
errors. So, libMLIR.so requires LLVM_BUILD_LLVM_DYLIB=on

FYI, Currently it appears that LLVM_LINK_LLVM_DYLIB is broken
because mlir-tblgen is linked against libLLVM.so and
and independent LLVM components.

Previous version of this patch broke depencies on TableGen
targets.  This appears to be because it compiled all
libraries to OBJECT libraries (probably because cmake
is generating different target names).  Avoiding object
libraries results in correct dependencies.

(updated by Stephen Neuendorffer)

Differential Revision: https://reviews.llvm.org/D73130

Author: vchuravy

mlir/CMakeLists.txt

@@ -34,9 +34,11 @@ include_directories( ${MLIR_INCLUDE_DIR})
 
 add_subdirectory(include/mlir)
 add_subdirectory(lib)
-add_subdirectory(tools)
 add_subdirectory(unittests)
 add_subdirectory(test)
+# Tools needs to come late to ensure that MLIR_ALL_LIBS is populated.
+# Generally things after this point may depend on MLIR_ALL_LIBS or libMLIR.so.
+add_subdirectory(tools)
 
 if( LLVM_INCLUDE_EXAMPLES )
   add_subdirectory(examples)

mlir/cmake/modules/AddMLIR.cmake

@@ -49,14 +49,20 @@ function(add_mlir_dialect dialect dialect_doc_filename)
   add_dependencies(mlir-doc ${dialect_doc_filename}DocGen)
 endfunction()
 
+# Declare a library which can be compiled in libMLIR.so
+macro(add_mlir_library name)
+  set_property(GLOBAL APPEND PROPERTY MLIR_ALL_LIBS ${name})
+  add_llvm_library(${ARGV})
+endmacro(add_mlir_library)
+
 # Declare the library associated with a dialect.
 function(add_mlir_dialect_library name)
   set_property(GLOBAL APPEND PROPERTY MLIR_DIALECT_LIBS ${name})
-  add_llvm_library(${ARGV})
+  add_mlir_library(${ARGV})
 endfunction(add_mlir_dialect_library)
 
 # Declare the library associated with a conversion.
 function(add_mlir_conversion_library name)
   set_property(GLOBAL APPEND PROPERTY MLIR_CONVERSION_LIBS ${name})
-  add_llvm_library(${ARGV})
+  add_mlir_library(${ARGV})
 endfunction(add_mlir_conversion_library)

mlir/lib/Analysis/CMakeLists.txt

@@ -13,7 +13,7 @@ set(LLVM_OPTIONAL_SOURCES
   Verifier.cpp
   )
 
-add_llvm_library(MLIRAnalysis
+add_mlir_library(MLIRAnalysis
   CallGraph.cpp
   ControlFlowInterfaces.cpp
   InferTypeOpInterface.cpp
@@ -37,7 +37,7 @@ target_link_libraries(MLIRAnalysis
   MLIRLoopOps
   )
 
-add_llvm_library(MLIRLoopAnalysis
+add_mlir_library(MLIRLoopAnalysis
   AffineAnalysis.cpp
   AffineStructures.cpp
   LoopAnalysis.cpp

mlir/lib/Dialect/CMakeLists.txt

@@ -12,12 +12,11 @@ add_subdirectory(SPIRV)
 add_subdirectory(StandardOps)
 add_subdirectory(VectorOps)
 
-
 set(LLVM_OPTIONAL_SOURCES
   Traits.cpp
 )
 
-add_llvm_library(MLIRDialect
+add_mlir_library(MLIRDialect
   Traits.cpp
 
   ADDITIONAL_HEADER_DIRS

mlir/lib/EDSC/CMakeLists.txt

@@ -3,7 +3,7 @@ set(LLVM_OPTIONAL_SOURCES
   CoreAPIs.cpp
   )
 
-add_llvm_library(MLIREDSC
+add_mlir_library(MLIREDSC
   Builders.cpp
 
   ADDITIONAL_HEADER_DIRS
@@ -16,7 +16,7 @@ target_link_libraries(MLIREDSC
   MLIRSupport
   )
 
-add_llvm_library(MLIREDSCInterface
+add_mlir_library(MLIREDSCInterface
   CoreAPIs.cpp
 
   ADDITIONAL_HEADER_DIRS

mlir/lib/ExecutionEngine/CMakeLists.txt

@@ -6,7 +6,7 @@ set(LLVM_OPTIONAL_SOURCES
   )
 
 llvm_map_components_to_libnames(outlibs "nativecodegen" "IPO")
-add_llvm_library(MLIRExecutionEngine
+add_mlir_library(MLIRExecutionEngine
   ExecutionEngine.cpp
   OptUtils.cpp
 

mlir/lib/IR/CMakeLists.txt

@@ -1,5 +1,5 @@
 file(GLOB globbed *.c *.cpp)
-add_llvm_library(MLIRIR
+add_mlir_library(MLIRIR
   ${globbed}
 
   ADDITIONAL_HEADER_DIRS

mlir/lib/Parser/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_llvm_library(MLIRParser
+add_mlir_library(MLIRParser
   Lexer.cpp
   Parser.cpp
   Token.cpp

mlir/lib/Pass/CMakeLists.txt

@@ -1,5 +1,5 @@
 file(GLOB globbed *.c *.cpp)
-add_llvm_library(MLIRPass
+add_mlir_library(MLIRPass
   ${globbed}
 
   ADDITIONAL_HEADER_DIRS

mlir/lib/Quantizer/CMakeLists.txt

@@ -1,5 +1,5 @@
 # Support.
-add_llvm_library(MLIRQuantizerSupport
+add_mlir_library(MLIRQuantizerSupport
   Support/Configuration.cpp
   Support/ConstraintAnalysisGraph.cpp
   Support/Metadata.cpp
@@ -21,7 +21,7 @@ target_link_libraries(MLIRQuantizerSupport
   )
 
 # Configurations.
-add_llvm_library(MLIRQuantizerFxpMathConfig
+add_mlir_library(MLIRQuantizerFxpMathConfig
   Configurations/FxpMathConfig.cpp
 
   ADDITIONAL_HEADER_DIRS
@@ -42,7 +42,7 @@ target_link_libraries(MLIRQuantizerFxpMathConfig
   )
 
 # Transforms.
-add_llvm_library(MLIRQuantizerTransforms
+add_mlir_library(MLIRQuantizerTransforms
   Transforms/AddDefaultStatsTestPass.cpp
   Transforms/InferQuantizedTypesPass.cpp
   Transforms/RemoveInstrumentationPass.cpp

mlir/lib/Support/CMakeLists.txt

@@ -7,7 +7,7 @@ set(LLVM_OPTIONAL_SOURCES
   TranslateClParser.cpp
 )
 
-add_llvm_library(MLIRSupport
+add_mlir_library(MLIRSupport
   FileUtilities.cpp
   StorageUniquer.cpp
   ToolUtilities.cpp
@@ -20,7 +20,7 @@ target_link_libraries(MLIRSupport
   LLVMSupport
   ${LLVM_PTHREAD_LIB})
 
-add_llvm_library(MLIROptLib
+add_mlir_library(MLIROptLib
   MlirOptMain.cpp
 
   ADDITIONAL_HEADER_DIRS
@@ -34,7 +34,7 @@ target_link_libraries(MLIROptLib
   MLIRSupport
   )
 
-add_llvm_library(MLIRTranslateClParser
+add_mlir_library(MLIRTranslateClParser
   TranslateClParser.cpp
 
   ADDITIONAL_HEADER_DIRS

mlir/lib/Target/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_llvm_library(MLIRTargetLLVMIRModuleTranslation
+add_mlir_library(MLIRTargetLLVMIRModuleTranslation
   LLVMIR/DebugTranslation.cpp
   LLVMIR/ModuleTranslation.cpp
 
@@ -9,9 +9,15 @@ add_llvm_library(MLIRTargetLLVMIRModuleTranslation
   )
 target_link_libraries(MLIRTargetLLVMIRModuleTranslation
   PUBLIC
-  MLIRLLVMIR LLVMCore LLVMIRReader LLVMSupport LLVMTransformUtils
-  MLIRTranslation)
-add_llvm_library(MLIRTargetLLVMIR
+  MLIRLLVMIR
+  LLVMCore
+  LLVMIRReader
+  LLVMSupport
+  LLVMTransformUtils
+  MLIRTranslation
+  )
+
+add_mlir_library(MLIRTargetLLVMIR
   LLVMIR/ConvertFromLLVMIR.cpp
   LLVMIR/ConvertToLLVMIR.cpp
 
@@ -20,8 +26,10 @@ add_llvm_library(MLIRTargetLLVMIR
   )
 target_link_libraries(MLIRTargetLLVMIR
   PUBLIC
-  MLIRTargetLLVMIRModuleTranslation)
-add_llvm_library(MLIRTargetNVVMIR
+  MLIRTargetLLVMIRModuleTranslation
+  )
+
+add_mlir_library(MLIRTargetNVVMIR
   LLVMIR/ConvertToNVVMIR.cpp
 
   ADDITIONAL_HEADER_DIRS
@@ -37,7 +45,8 @@ target_link_libraries(MLIRTargetNVVMIR
   MLIRNVVMIR
   MLIRTargetLLVMIRModuleTranslation
   )
-add_llvm_library(MLIRTargetROCDLIR
+
+add_mlir_library(MLIRTargetROCDLIR
   LLVMIR/ConvertToROCDLIR.cpp
 
   ADDITIONAL_HEADER_DIRS

mlir/lib/Transforms/CMakeLists.txt

@@ -1,6 +1,6 @@
 add_subdirectory(Utils)
 
-add_llvm_library(MLIRTransforms
+add_mlir_library(MLIRTransforms
   AffineDataCopyGeneration.cpp
   AffineLoopInvariantCodeMotion.cpp
   Canonicalizer.cpp

mlir/lib/Transforms/Utils/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_llvm_library(MLIRTransformUtils
+add_mlir_library(MLIRTransformUtils
   FoldUtils.cpp
   GreedyPatternRewriteDriver.cpp
   InliningUtils.cpp

mlir/lib/Translation/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_llvm_library(MLIRTranslation
+add_mlir_library(MLIRTranslation
   Translation.cpp
 
   ADDITIONAL_HEADER_DIRS

mlir/tools/CMakeLists.txt

@@ -4,3 +4,4 @@ add_subdirectory(mlir-opt)
 add_subdirectory(mlir-tblgen)
 add_subdirectory(mlir-translate)
 add_subdirectory(mlir-vulkan-runner)
+add_subdirectory(mlir-shlib)

mlir/tools/mlir-opt/CMakeLists.txt

@@ -4,6 +4,7 @@ set(LLVM_OPTIONAL_SOURCES
 
 set(LIB_LIBS
   MLIRAnalysis
+  MLIRIR
   MLIRLLVMIR
   MLIROptLib
   MLIRParser

mlir/tools/mlir-shlib/CMakeLists.txt

@@ -0,0 +1,42 @@
+# Building libmlir-cpp.so fails if LLVM_ENABLE_PIC=Off
+if (NOT LLVM_ENABLE_PIC)
+  return()
+endif()
+
+# Building libmlir-cpp.so may not work on MSVC
+if (MSVC)
+  return()
+endif()
+
+get_property(mlir_libs GLOBAL PROPERTY MLIR_ALL_LIBS)
+list(REMOVE_DUPLICATES mlir_libs)
+
+foreach (lib ${mlir_libs})
+  if(XCODE)
+    # Xcode doesn't support object libraries, so we have to trick it into
+    # linking the static libraries instead.
+    list(APPEND _DEPS "-force_load" ${lib})
+  else()
+    list(APPEND _OBJECTS $<TARGET_OBJECTS:obj.${lib}>)
+  endif()
+  list(APPEND _DEPS $<TARGET_PROPERTY:${lib},LINK_LIBRARIES>)
+endforeach ()
+
+if(MLIR_LINK_MLIR_DYLIB)
+  set(INSTALL_WITH_TOOLCHAIN INSTALL_WITH_TOOLCHAIN)
+endif()
+
+# libMLIR.so depends on LLVM components.  To avoid multiple
+# copies of those LLVM components, libMLIR.so depends on libLLVM.so.
+# This probably won't work if some LLVM components are not included
+# in libLLVM.so.
+if(LLVM_BUILD_LLVM_DYLIB)
+  add_llvm_library(MLIR
+    SHARED
+    ${INSTALL_WITH_TOOLCHAIN}
+
+    mlir-shlib.cpp
+    )
+  target_link_libraries(MLIR PRIVATE LLVM ${LLVM_PTHREAD_LIB})
+  whole_archive_link(MLIR ${mlir_libs})
+endif()

mlir/tools/mlir-shlib/mlir-shlib.cpp

@@ -0,0 +1 @@
+// Intentionally empty source file to make CMake happy