[MLIR][LLVMIR] Add support for the full form of global_{ctor,dtor}

flang/lib/Optimizer/Transforms/CUFAddConstructor.cpp

@@ -23,6 +23,7 @@
 #include "flang/Runtime/CUDA/registration.h"
 #include "flang/Runtime/entry-names.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
+#include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/IR/Value.h"
 #include "mlir/Pass/Pass.h"
@@ -157,10 +158,12 @@ struct CUFAddConstructor
     funcs.push_back(
         mlir::FlatSymbolRefAttr::get(mod.getContext(), func.getSymName()));
     llvm::SmallVector<int> priorities;
+    llvm::SmallVector<mlir::Attribute> data;
     priorities.push_back(0);
+    data.push_back(mlir::LLVM::ZeroAttr::get(mod.getContext()));
     builder.create<mlir::LLVM::GlobalCtorsOp>(
         mod.getLoc(), builder.getArrayAttr(funcs),
-        builder.getI32ArrayAttr(priorities));
+        builder.getI32ArrayAttr(priorities), builder.getArrayAttr(data));
   }
 };
 

flang/test/Fir/CUDA/cuda-constructor.f90

@@ -9,4 +9,4 @@ program main
 ! CHECK:   llvm.call @_FortranACUFRegisterAllocator() : () -> ()
 ! CHECK:   llvm.return
 ! CHECK: }
-! CHECK: llvm.mlir.global_ctors {ctors = [@__cudaFortranConstructor], priorities = [0 : i32]}
+! CHECK: llvm.mlir.global_ctors ctors = [@__cudaFortranConstructor], priorities = [0 : i32], data = [#llvm.zero]

mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td

@@ -1438,61 +1438,73 @@ def LLVM_GlobalOp : LLVM_Op<"mlir.global",
 
 def LLVM_GlobalCtorsOp : LLVM_Op<"mlir.global_ctors", [
                            DeclareOpInterfaceMethods<SymbolUserOpInterface>]> {
-  let arguments = (ins FlatSymbolRefArrayAttr
-                   : $ctors, I32ArrayAttr
-                   : $priorities);
+  let arguments = (ins FlatSymbolRefArrayAttr:$ctors,
+                       I32ArrayAttr:$priorities,
+                       ArrayAttr:$data);
   let summary = "LLVM dialect global_ctors.";
   let description = [{
-    Specifies a list of constructor functions and priorities. The functions
-    referenced by this array will be called in ascending order of priority (i.e.
-    lowest first) when the module is loaded. The order of functions with the
-    same priority is not defined. This operation is translated to LLVM's
-    global_ctors global variable. The initializer functions are run at load
-    time. The `data` field present in LLVM's global_ctors variable is not
-    modeled here.
+    Specifies a list of constructor functions, priorities, and associated data.
+    The functions referenced by this array will be called in ascending order
+    of priority (i.e. lowest first) when the module is loaded. The order of
+    functions with the same priority is not defined. This operation is
+    translated to LLVM's global_ctors global variable. The initializer
+    functions are run at load time. However, if the associated data is not
+    `#llvm.zero`, functions only run if the data is not discarded.
 
     Examples:
 
     ```mlir
-    llvm.mlir.global_ctors {@ctor}
-
     llvm.func @ctor() {
       ...
       llvm.return
     }
+    llvm.mlir.global_ctors ctors = [@ctor], priorities = [0],
+                                   data = [#llvm.zero]
     ```
 
   }];
-  let assemblyFormat = "attr-dict";
+  let assemblyFormat = [{
+    `ctors` `=` $ctors
+    `,` `priorities` `=` $priorities
+    `,` `data` `=` $data
+    attr-dict
+  }];
   let hasVerifier = 1;
 }
 
 def LLVM_GlobalDtorsOp : LLVM_Op<"mlir.global_dtors", [
                            DeclareOpInterfaceMethods<SymbolUserOpInterface>]> {
   let arguments = (ins
     FlatSymbolRefArrayAttr:$dtors,
-    I32ArrayAttr:$priorities
+    I32ArrayAttr:$priorities,
+    ArrayAttr:$data
   );
   let summary = "LLVM dialect global_dtors.";
   let description = [{
     Specifies a list of destructor functions and priorities. The functions
-    referenced by this array will be called in descending order of priority (i.e.
-    highest first) when the module is unloaded. The order of functions with the
-    same priority is not defined. This operation is translated to LLVM's
-    global_dtors global variable. The `data` field present in LLVM's
-    global_dtors variable is not modeled here.
+    referenced by this array will be called in descending order of priority
+    (i.e. highest first) when the module is unloaded. The order of functions
+    with the same priority is not defined. This operation is translated to
+    LLVM's global_dtors global variable. The destruction functions are run at
+    load time. However, if the associated data is not `#llvm.zero`, functions
+    only run if the data is not discarded.
 
     Examples:
 
     ```mlir
     llvm.func @dtor() {
       llvm.return
     }
-    llvm.mlir.global_dtors {@dtor}
+    llvm.mlir.global_dtors dtors = [@dtor], priorities = [0],
+                                   data = [#llvm.zero]
     ```
-
   }];
-  let assemblyFormat = "attr-dict";
+  let assemblyFormat = [{
+    `dtors` `=` $dtors
+    `,` `priorities` `=` $priorities
+    `,` `data` `=` $data
+    attr-dict
+  }];
   let hasVerifier = 1;
 }
 

mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp

@@ -2508,6 +2508,17 @@ LogicalResult GlobalOp::verifyRegions() {
 // LLVM::GlobalCtorsOp
 //===----------------------------------------------------------------------===//
 
+LogicalResult checkGlobalXtorData(Operation *op, ArrayAttr data) {
+  if (data.empty())
+    return success();
+
+  if (llvm::all_of(data.getAsRange<Attribute>(), [](Attribute v) {
+        return isa<FlatSymbolRefAttr, ZeroAttr>(v);
+      }))
+    return success();
+  return op->emitError("data element must be symbol or #llvm.zero");
+}
+
 LogicalResult
 GlobalCtorsOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
   for (Attribute ctor : getCtors()) {
@@ -2519,10 +2530,14 @@ GlobalCtorsOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
 }
 
 LogicalResult GlobalCtorsOp::verify() {
-  if (getCtors().size() != getPriorities().size())
-    return emitError(
-        "mismatch between the number of ctors and the number of priorities");
-  return success();
+  if (checkGlobalXtorData(*this, getData()).failed())
+    return failure();
+
+  if (getCtors().size() == getPriorities().size() &&
+      getCtors().size() == getData().size())
+    return success();
+  return emitError(
+      "ctors, priorities, and data must have the same number of elements");
 }
 
 //===----------------------------------------------------------------------===//
@@ -2540,10 +2555,14 @@ GlobalDtorsOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
 }
 
 LogicalResult GlobalDtorsOp::verify() {
-  if (getDtors().size() != getPriorities().size())
-    return emitError(
-        "mismatch between the number of dtors and the number of priorities");
-  return success();
+  if (checkGlobalXtorData(*this, getData()).failed())
+    return failure();
+
+  if (getDtors().size() == getPriorities().size() &&
+      getDtors().size() == getData().size())
+    return success();
+  return emitError(
+      "dtors, priorities, and data must have the same number of elements");
 }
 
 //===----------------------------------------------------------------------===//

mlir/lib/Target/LLVMIR/ModuleImport.cpp

@@ -1115,6 +1115,7 @@ ModuleImport::convertGlobalCtorsAndDtors(llvm::GlobalVariable *globalVar) {
 
   SmallVector<Attribute> funcs;
   SmallVector<int32_t> priorities;
+  SmallVector<Attribute> dataList;
   for (llvm::Value *operand : initializer->operands()) {
     auto *aggregate = dyn_cast<llvm::ConstantAggregate>(operand);
     if (!aggregate || aggregate->getNumOperands() != 3)
@@ -1126,12 +1127,18 @@ ModuleImport::convertGlobalCtorsAndDtors(llvm::GlobalVariable *globalVar) {
     if (!priority || !func || !data)
       return failure();
 
-    // GlobalCtorsOps and GlobalDtorsOps do not support non-null data fields.
-    if (!data->isNullValue())
+    auto *gv = dyn_cast_or_null<llvm::GlobalValue>(data);
+    Attribute dataAttr;
+    if (gv)
+      dataAttr = FlatSymbolRefAttr::get(context, gv->getName());
+    else if (data->isNullValue())
+      dataAttr = ZeroAttr::get(context);
+    else
       return failure();
 
     funcs.push_back(FlatSymbolRefAttr::get(context, func->getName()));
     priorities.push_back(priority->getValue().getZExtValue());
+    dataList.push_back(dataAttr);
   }
 
   // Insert the global after the last one or at the start of the module.
@@ -1140,12 +1147,12 @@ ModuleImport::convertGlobalCtorsAndDtors(llvm::GlobalVariable *globalVar) {
   if (globalVar->getName() == getGlobalCtorsVarName()) {
     globalInsertionOp = builder.create<LLVM::GlobalCtorsOp>(
         mlirModule.getLoc(), builder.getArrayAttr(funcs),
-        builder.getI32ArrayAttr(priorities));
+        builder.getI32ArrayAttr(priorities), builder.getArrayAttr(dataList));
     return success();
   }
   globalInsertionOp = builder.create<LLVM::GlobalDtorsOp>(
       mlirModule.getLoc(), builder.getArrayAttr(funcs),
-      builder.getI32ArrayAttr(priorities));
+      builder.getI32ArrayAttr(priorities), builder.getArrayAttr(dataList));
   return success();
 }
 

mlir/test/Dialect/LLVMIR/global.mlir

@@ -228,25 +228,25 @@ llvm.func @ctor() {
   llvm.return
 }
 
-// CHECK: llvm.mlir.global_ctors {ctors = [@ctor], priorities = [0 : i32]}
-llvm.mlir.global_ctors { ctors = [@ctor], priorities = [0 : i32]}
+// CHECK: llvm.mlir.global_ctors ctors = [@ctor], priorities = [0 : i32], data = [#llvm.zero]
+llvm.mlir.global_ctors ctors = [@ctor], priorities = [0 : i32], data = [#llvm.zero]
 
 // -----
 
-// CHECK: llvm.mlir.global_ctors {ctors = [], priorities = []}
-llvm.mlir.global_ctors {ctors = [], priorities = []}
+// CHECK: llvm.mlir.global_ctors ctors = [], priorities = [], data = []
+llvm.mlir.global_ctors ctors = [], priorities = [], data = []
 
-// CHECK: llvm.mlir.global_dtors {dtors = [], priorities = []}
-llvm.mlir.global_dtors {dtors = [], priorities = []}
+// CHECK: llvm.mlir.global_dtors dtors = [], priorities = [], data = []
+llvm.mlir.global_dtors dtors = [], priorities = [], data = []
 
 // -----
 
 llvm.func @dtor() {
   llvm.return
 }
 
-// CHECK: llvm.mlir.global_dtors {dtors = [@dtor], priorities = [0 : i32]}
-llvm.mlir.global_dtors { dtors = [@dtor], priorities = [0 : i32]}
+// CHECK: llvm.mlir.global_dtors dtors = [@dtor], priorities = [0 : i32], data = [#llvm.zero]
+llvm.mlir.global_dtors dtors = [@dtor], priorities = [0 : i32], data = [#llvm.zero]
 
 // -----
 

mlir/test/Dialect/LLVMIR/invalid.mlir

@@ -9,29 +9,38 @@ llvm.func @ctor() {
   llvm.return
 }
 
-// expected-error@+1{{mismatch between the number of ctors and the number of priorities}}
-llvm.mlir.global_ctors {ctors = [@ctor], priorities = []}
+// expected-error@+1{{ctors, priorities, and data must have the same number of elements}}
+llvm.mlir.global_ctors ctors = [@ctor], priorities = [], data = [#llvm.zero]
 
 // -----
 
 llvm.func @dtor() {
   llvm.return
 }
 
-// expected-error@+1{{mismatch between the number of dtors and the number of priorities}}
-llvm.mlir.global_dtors {dtors = [@dtor], priorities = [0 : i32, 32767 : i32]}
+// expected-error@+1{{dtors, priorities, and data must have the same number of elements}}
+llvm.mlir.global_dtors dtors = [@dtor], priorities = [0 : i32, 32767 : i32], data = [#llvm.zero]
 
 // -----
 
 // expected-error@+1{{'ctor' does not reference a valid LLVM function}}
-llvm.mlir.global_ctors {ctors = [@ctor], priorities = [0 : i32]}
+llvm.mlir.global_ctors ctors = [@ctor], priorities = [0 : i32], data = [#llvm.zero]
 
 // -----
 
 llvm.func @dtor()
 
 // expected-error@+1{{'dtor' does not have a definition}}
-llvm.mlir.global_dtors {dtors = [@dtor], priorities = [0 : i32]}
+llvm.mlir.global_dtors dtors = [@dtor], priorities = [0 : i32], data = [#llvm.zero]
+
+// -----
+
+llvm.func @dtor() {
+  llvm.return
+}
+
+// expected-error@+1{{data element must be symbol or #llvm.zero}}
+llvm.mlir.global_dtors dtors = [@dtor], priorities = [0 : i32], data = [0 : i32]
 
 ////////////////////////////////////////////////////////////////////////////////
 

mlir/test/Target/LLVMIR/Import/global-variables.ll

@@ -241,8 +241,8 @@
 
 ; // -----
 
-; CHECK: llvm.mlir.global_ctors {ctors = [@foo, @bar], priorities = [0 : i32, 42 : i32]}
-; CHECK: llvm.mlir.global_dtors {dtors = [@foo], priorities = [0 : i32]}
+; CHECK: llvm.mlir.global_ctors ctors = [@foo, @bar], priorities = [0 : i32, 42 : i32], data = [#llvm.zero, #llvm.zero]
+; CHECK: llvm.mlir.global_dtors dtors = [@foo], priorities = [0 : i32], data = [#llvm.zero]
 @llvm.global_ctors = appending global [2 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 0, ptr @foo, ptr null }, { i32, ptr, ptr } { i32 42, ptr @bar, ptr null }]
 @llvm.global_dtors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 0, ptr @foo, ptr null }]
 
@@ -256,14 +256,23 @@ define void @bar() {
 
 ; // -----
 
-; CHECK: llvm.mlir.global_ctors {ctors = [], priorities = []}
+; CHECK: llvm.mlir.global_ctors ctors = [], priorities = [], data = []
 @llvm.global_ctors = appending global [0 x { i32, ptr, ptr }] zeroinitializer
 
-; CHECK: llvm.mlir.global_dtors {dtors = [], priorities = []}
+; CHECK: llvm.mlir.global_dtors dtors = [], priorities = [], data = []
 @llvm.global_dtors = appending global [0 x { i32, ptr, ptr }] zeroinitializer
 
 ; // -----
 
+; llvm.mlir.global_dtors dtors = [@foo], priorities = [0 : i32], data = [@foo]
+@llvm.global_dtors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 0, ptr @foo, ptr @foo }]
+
+define void @foo() {
+  ret void
+}
+
+; // -----
+
 ; Visibility attribute.
 
 ; CHECK: llvm.mlir.global external hidden constant @hidden("string")

mlir/test/Target/LLVMIR/Import/import-failure.ll

@@ -64,17 +64,6 @@ define void @unsupported_argument(i64 %arg1) {
 
 ; // -----
 
-; global_dtors with non-null data fields cannot be represented in MLIR.
-; CHECK:      <unknown>
-; CHECK-SAME: error: unhandled global variable: @llvm.global_dtors
-@llvm.global_dtors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 0, ptr @foo, ptr @foo }]
-
-define void @foo() {
-  ret void
-}
-
-; // -----
-
 ; CHECK:      import-failure.ll
 ; CHECK-SAME: error: unsupported TBAA node format: !{{.*}} = !{!{{.*}}, i64 1, !"omnipotent char"}
 define dso_local void @tbaa(ptr %0) {

mlir/test/Target/LLVMIR/llvmir.mlir

@@ -1851,7 +1851,7 @@ llvm.mlir.global linkonce @take_self_address() : !llvm.struct<(i32, !llvm.ptr)>
 // -----
 
 // CHECK: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 0, ptr @foo, ptr null }]
-llvm.mlir.global_ctors { ctors = [@foo], priorities = [0 : i32]}
+llvm.mlir.global_ctors ctors = [@foo], priorities = [0 : i32], data = [#llvm.zero]
 
 llvm.func @foo() {
   llvm.return
@@ -1860,15 +1860,15 @@ llvm.func @foo() {
 // -----
 
 // CHECK: @llvm.global_ctors = appending global [0 x { i32, ptr, ptr }] zeroinitializer
-llvm.mlir.global_ctors {ctors = [], priorities = []}
+llvm.mlir.global_ctors ctors = [], priorities = [], data = []
 
 // CHECK: @llvm.global_dtors = appending global [0 x { i32, ptr, ptr }] zeroinitializer
-llvm.mlir.global_dtors {dtors = [], priorities = []}
+llvm.mlir.global_dtors dtors = [], priorities = [], data = []
 
 // -----
 
 // CHECK: @llvm.global_dtors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 0, ptr @foo, ptr null }]
-llvm.mlir.global_dtors { dtors = [@foo], priorities = [0 : i32]}
+llvm.mlir.global_dtors dtors = [@foo], priorities = [0 : i32], data = [#llvm.zero]
 
 llvm.func @foo() {
   llvm.return

mlir/test/mlir-runner/global-constructors.mlir

@@ -7,8 +7,8 @@
 module {
   llvm.func @printNewline()
   llvm.func @printI64(i64)
-  llvm.mlir.global_ctors {ctors = [@ctor], priorities = [0 : i32]}
-  llvm.mlir.global_dtors {dtors = [@dtor], priorities = [0 : i32]}
+  llvm.mlir.global_ctors ctors = [@ctor], priorities = [0 : i32], data = [#llvm.zero]
+  llvm.mlir.global_dtors dtors = [@dtor], priorities = [0 : i32], data = [#llvm.zero]
   llvm.func @ctor() {
     %0 = llvm.mlir.constant(1 : i64) : i64
     llvm.call @printI64(%0) : (i64) -> ()