Implemented SPIR-V support for loopcount attributes (#1062)

* Implemented SPIR-V support for loopcount attrs

There are 3 metadata, which map on LoopControlLoopCountINTELMask. Mask
has 3 parameters. Default values are -1.

Signed-off-by: Leonid Pauzin <[email protected]>

Author: Leonid Pauzin

lib/SPIRV/SPIRVInternal.h

@@ -803,7 +803,7 @@ std::vector<Value *> getInt32(Module *M, const std::vector<int> &Value);
 ConstantInt *getSizet(Module *M, uint64_t Value);
 
 /// Get metadata operand as int.
-int getMDOperandAsInt(MDNode *N, unsigned I);
+int64_t getMDOperandAsInt(MDNode *N, unsigned I);
 
 /// Get metadata operand as string.
 std::string getMDOperandAsString(MDNode *N, unsigned I);

lib/SPIRV/SPIRVReader.cpp

@@ -739,6 +739,16 @@ SPIRVToLLVM::getMetadataFromNameAndParameter(std::string Name,
               ConstantInt::get(Type::getInt32Ty(*Context), Parameter))};
 }
 
+inline llvm::MDNode *
+SPIRVToLLVM::getMetadataFromNameAndParameter(std::string Name,
+                                             int64_t Parameter) {
+  std::vector<llvm::Metadata *> Metadata = {
+      MDString::get(*Context, Name),
+      ConstantAsMetadata::get(
+          ConstantInt::get(Type::getInt64Ty(*Context), Parameter))};
+  return llvm::MDNode::get(*Context, Metadata);
+}
+
 template <typename LoopInstType>
 void SPIRVToLLVM::setLLVMLoopMetadata(const LoopInstType *LM,
                                       const Loop *LoopObj) {
@@ -987,6 +997,39 @@ void SPIRVToLLVM::setLLVMLoopMetadata(const LoopInstType *LM,
   }
   if (LC & LoopControlNoFusionINTELMask)
     Metadata.push_back(getMetadataFromName("llvm.loop.fusion.disable"));
+  if (LC & spv::internal::LoopControlLoopCountINTELMask) {
+    // LoopCountINTELMask parameters are int64 and each parameter is stored
+    // as 2 SPIRVWords (int32)
+    assert(NumParam + 6 <= LoopControlParameters.size() &&
+           "Missing loop control parameter!");
+
+    uint64_t LoopCountMin =
+        static_cast<uint64_t>(LoopControlParameters[NumParam++]);
+    LoopCountMin |= static_cast<uint64_t>(LoopControlParameters[NumParam++])
+                    << 32;
+    if (static_cast<int64_t>(LoopCountMin) >= 0) {
+      Metadata.push_back(getMetadataFromNameAndParameter(
+          "llvm.loop.intel.loopcount_min", static_cast<int64_t>(LoopCountMin)));
+    }
+
+    uint64_t LoopCountMax =
+        static_cast<uint64_t>(LoopControlParameters[NumParam++]);
+    LoopCountMax |= static_cast<uint64_t>(LoopControlParameters[NumParam++])
+                    << 32;
+    if (static_cast<int64_t>(LoopCountMax) >= 0) {
+      Metadata.push_back(getMetadataFromNameAndParameter(
+          "llvm.loop.intel.loopcount_max", static_cast<int64_t>(LoopCountMax)));
+    }
+
+    uint64_t LoopCountAvg =
+        static_cast<uint64_t>(LoopControlParameters[NumParam++]);
+    LoopCountAvg |= static_cast<uint64_t>(LoopControlParameters[NumParam++])
+                    << 32;
+    if (static_cast<int64_t>(LoopCountAvg) >= 0) {
+      Metadata.push_back(getMetadataFromNameAndParameter(
+          "llvm.loop.intel.loopcount_avg", static_cast<int64_t>(LoopCountAvg)));
+    }
+  }
   llvm::MDNode *Node = llvm::MDNode::get(*Context, Metadata);
 
   // Set the first operand to refer itself

lib/SPIRV/SPIRVReader.h

@@ -258,6 +258,8 @@ class SPIRVToLLVM {
   inline llvm::Metadata *getMetadataFromName(std::string Name);
   inline std::vector<llvm::Metadata *>
   getMetadataFromNameAndParameter(std::string Name, SPIRVWord Parameter);
+  inline MDNode *getMetadataFromNameAndParameter(std::string Name,
+                                                 int64_t Parameter);
   void insertImageNameAccessQualifier(SPIRV::SPIRVTypeImage *ST,
                                       std::string &Name);
   template <class Source, class Func> bool foreachFuncCtlMask(Source, Func);

lib/SPIRV/SPIRVUtil.cpp

@@ -842,7 +842,7 @@ ConstantInt *getSizet(Module *M, uint64_t Value) {
 // Functions for getting metadata
 //
 ///////////////////////////////////////////////////////////////////////////////
-int getMDOperandAsInt(MDNode *N, unsigned I) {
+int64_t getMDOperandAsInt(MDNode *N, unsigned I) {
   return mdconst::dyn_extract<ConstantInt>(N->getOperand(I))->getZExtValue();
 }
 

lib/SPIRV/SPIRVWriter.cpp

@@ -1113,6 +1113,12 @@ LLVMToSPIRVBase::getLoopControl(const BranchInst *Branch,
 
   size_t LoopControl = spv::LoopControlMaskNone;
   std::vector<std::pair<SPIRVWord, SPIRVWord>> ParametersToSort;
+  // If only a subset of loop count parameters is defined in metadata
+  // then undefined ones should have a default value -1 in SPIR-V.
+  // Preset all loop count parameters with the default value.
+  struct LoopCountInfo {
+    int64_t Min = -1, Max = -1, Avg = -1;
+  } LoopCount;
 
   // Unlike with most of the cases, some loop metadata specifications
   // can occur multiple times - for these, all correspondent tokens
@@ -1212,11 +1218,41 @@ LLVMToSPIRVBase::getLoopControl(const BranchInst *Branch,
           BM->addExtension(ExtensionID::SPV_INTEL_fpga_loop_controls);
           BM->addCapability(CapabilityFPGALoopControlsINTEL);
           LoopControl |= spv::LoopControlNoFusionINTELMask;
+        } else if (S == "llvm.loop.intel.loopcount_min") {
+          BM->addExtension(ExtensionID::SPV_INTEL_fpga_loop_controls);
+          BM->addCapability(CapabilityFPGALoopControlsINTEL);
+          LoopCount.Min = getMDOperandAsInt(Node, 1);
+          LoopControl |= spv::internal::LoopControlLoopCountINTELMask;
+        } else if (S == "llvm.loop.intel.loopcount_max") {
+          BM->addExtension(ExtensionID::SPV_INTEL_fpga_loop_controls);
+          BM->addCapability(CapabilityFPGALoopControlsINTEL);
+          LoopCount.Max = getMDOperandAsInt(Node, 1);
+          LoopControl |= spv::internal::LoopControlLoopCountINTELMask;
+        } else if (S == "llvm.loop.intel.loopcount_avg") {
+          BM->addExtension(ExtensionID::SPV_INTEL_fpga_loop_controls);
+          BM->addCapability(CapabilityFPGALoopControlsINTEL);
+          LoopCount.Avg = getMDOperandAsInt(Node, 1);
+          LoopControl |= spv::internal::LoopControlLoopCountINTELMask;
         }
       }
     }
   }
-
+  if (LoopControl & spv::internal::LoopControlLoopCountINTELMask) {
+    // LoopCountINTELMask have int64 literal parameters and we need to store
+    // int64 into 2 SPIRVWords
+    ParametersToSort.emplace_back(spv::internal::LoopControlLoopCountINTELMask,
+                                  static_cast<SPIRVWord>(LoopCount.Min));
+    ParametersToSort.emplace_back(spv::internal::LoopControlLoopCountINTELMask,
+                                  static_cast<SPIRVWord>(LoopCount.Min >> 32));
+    ParametersToSort.emplace_back(spv::internal::LoopControlLoopCountINTELMask,
+                                  static_cast<SPIRVWord>(LoopCount.Max));
+    ParametersToSort.emplace_back(spv::internal::LoopControlLoopCountINTELMask,
+                                  static_cast<SPIRVWord>(LoopCount.Max >> 32));
+    ParametersToSort.emplace_back(spv::internal::LoopControlLoopCountINTELMask,
+                                  static_cast<SPIRVWord>(LoopCount.Avg));
+    ParametersToSort.emplace_back(spv::internal::LoopControlLoopCountINTELMask,
+                                  static_cast<SPIRVWord>(LoopCount.Avg >> 32));
+  }
   // If any loop control parameters were held back until fully collected,
   // now is the time to move the information to the main parameters collection
   if (!DependencyArrayParameters.empty()) {
@@ -1235,11 +1271,11 @@ LLVMToSPIRVBase::getLoopControl(const BranchInst *Branch,
     LoopControl |= spv::LoopControlDependencyArrayINTELMask;
   }
 
-  std::sort(ParametersToSort.begin(), ParametersToSort.end(),
-            [](const std::pair<SPIRVWord, SPIRVWord> &CompareLeft,
-               const std::pair<SPIRVWord, SPIRVWord> &CompareRight) {
-              return CompareLeft.first < CompareRight.first;
-            });
+  std::stable_sort(ParametersToSort.begin(), ParametersToSort.end(),
+                   [](const std::pair<SPIRVWord, SPIRVWord> &CompareLeft,
+                      const std::pair<SPIRVWord, SPIRVWord> &CompareRight) {
+                     return CompareLeft.first < CompareRight.first;
+                   });
   for (auto Param : ParametersToSort)
     Parameters.push_back(Param.second);
 

lib/SPIRV/libSPIRV/spirv_internal.hpp

@@ -71,6 +71,8 @@ enum InternalMemoryAccessMask {
 
 enum InternalExecutionMode { IExecModeFastCompositeKernelINTEL = 6088 };
 
+enum InternalLoopControlMask { ILoopControlLoopCountINTELMask = 0x1000000 };
+
 constexpr LinkageType LinkageTypeInternal =
     static_cast<LinkageType>(ILTInternal);
 
@@ -121,6 +123,9 @@ constexpr MemoryAccessMask MemoryAccessNoAliasINTELMask =
 constexpr ExecutionMode ExecutionModeFastCompositeKernelINTEL =
     static_cast<ExecutionMode>(IExecModeFastCompositeKernelINTEL);
 
+constexpr LoopControlMask LoopControlLoopCountINTELMask =
+    static_cast<LoopControlMask>(ILoopControlLoopCountINTELMask);
+
 } // namespace internal
 } // namespace spv
 

test/transcoding/SPV_INTEL_fpga_loop_controls/FPGALoopAttr.ll

@@ -168,6 +168,96 @@ for.end36:                                        ; preds = %for.cond29
   ret void
 }
 
+; Function Attrs: convergent norecurse nounwind mustprogress
+define linkonce_odr dso_local spir_func void @_Z18loop_count_controlILi12EEvv() #0 {
+entry:
+  %a = alloca [10 x i32], align 4
+  %a.ascast = addrspacecast [10 x i32]* %a to [10 x i32] addrspace(4)*
+  %i = alloca i32, align 4
+  %i.ascast = addrspacecast i32* %i to i32 addrspace(4)*
+  %cleanup.dest.slot = alloca i32, align 4
+  %i1 = alloca i32, align 4
+  %i1.ascast = addrspacecast i32* %i1 to i32 addrspace(4)*
+  %cleanup.dest.slot5 = alloca i32, align 4
+  %0 = bitcast [10 x i32]* %a to i8*
+  call void @llvm.lifetime.start.p0i8(i64 40, i8* %0)
+  %1 = bitcast i32* %i to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %1)
+  store i32 0, i32 addrspace(4)* %i.ascast, align 4
+  br label %for.cond
+; Per SPIR-V spec extension INTEL/SPV_INTEL_fpga_loop_controls,
+; LoopControlLoopCountINTELMask = 0x1000000 (16777216)
+; CHECK-SPIRV: LoopMerge [[#]] [[#]] 16777216 4294967295 4294967295 4294967295 4294967295 12 0
+; CHECK-SPIRV-NEXT: BranchConditional [[#]] [[#]] [[#]]
+; CHECK-SPIRV-NEGATIVE-NOT: LoopMerge [[#]] [[#]] 16777216
+for.cond:                                         ; preds = %for.inc, %entry
+  %2 = load i32, i32 addrspace(4)* %i.ascast, align 4
+  %cmp = icmp ne i32 %2, 10
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  %3 = bitcast i32* %i to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %3)
+  br label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %4 = load i32, i32 addrspace(4)* %i.ascast, align 4
+  %idxprom = sext i32 %4 to i64
+  %arrayidx = getelementptr inbounds [10 x i32], [10 x i32] addrspace(4)* %a.ascast, i64 0, i64 %idxprom
+  store i32 0, i32 addrspace(4)* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %5 = load i32, i32 addrspace(4)* %i.ascast, align 4
+  %inc = add nsw i32 %5, 1
+  store i32 %inc, i32 addrspace(4)* %i.ascast, align 4
+  br label %for.cond, !llvm.loop !12
+
+for.end:                                          ; preds = %for.cond.cleanup
+  %6 = bitcast i32* %i1 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %6)
+  store i32 0, i32 addrspace(4)* %i1.ascast, align 4
+  br label %for.cond2
+
+; Per SPIR-V spec extension INTEL/SPV_INTEL_fpga_loop_controls,
+; spv::internal::LoopControlLoopCountINTELMask = 0x1000000 (16777216)
+; Parameters 4 0 = 4, 100 1 = 4294967396, 21 0 = 21
+; CHECK-SPIRV: LoopMerge [[#]] [[#]] 16777216 4 0 100 1 21 0
+; CHECK-SPIRV-NEXT: BranchConditional [[#]] [[#]] [[#]]
+; CHECK-SPIRV-NEGATIVE-NOT: LoopMerge [[#]] [[#]] 16777216
+for.cond2:                                        ; preds = %for.inc9, %for.end
+  %7 = load i32, i32 addrspace(4)* %i1.ascast, align 4
+  %cmp3 = icmp ne i32 %7, 10
+  br i1 %cmp3, label %for.body6, label %for.cond.cleanup4
+
+for.cond.cleanup4:                                ; preds = %for.cond2
+  %8 = bitcast i32* %i1 to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %8)
+  br label %for.end11
+
+for.body6:                                        ; preds = %for.cond2
+  %9 = load i32, i32 addrspace(4)* %i1.ascast, align 4
+  %idxprom7 = sext i32 %9 to i64
+  %arrayidx8 = getelementptr inbounds [10 x i32], [10 x i32] addrspace(4)* %a.ascast, i64 0, i64 %idxprom7
+  store i32 0, i32 addrspace(4)* %arrayidx8, align 4
+  br label %for.inc9
+
+for.inc9:                                         ; preds = %for.body6
+  %10 = load i32, i32 addrspace(4)* %i1.ascast, align 4
+  %inc10 = add nsw i32 %10, 1
+  store i32 %inc10, i32 addrspace(4)* %i1.ascast, align 4
+  br label %for.cond2, !llvm.loop !15
+
+for.end11:                                        ; preds = %for.cond.cleanup4
+  %11 = bitcast [10 x i32]* %a to i8*
+  call void @llvm.lifetime.end.p0i8(i64 40, i8* %11) 
+  ret void
+}
+
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture)
+
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture)
+
 attributes #0 = { convergent noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "denorms-are-zero"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "uniform-work-group-size"="true" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0}
@@ -186,12 +276,22 @@ attributes #0 = { convergent noinline nounwind optnone "correctly-rounded-divide
 !9 = distinct !{!9, !10}
 !10 = !{!"llvm.loop.max_concurrency.count", i32 2}
 !11 = distinct !{!11, !8, !10}
+!12 = distinct !{!12, !13, !14}
+!13 = !{!"llvm.loop.mustprogress"}
+!14 = !{!"llvm.loop.intel.loopcount_avg", i64 12}
+!15 = distinct !{!15, !13, !16, !17, !18}
+!16 = !{!"llvm.loop.intel.loopcount_min", i64 4}
+;4294967396 = 2^32 + 100
+!17 = !{!"llvm.loop.intel.loopcount_max", i64 4294967396}
+!18 = !{!"llvm.loop.intel.loopcount_avg", i64 21}
 
 ; CHECK-LLVM: br label %for.cond{{[0-9]*}}, !llvm.loop ![[MD_A:[0-9]+]]
 ; CHECK-LLVM: br label %for.cond{{[0-9]+}}, !llvm.loop ![[MD_B:[0-9]+]]
 ; CHECK-LLVM: br label %for.cond{{[0-9]+}}, !llvm.loop ![[MD_C:[0-9]+]]
 ; CHECK-LLVM: br label %for.cond{{[0-9]+}}, !llvm.loop ![[MD_D:[0-9]+]]
 ; CHECK-LLVM: br label %for.cond{{[0-9]+}}, !llvm.loop ![[MD_E:[0-9]+]]
+; CHECK-LLVM: br label %for.cond{{.*}}, !llvm.loop ![[#MD_F:]]
+; CHECK-LLVM: br label %for.cond{{.*}}, !llvm.loop ![[#MD_G:]]
 
 ; CHECK-LLVM-NEGATIVE: br label %for.cond{{[0-9]*}}, !llvm.loop ![[MD_A:[0-9]+]]
 ; CHECK-LLVM-NEGATIVE: br label %for.cond{{[0-9]+}}, !llvm.loop ![[MD_B:[0-9]+]]
@@ -206,10 +306,19 @@ attributes #0 = { convergent noinline nounwind optnone "correctly-rounded-divide
 ; CHECK-LLVM: ![[MD_D]] = distinct !{![[MD_D]], ![[MD_max_concurrency:[0-9]+]]}
 ; CHECK-LLVM: ![[MD_max_concurrency]] = !{!"llvm.loop.max_concurrency.count", i32 2}
 ; CHECK-LLVM: ![[MD_E]] = distinct !{![[MD_E]], ![[MD_ii:[0-9]+]], ![[MD_max_concurrency:[0-9]+]]}
+; CHECK-LLVM: ![[#MD_F]] = distinct !{![[#MD_F]], ![[#MD_loop_count_avg:]]}
+; CHECK-LLVM: ![[#MD_loop_count_avg]] = !{!"llvm.loop.intel.loopcount_avg", i64 12}
+; CHECK-LLVM: ![[#MD_G]] = distinct !{![[#MD_G]], ![[#MD_loop_count_min:]], ![[#MD_loop_count_max:]], ![[#MD_loop_count_avg_1:]]}
+; CHECK-LLVM: ![[#MD_loop_count_min]] = !{!"llvm.loop.intel.loopcount_min", i64 4}
+; CHECK-LLVM: ![[#MD_loop_count_max]] = !{!"llvm.loop.intel.loopcount_max", i64 4294967396}
+; CHECK-LLVM: ![[#MD_loop_count_avg_1]] = !{!"llvm.loop.intel.loopcount_avg", i64 21}
 
 ; CHECK-LLVM-NEGATIVE: ![[MD_A]] = distinct !{![[MD_A]], ![[MD_ivdep_enable:[0-9]+]]}
 ; CHECK-LLVM-NEGATIVE: ![[MD_ivdep_enable]] = !{!"llvm.loop.ivdep.enable"}
 ; CHECK-LLVM-NEGATIVE: ![[MD_B]] = distinct !{![[MD_B]], ![[MD_ivdep:[0-9]+]]}
 ; CHECK-LLVM-NEGATIVE: ![[MD_ivdep]] = !{!"llvm.loop.ivdep.safelen", i32 2}
 ; CHECK-LLVM-NEGATIVE-NOT: !{{.*}} = !{!"llvm.loop.ii.count"{{.*}}}
 ; CHECK-LLVM-NEGATIVE-NOT: !{{.*}} = !{!"llvm.loop.max_concurrency.count"{{.*}}}
+; CHECK-LLVM-NEGATIVE-NOT: !{{.*}} = !{!"llvm.loop.intel.loopcount_min"{{.*}}}
+; CHECK-LLVM-NEGATIVE-NOT: !{{.*}} = !{!"llvm.loop.intel.loopcount_max"{{.*}}}
+; CHECK-LLVM-NEGATIVE-NOT: !{{.*}} = !{!"llvm.loop.intel.loopcount_avg"{{.*}}}