Merge from 'sycl' to 'sycl-web' (#1)

Author: iclsrc

llvm-spirv/lib/SPIRV/SPIRVInternal.h

@@ -777,6 +777,9 @@ int getMDOperandAsInt(MDNode *N, unsigned I);
 /// Get metadata operand as string.
 std::string getMDOperandAsString(MDNode *N, unsigned I);
 
+/// Get metadata operand as another metadata node
+MDNode *getMDOperandAsMDNode(MDNode *N, unsigned I);
+
 /// Get metadata operand as type.
 Type *getMDOperandAsType(MDNode *N, unsigned I);
 
@@ -935,10 +938,6 @@ template <> inline void SPIRVMap<std::string, Op, SPIRVOpaqueType>::init() {
 // Check if the module contains llvm.loop.* metadata
 bool hasLoopMetadata(const Module *M);
 
-// If the branch instruction has !llvm.loop metadata, go through its operands
-// and find Loop Control mask and possible parameters.
-spv::LoopControlMask getLoopControl(const BranchInst *Branch,
-                                    std::vector<SPIRVWord> &Parameters);
 } // namespace SPIRV
 
 #endif // SPIRV_SPIRVINTERNAL_H

llvm-spirv/lib/SPIRV/SPIRVReader.cpp

@@ -51,10 +51,11 @@
 #include "SPIRVValue.h"
 
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/Analysis/CFG.h"
+#include "llvm/Analysis/LoopInfo.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Dominators.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
@@ -590,18 +591,17 @@ SPIRVToLLVM::getMetadataFromNameAndParameter(std::string Name,
 }
 
 template <typename LoopInstType>
-void SPIRVToLLVM::setLLVMLoopMetadata(const LoopInstType *LM, Instruction *BI) {
+void SPIRVToLLVM::setLLVMLoopMetadata(const LoopInstType *LM,
+                                      const Loop *LoopObj) {
   if (!LM)
     return;
 
-  assert(BI && isa<BranchInst>(BI));
-
   auto Temp = MDNode::getTemporary(*Context, None);
   auto Self = MDNode::get(*Context, Temp.get());
   Self->replaceOperandWith(0, Self);
   SPIRVWord LC = LM->getLoopControl();
   if (LC == LoopControlMaskNone) {
-    BI->setMetadata("llvm.loop", Self);
+    LoopObj->setLoopID(Self);
     return;
   }
 
@@ -685,49 +685,137 @@ void SPIRVToLLVM::setLLVMLoopMetadata(const LoopInstType *LM, Instruction *BI) {
                           LoopControlParameters[++NumParam])));
         break;
       }
-      default:
+      case DependencyArrayINTEL: {
+        // Collect array variable <-> safelen information
+        std::map<Value *, unsigned> ArraySflnMap;
+        unsigned NumOperandPairs = LoopControlParameters[++NumParam];
+        unsigned OperandsEndIndex = NumParam + NumOperandPairs * 2;
+        assert(OperandsEndIndex <= LoopControlParameters.size() &&
+               "Missing loop control parameter!");
+        SPIRVModule *M = LM->getModule();
+        while (NumParam < OperandsEndIndex) {
+          SPIRVId ArraySPIRVId = LoopControlParameters[++NumParam];
+          Value *ArrayVar = ValueMap[M->getValue(ArraySPIRVId)];
+          unsigned Safelen = LoopControlParameters[++NumParam];
+          ArraySflnMap.emplace(ArrayVar, Safelen);
+        }
+
+        // A single run over the loop to retrieve all GetElementPtr instructions
+        // that access relevant array variables
+        std::map<Value *, std::vector<GetElementPtrInst *>> ArrayGEPMap;
+        for (auto &BB : LoopObj->blocks()) {
+          for (Instruction &I : *BB) {
+            auto *GEP = dyn_cast<GetElementPtrInst>(&I);
+            if (!GEP)
+              continue;
+
+            Value *AccessedArray = GEP->getPointerOperand();
+            auto ArraySflnIt = ArraySflnMap.find(AccessedArray);
+            if (ArraySflnIt != ArraySflnMap.end())
+              ArrayGEPMap[AccessedArray].push_back(GEP);
+          }
+        }
+
+        // Create index group metadata nodes specific - one per each array
+        // variables. Mark each GEP accessing a particular array variable
+        // into a corresponding index group
+        std::map<unsigned, std::vector<MDNode *>> SafelenIdxGroupMap;
+        for (auto &ArrayGEPIt : ArrayGEPMap) {
+          // Emit a distinct index group that will be referenced from
+          // llvm.loop.parallel_access_indices metadata
+          auto *CurrentDepthIdxGroup =
+              llvm::MDNode::getDistinct(*Context, None);
+          unsigned Safelen = ArraySflnMap.find(ArrayGEPIt.first)->second;
+          SafelenIdxGroupMap[Safelen].push_back(CurrentDepthIdxGroup);
+
+          for (auto *GEP : ArrayGEPIt.second) {
+            StringRef IdxGroupMDName("llvm.index.group");
+            llvm::MDNode *PreviousIdxGroup = GEP->getMetadata(IdxGroupMDName);
+            if (!PreviousIdxGroup) {
+              GEP->setMetadata(IdxGroupMDName, CurrentDepthIdxGroup);
+              continue;
+            }
+
+            // If we're dealing with an embedded loop, it may be the case
+            // that GEP instructions for some of the arrays were already
+            // marked by the algorithm when it went over the outer level loops.
+            // In order to retain the IVDep information for each "loop
+            // dimension", we will mark such GEP's into a separate joined node
+            // that will refer to the previous levels' index groups AND to the
+            // index group specific to the current loop.
+            std::vector<llvm::Metadata *> CurrentDepthOperands;
+            for (auto &Op : PreviousIdxGroup->operands())
+              CurrentDepthOperands.push_back(Op);
+            if (CurrentDepthOperands.size() == 0)
+              CurrentDepthOperands.push_back(PreviousIdxGroup);
+            CurrentDepthOperands.push_back(CurrentDepthIdxGroup);
+            auto *JointIdxGroup =
+                llvm::MDNode::get(*Context, CurrentDepthOperands);
+            GEP->setMetadata(IdxGroupMDName, JointIdxGroup);
+          }
+        }
+
+        for (auto &SflnIdxGroupIt : SafelenIdxGroupMap) {
+          auto *Name =
+              MDString::get(*Context, "llvm.loop.parallel_access_indices");
+          unsigned SflnValue = SflnIdxGroupIt.first;
+          llvm::Metadata *SafelenMDOp =
+              SflnValue ? ConstantAsMetadata::get(ConstantInt::get(
+                              Type::getInt32Ty(*Context), SflnValue))
+                        : nullptr;
+          std::vector<llvm::Metadata *> Parameters{Name};
+          for (auto *Node : SflnIdxGroupIt.second)
+            Parameters.push_back(Node);
+          if (SafelenMDOp)
+            Parameters.push_back(SafelenMDOp);
+          Metadata.push_back(llvm::MDNode::get(*Context, Parameters));
+        }
         break;
       }
+      default:
+        llvm_unreachable(
+            "Unexpected token in LoopControlExtendedConstrolsMask");
+      }
       ++NumParam;
     }
   }
   llvm::MDNode *Node = llvm::MDNode::get(*Context, Metadata);
 
   // Set the first operand to refer itself
   Node->replaceOperandWith(0, Node);
-  BI->setMetadata("llvm.loop", Node);
+  LoopObj->setLoopID(Node);
 }
 
 void SPIRVToLLVM::transLLVMLoopMetadata(const Function *F) {
   assert(F);
 
-  if (!FuncLoopMetadataMap.empty()) {
-    // In SPIRV loop metadata is linked to a header basic block of a loop
-    // whilst in LLVM IR it is linked to a latch basic block (the one
-    // whose back edge goes to a header basic block) of the loop.
-
-    using Edge = std::pair<const BasicBlock *, const BasicBlock *>;
-    SmallVector<Edge, 32> Edges;
-    FindFunctionBackedges(*F, Edges);
-
-    for (const auto &BkEdge : Edges) {
-      // Check that loop header BB contains loop metadata.
-      const auto LMDItr = FuncLoopMetadataMap.find(BkEdge.second);
-      if (LMDItr == FuncLoopMetadataMap.end())
-        continue;
-
-      auto *BI = const_cast<Instruction *>(BkEdge.first->getTerminator());
-      const auto *LMD = LMDItr->second;
-      if (LMD->getOpCode() == OpLoopMerge) {
-        const auto *LM = static_cast<const SPIRVLoopMerge *>(LMD);
-        setLLVMLoopMetadata<SPIRVLoopMerge>(LM, BI);
-      } else if (LMD->getOpCode() == OpLoopControlINTEL) {
-        const auto *LCI = static_cast<const SPIRVLoopControlINTEL *>(LMD);
-        setLLVMLoopMetadata<SPIRVLoopControlINTEL>(LCI, BI);
-      }
+  if (FuncLoopMetadataMap.empty())
+    return;
 
-      FuncLoopMetadataMap.erase(LMDItr);
+  DominatorTree DomTree(*(const_cast<Function *>(F)));
+  LoopInfo LI(DomTree);
+
+  // In SPIRV loop metadata is linked to a header basic block of a loop
+  // whilst in LLVM IR it is linked to a latch basic block (the one
+  // whose back edge goes to a header basic block) of the loop.
+  // To ensure consistent behaviour, we can rely on the `llvm::Loop`
+  // class to handle the metadata placement
+  for (const auto *LoopObj : LI.getLoopsInPreorder()) {
+    // Check that loop header BB contains loop metadata.
+    const auto LMDItr = FuncLoopMetadataMap.find(LoopObj->getHeader());
+    if (LMDItr == FuncLoopMetadataMap.end())
+      continue;
+
+    const auto *LMD = LMDItr->second;
+    if (LMD->getOpCode() == OpLoopMerge) {
+      const auto *LM = static_cast<const SPIRVLoopMerge *>(LMD);
+      setLLVMLoopMetadata<SPIRVLoopMerge>(LM, LoopObj);
+    } else if (LMD->getOpCode() == OpLoopControlINTEL) {
+      const auto *LCI = static_cast<const SPIRVLoopControlINTEL *>(LMD);
+      setLLVMLoopMetadata<SPIRVLoopControlINTEL>(LCI, LoopObj);
     }
+
+    FuncLoopMetadataMap.erase(LMDItr);
   }
 }
 

llvm-spirv/lib/SPIRV/SPIRVReader.h

@@ -50,10 +50,10 @@
 namespace llvm {
 class Module;
 class Type;
-class Value;
 class Instruction;
 class CallInst;
 class BasicBlock;
+class Loop;
 class Function;
 class GlobalVariable;
 class LLVMContext;
@@ -255,7 +255,7 @@ class SPIRVToLLVM {
   Value *oclTransConstantPipeStorage(SPIRV::SPIRVConstantPipeStorage *BCPS);
   void setName(llvm::Value *V, SPIRVValue *BV);
   template <typename LoopInstType>
-  void setLLVMLoopMetadata(const LoopInstType *LM, Instruction *BI);
+  void setLLVMLoopMetadata(const LoopInstType *LM, const Loop *LoopObj);
   void transLLVMLoopMetadata(const Function *F);
   inline llvm::Metadata *getMetadataFromName(std::string Name);
   inline std::vector<llvm::Metadata *>

llvm-spirv/lib/SPIRV/SPIRVUtil.cpp

@@ -851,20 +851,23 @@ int getMDOperandAsInt(MDNode *N, unsigned I) {
   return mdconst::dyn_extract<ConstantInt>(N->getOperand(I))->getZExtValue();
 }
 
-std::string getMDOperandAsString(MDNode *N, unsigned I) {
+// Additional helper function to be reused by getMDOperandAs* helpers
+Metadata *getMDOperandOrNull(MDNode *N, unsigned I) {
   if (!N)
-    return "";
-
-  Metadata *Op = N->getOperand(I);
-  if (!Op)
-    return "";
+    return nullptr;
+  return N->getOperand(I);
+}
 
-  if (MDString *Str = dyn_cast<MDString>(Op)) {
+std::string getMDOperandAsString(MDNode *N, unsigned I) {
+  if (auto *Str = dyn_cast_or_null<MDString>(getMDOperandOrNull(N, I)))
     return Str->getString().str();
-  }
   return "";
 }
 
+MDNode *getMDOperandAsMDNode(MDNode *N, unsigned I) {
+  return dyn_cast_or_null<MDNode>(getMDOperandOrNull(N, I));
+}
+
 Type *getMDOperandAsType(MDNode *N, unsigned I) {
   return cast<ValueAsMetadata>(N->getOperand(I))->getType();
 }
@@ -1511,57 +1514,4 @@ bool hasLoopMetadata(const Module *M) {
   return false;
 }
 
-spv::LoopControlMask getLoopControl(const BranchInst *Branch,
-                                    std::vector<SPIRVWord> &Parameters) {
-  if (!Branch)
-    return spv::LoopControlMaskNone;
-  MDNode *LoopMD = Branch->getMetadata("llvm.loop");
-  if (!LoopMD)
-    return spv::LoopControlMaskNone;
-
-  size_t LoopControl = spv::LoopControlMaskNone;
-  for (const MDOperand &MDOp : LoopMD->operands()) {
-    if (MDNode *Node = dyn_cast<MDNode>(MDOp)) {
-      std::string S = getMDOperandAsString(Node, 0);
-      // Set the loop control bits. Parameters are set in the order described
-      // in 3.23 SPIR-V Spec. rev. 1.4:
-      // Bits that are set can indicate whether an additional operand follows,
-      // as described by the table. If there are multiple following operands
-      // indicated, they are ordered: Those indicated by smaller-numbered bits
-      // appear first.
-      if (S == "llvm.loop.unroll.disable")
-        LoopControl |= spv::LoopControlDontUnrollMask;
-      else if (S == "llvm.loop.unroll.full" || S == "llvm.loop.unroll.enable")
-        LoopControl |= spv::LoopControlUnrollMask;
-      if (S == "llvm.loop.ivdep.enable")
-        LoopControl |= spv::LoopControlDependencyInfiniteMask;
-      if (S == "llvm.loop.ivdep.safelen") {
-        size_t I = getMDOperandAsInt(Node, 1);
-        Parameters.push_back(I);
-        LoopControl |= spv::LoopControlDependencyLengthMask;
-      }
-      // PartialCount must not be used with the DontUnroll bit
-      if (S == "llvm.loop.unroll.count" &&
-          !(LoopControl & LoopControlDontUnrollMask)) {
-        size_t I = getMDOperandAsInt(Node, 1);
-        Parameters.push_back(I);
-        LoopControl |= spv::LoopControlPartialCountMask;
-      }
-      if (S == "llvm.loop.ii.count") {
-        Parameters.push_back(InitiationIntervalINTEL);
-        size_t I = getMDOperandAsInt(Node, 1);
-        Parameters.push_back(I);
-        LoopControl |= spv::LoopControlExtendedControlsMask;
-      }
-      if (S == "llvm.loop.max_concurrency.count") {
-        Parameters.push_back(MaxConcurrencyINTEL);
-        size_t I = getMDOperandAsInt(Node, 1);
-        Parameters.push_back(I);
-        LoopControl |= spv::LoopControlExtendedControlsMask;
-      }
-    }
-  }
-  return static_cast<spv::LoopControlMask>(LoopControl);
-}
-
 } // namespace SPIRV