Sync with Khronos SPIR-V Translator 16a9caf09d

Signed-off-by: Dmitry Sidorov <[email protected]>

Author: MrSidims

llvm-spirv/lib/SPIRV/OCLUtil.cpp

@@ -114,6 +114,12 @@ size_t getAtomicBuiltinNumMemoryOrderArgs(StringRef Name) {
   return 1;
 }
 
+size_t getSPIRVAtomicBuiltinNumMemoryOrderArgs(Op OC) {
+  if (OC == OpAtomicCompareExchange || OC == OpAtomicCompareExchangeWeak)
+    return 2;
+  return 1;
+}
+
 bool isComputeAtomicOCLBuiltin(StringRef DemangledName) {
   if (!DemangledName.startswith(kOCLBuiltinName::AtomicPrefix) &&
       !DemangledName.startswith(kOCLBuiltinName::AtomPrefix))
@@ -556,7 +562,6 @@ class OCLBuiltinFuncMangleInfo : public SPIRV::BuiltinFuncMangleInfo {
       addVoidPtrArg(1);
       addUnsignedArg(2);
       addUnsignedArg(3);
-
     } else if (UnmangledName == "read_pipe_4" ||
                UnmangledName == "write_pipe_4") {
       // with 4 arguments (plus two i32 literals):

llvm-spirv/lib/SPIRV/OCLUtil.h

@@ -66,6 +66,18 @@ enum OCLMemFenceKind {
   OCLMF_Image = 4,
 };
 
+// This enum declares extra constants for OpenCL mem_fence flag. It includes
+// combinations of local/global/image flags.
+enum OCLMemFenceExtendedKind {
+  OCLMFEx_Local = OCLMF_Local,
+  OCLMFEx_Global = OCLMF_Global,
+  OCLMFEx_Local_Global = OCLMF_Global | OCLMF_Local,
+  OCLMFEx_Image = OCLMF_Image,
+  OCLMFEx_Image_Local = OCLMF_Image | OCLMF_Local,
+  OCLMFEx_Image_Global = OCLMF_Image | OCLMF_Global,
+  OCLMFEx_Image_Local_Global = OCLMF_Image | OCLMF_Global | OCLMF_Local,
+};
+
 enum OCLScopeKind {
   OCLMS_work_item,
   OCLMS_work_group,
@@ -95,6 +107,9 @@ enum OCLMemOrderKind {
 
 typedef SPIRVMap<OCLMemFenceKind, MemorySemanticsMask> OCLMemFenceMap;
 
+typedef SPIRVMap<OCLMemFenceExtendedKind, MemorySemanticsMask>
+    OCLMemFenceExtendedMap;
+
 typedef SPIRVMap<OCLMemOrderKind, unsigned, MemorySemanticsMask> OCLMemOrderMap;
 
 typedef SPIRVMap<OCLScopeKind, Scope> OCLMemScopeMap;
@@ -107,6 +122,9 @@ typedef SPIRVMap<std::string, SPIRVFPRoundingModeKind>
 
 typedef SPIRVMap<std::string, Op, SPIRVInstruction> OCLSPIRVBuiltinMap;
 
+class OCL12Builtin;
+typedef SPIRVMap<std::string, Op, OCL12Builtin> OCL12SPIRVBuiltinMap;
+
 typedef SPIRVMap<std::string, SPIRVBuiltinVariableKind>
     SPIRSPIRVBuiltinVariableMap;
 
@@ -305,6 +323,9 @@ BarrierLiterals getBarrierLiterals(CallInst *CI);
 /// Get number of memory order arguments for atomic builtin function.
 size_t getAtomicBuiltinNumMemoryOrderArgs(StringRef Name);
 
+/// Get number of memory order arguments for spirv atomic builtin function.
+size_t getSPIRVAtomicBuiltinNumMemoryOrderArgs(Op OC);
+
 /// Return true for OpenCL builtins which do compute operations
 /// (like add, sub, min, max, inc, dec, ...) atomically
 bool isComputeAtomicOCLBuiltin(StringRef DemangledName);
@@ -422,6 +443,22 @@ template <> inline void SPIRVMap<OCLMemFenceKind, MemorySemanticsMask>::init() {
   add(OCLMF_Image, MemorySemanticsImageMemoryMask);
 }
 
+template <>
+inline void SPIRVMap<OCLMemFenceExtendedKind, MemorySemanticsMask>::init() {
+  add(OCLMFEx_Local, MemorySemanticsWorkgroupMemoryMask);
+  add(OCLMFEx_Global, MemorySemanticsCrossWorkgroupMemoryMask);
+  add(OCLMFEx_Local_Global, MemorySemanticsWorkgroupMemoryMask |
+                                MemorySemanticsCrossWorkgroupMemoryMask);
+  add(OCLMFEx_Image, MemorySemanticsImageMemoryMask);
+  add(OCLMFEx_Image_Local,
+      MemorySemanticsWorkgroupMemoryMask | MemorySemanticsImageMemoryMask);
+  add(OCLMFEx_Image_Global,
+      MemorySemanticsCrossWorkgroupMemoryMask | MemorySemanticsImageMemoryMask);
+  add(OCLMFEx_Image_Local_Global, MemorySemanticsWorkgroupMemoryMask |
+                                      MemorySemanticsCrossWorkgroupMemoryMask |
+                                      MemorySemanticsImageMemoryMask);
+}
+
 template <>
 inline void SPIRVMap<OCLMemOrderKind, unsigned, MemorySemanticsMask>::init() {
   add(OCLMO_relaxed, MemorySemanticsMaskNone);
@@ -622,6 +659,7 @@ template <> inline void SPIRVMap<std::string, Op, SPIRVInstruction>::init() {
   _SPIRV_OP(signbit, SignBitSet)
   _SPIRV_OP(any, Any)
   _SPIRV_OP(all, All)
+  _SPIRV_OP(popcount, BitCount)
   _SPIRV_OP(get_fence, GenericPtrMemSemantics)
   // CL 2.0 kernel enqueue builtins
   _SPIRV_OP(enqueue_marker, EnqueueMarker)
@@ -696,6 +734,22 @@ template <> inline void SPIRVMap<std::string, Op, SPIRVInstruction>::init() {
 #undef _SPIRV_OP
 }
 
+template <> inline void SPIRVMap<std::string, Op, OCL12Builtin>::init() {
+#define _SPIRV_OP(x, y) add("atomic_" #x, Op##y);
+  _SPIRV_OP(add, AtomicIAdd)
+  _SPIRV_OP(sub, AtomicISub)
+  _SPIRV_OP(xchg, AtomicExchange)
+  _SPIRV_OP(cmpxchg, AtomicCompareExchange)
+  _SPIRV_OP(inc, AtomicIIncrement)
+  _SPIRV_OP(dec, AtomicIDecrement)
+  _SPIRV_OP(min, AtomicSMin)
+  _SPIRV_OP(max, AtomicSMax)
+  _SPIRV_OP(and, AtomicAnd)
+  _SPIRV_OP(or, AtomicOr)
+  _SPIRV_OP(xor, AtomicXor)
+#undef _SPIRV_OP
+}
+
 // SPV_INTEL_device_side_avc_motion_estimation extension builtins
 class SPIRVSubgroupsAVCIntelInst;
 template <>

llvm-spirv/lib/SPIRV/SPIRVInternal.h

@@ -337,6 +337,7 @@ const static char TranslateOCLMemOrder[] = "__translate_ocl_memory_order";
 const static char TranslateOCLMemScope[] = "__translate_ocl_memory_scope";
 const static char TranslateSPIRVMemOrder[] = "__translate_spirv_memory_order";
 const static char TranslateSPIRVMemScope[] = "__translate_spirv_memory_scope";
+const static char TranslateSPIRVMemFence[] = "__translate_spirv_memory_fence";
 } // namespace kSPIRVName
 
 namespace kSPIRVPostfix {

llvm-spirv/lib/SPIRV/SPIRVReader.cpp

@@ -576,7 +576,8 @@ SPIRVToLLVM::getMetadataFromNameAndParameter(std::string Name,
               ConstantInt::get(Type::getInt32Ty(*Context), Parameter))};
 }
 
-void SPIRVToLLVM::setLLVMLoopMetadata(SPIRVLoopMerge *LM, BranchInst *BI) {
+template <typename LoopInstType>
+void SPIRVToLLVM::setLLVMLoopMetadata(LoopInstType *LM, BranchInst *BI) {
   if (!LM)
     return;
   auto Temp = MDNode::getTemporary(*Context, None);
@@ -1358,7 +1359,10 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F,
     auto Prev = BR->getPrevious();
     if (Prev && Prev->getOpCode() == OpLoopMerge) {
       auto LM = static_cast<SPIRVLoopMerge *>(Prev);
-      setLLVMLoopMetadata(LM, BI);
+      setLLVMLoopMetadata<SPIRVLoopMerge>(LM, BI);
+    } else if (Prev && Prev->getOpCode() == OpLoopControlINTEL) {
+      auto LCI = static_cast<SPIRVLoopControlINTEL *>(Prev);
+      setLLVMLoopMetadata<SPIRVLoopControlINTEL>(LCI, BI);
     }
     return mapValue(BV, BI);
   }
@@ -1372,7 +1376,10 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F,
     auto Prev = BR->getPrevious();
     if (Prev && Prev->getOpCode() == OpLoopMerge) {
       auto LM = static_cast<SPIRVLoopMerge *>(Prev);
-      setLLVMLoopMetadata(LM, BC);
+      setLLVMLoopMetadata<SPIRVLoopMerge>(LM, BC);
+    } else if (Prev && Prev->getOpCode() == OpLoopControlINTEL) {
+      auto LCI = static_cast<SPIRVLoopControlINTEL *>(Prev);
+      setLLVMLoopMetadata<SPIRVLoopControlINTEL>(LCI, BC);
     }
     return mapValue(BV, BC);
   }
@@ -1498,9 +1505,9 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F,
 
   case OpVmeImageINTEL:
   case OpLine:
-  case OpSelectionMerge: // OpenCL Compiler does not use this instruction
-  case OpLoopMerge: // Should be translated at OpBranch or OpBranchConditional
-                    // cases
+  case OpSelectionMerge:   // OpenCL Compiler does not use this instruction
+  case OpLoopMerge:        // Should be translated at OpBranch or
+  case OpLoopControlINTEL: // OpBranchConditional cases
     return nullptr;
 
   case OpSwitch: {
@@ -1659,6 +1666,16 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F,
                                           BV->getName(), BB));
   }
 
+  case OpBitReverse: {
+    auto *BR = static_cast<SPIRVUnary *>(BV);
+    IntegerType *Int32Ty = IntegerType::get(*Context, 32);
+    Function *intr =
+        Intrinsic::getDeclaration(M, llvm::Intrinsic::bitreverse, Int32Ty);
+    auto *Call = CallInst::Create(intr, transValue(BR->getOperand(0), F, BB),
+                                  BR->getName(), BB);
+    return mapValue(BV, Call);
+  }
+
   case OpFunctionCall: {
     SPIRVFunctionCall *BC = static_cast<SPIRVFunctionCall *>(BV);
     auto Call = CallInst::Create(transFunction(BC->getFunction()),
@@ -2493,21 +2510,21 @@ void SPIRVToLLVM::transIntelFPGADecorations(SPIRVValue *BV, Value *V) {
             Builder.CreateCall(AnnotationFn, Args);
           }
         }
-      } else {
-        SmallString<256> AnnotStr;
-        generateIntelFPGAAnnotation(BV, AnnotStr);
-        if (!AnnotStr.empty()) {
-          auto *GS = Builder.CreateGlobalStringPtr(AnnotStr);
-
-          auto AnnotationFn =
-              llvm::Intrinsic::getDeclaration(M, Intrinsic::var_annotation);
-
-          llvm::Value *Args[] = {
-              Builder.CreateBitCast(V, Int8PtrTyPrivate, V->getName()),
-              Builder.CreateBitCast(GS, Int8PtrTyPrivate), UndefInt8Ptr,
-              UndefInt32};
-          Builder.CreateCall(AnnotationFn, Args);
-        }
+      }
+
+      SmallString<256> AnnotStr;
+      generateIntelFPGAAnnotation(BV, AnnotStr);
+      if (!AnnotStr.empty()) {
+        auto *GS = Builder.CreateGlobalStringPtr(AnnotStr);
+
+        auto AnnotationFn =
+            llvm::Intrinsic::getDeclaration(M, Intrinsic::var_annotation);
+
+        llvm::Value *Args[] = {
+            Builder.CreateBitCast(V, Int8PtrTyPrivate, V->getName()),
+            Builder.CreateBitCast(GS, Int8PtrTyPrivate), UndefInt8Ptr,
+            UndefInt32};
+        Builder.CreateCall(AnnotationFn, Args);
       }
     }
   }

llvm-spirv/lib/SPIRV/SPIRVReader.h

@@ -245,7 +245,8 @@ class SPIRVToLLVM {
                                  BasicBlock *BB);
   Value *oclTransConstantPipeStorage(SPIRV::SPIRVConstantPipeStorage *BCPS);
   void setName(llvm::Value *V, SPIRVValue *BV);
-  void setLLVMLoopMetadata(SPIRVLoopMerge *LM, BranchInst *BI);
+  template <typename LoopInstType>
+  void setLLVMLoopMetadata(LoopInstType *LM, BranchInst *BI);
   inline llvm::Metadata *getMetadataFromName(std::string Name);
   inline std::vector<llvm::Metadata *>
   getMetadataFromNameAndParameter(std::string Name, SPIRVWord Parameter);

llvm-spirv/lib/SPIRV/SPIRVToOCL.cpp

@@ -362,119 +362,6 @@ std::string SPIRVToOCL::getGroupBuiltinPrefix(CallInst *CI) {
   return Prefix;
 }
 
-Instruction *SPIRVToOCL::visitCallSPIRVAtomicBuiltin(CallInst *CI, Op OC) {
-  assert(CI->getCalledFunction() && "Unexpected indirect call");
-  AttributeList Attrs = CI->getCalledFunction()->getAttributes();
-  Instruction *PInsertBefore = CI;
-
-  return mutateCallInstOCL(
-      M, CI,
-      [=](CallInst *, std::vector<Value *> &Args, Type *&RetTy) {
-        auto Ptr = findFirstPtr(Args);
-        auto Name = OCLSPIRVBuiltinMap::rmap(OC);
-        auto NumOrder = getAtomicBuiltinNumMemoryOrderArgs(Name);
-        auto ScopeIdx = Ptr + 1;
-        auto OrderIdx = Ptr + 2;
-        if (OC == OpAtomicIIncrement || OC == OpAtomicIDecrement) {
-          // Since OpenCL 1.2 atomic_inc and atomic_dec builtins don't have,
-          // memory scope and memory order syntax, and OpenCL 2.0 doesn't have
-          // such builtins, therefore we translate these instructions to
-          // atomic_fetch_add_explicit and atomic_fetch_sub_explicit OpenCL 2.0
-          // builtins with "operand" argument = 1.
-          Name = OCLSPIRVBuiltinMap::rmap(
-              OC == OpAtomicIIncrement ? OpAtomicIAdd : OpAtomicISub);
-          Type *ValueTy =
-              cast<PointerType>(Args[Ptr]->getType())->getElementType();
-          assert(ValueTy->isIntegerTy());
-          Args.push_back(llvm::ConstantInt::get(ValueTy, 1));
-        }
-        if (auto *ScopeInt = dyn_cast_or_null<ConstantInt>(Args[ScopeIdx])) {
-          Args[ScopeIdx] = mapUInt(M, ScopeInt, [](unsigned I) {
-            return rmap<OCLScopeKind>(static_cast<Scope>(I));
-          });
-        } else {
-          CallInst *TransCall = dyn_cast<CallInst>(Args[ScopeIdx]);
-          Function *F = TransCall ? TransCall->getCalledFunction() : nullptr;
-          if (F && F->getName().equals(kSPIRVName::TranslateOCLMemScope)) {
-            // In case the SPIR-V module was created from an OpenCL program by
-            // *this* SPIR-V generator, we know that the value passed to
-            // __translate_ocl_memory_scope is what we should pass to the OpenCL
-            // builtin now.
-            Args[ScopeIdx] = TransCall->getArgOperand(0);
-          } else {
-            Args[ScopeIdx] = getOrCreateSwitchFunc(
-                kSPIRVName::TranslateSPIRVMemScope, Args[ScopeIdx],
-                OCLMemScopeMap::getRMap(), true /*IsReverse*/, None, CI, M);
-          }
-        }
-        for (size_t I = 0; I < NumOrder; ++I) {
-          if (auto OrderInt =
-                  dyn_cast_or_null<ConstantInt>(Args[OrderIdx + I])) {
-            Args[OrderIdx + I] = mapUInt(M, OrderInt, [](unsigned Ord) {
-              return mapSPIRVMemOrderToOCL(Ord);
-            });
-          } else {
-            CallInst *TransCall = dyn_cast<CallInst>(Args[OrderIdx + I]);
-            Function *F = TransCall ? TransCall->getCalledFunction() : nullptr;
-            if (F && F->getName().equals(kSPIRVName::TranslateOCLMemOrder)) {
-              // In case the SPIR-V module was created from an OpenCL program by
-              // *this* SPIR-V generator, we know that the value passed to
-              // __translate_ocl_memory_order is what we should pass to the
-              // OpenCL builtin now.
-              Args[OrderIdx + I] = TransCall->getArgOperand(0);
-            } else {
-              Args[OrderIdx + I] = getOrCreateSwitchFunc(
-                  kSPIRVName::TranslateSPIRVMemOrder, Args[OrderIdx + I],
-                  OCLMemOrderMap::getRMap(), true /*IsReverse*/, None, CI, M);
-            }
-          }
-        }
-        std::swap(Args[ScopeIdx], Args.back());
-        if (OC == OpAtomicCompareExchange ||
-            OC == OpAtomicCompareExchangeWeak) {
-          // OpAtomicCompareExchange[Weak] semantics is different from
-          // atomic_compare_exchange_[strong|weak] semantics as well as
-          // arguments order.
-          // OCL built-ins returns boolean value and stores a new/original
-          // value by pointer passed as 2nd argument (aka expected) while SPIR-V
-          // instructions returns this new/original value as a resulting value.
-          AllocaInst *PExpected =
-              new AllocaInst(CI->getType(), 0, "expected",
-                             &(*PInsertBefore->getParent()
-                                    ->getParent()
-                                    ->getEntryBlock()
-                                    .getFirstInsertionPt()));
-          PExpected->setAlignment(CI->getType()->getScalarSizeInBits() / 8);
-          new StoreInst(Args[1], PExpected, PInsertBefore);
-          unsigned AddrSpc = SPIRAS_Generic;
-          Type *PtrTyAS =
-              PExpected->getType()->getElementType()->getPointerTo(AddrSpc);
-          Args[1] = CastInst::CreatePointerBitCastOrAddrSpaceCast(
-              PExpected, PtrTyAS, PExpected->getName() + ".as", PInsertBefore);
-          std::swap(Args[3], Args[4]);
-          std::swap(Args[2], Args[3]);
-          RetTy = Type::getInt1Ty(*Ctx);
-        }
-        return Name;
-      },
-      [=](CallInst *CI) -> Instruction * {
-        if (OC == OpAtomicCompareExchange ||
-            OC == OpAtomicCompareExchangeWeak) {
-          // OCL built-ins atomic_compare_exchange_[strong|weak] return boolean
-          // value. So, to obtain the same value as SPIR-V instruction is
-          // returning it has to be loaded from the memory where 'expected'
-          // value is stored. This memory must contain the needed value after a
-          // call to OCL built-in is completed.
-          LoadInst *POriginal =
-              new LoadInst(CI->getArgOperand(1), "original", PInsertBefore);
-          return POriginal;
-        }
-        // For other built-ins the return values match.
-        return CI;
-      },
-      &Attrs);
-}
-
 } // namespace SPIRV
 
 ModulePass *llvm::createSPIRVToOCL(Module &M) {

llvm-spirv/lib/SPIRV/SPIRVToOCL.h

@@ -101,10 +101,19 @@ class SPIRVToOCL : public ModulePass, public InstVisitor<SPIRVToOCL> {
   /// workgroup/subgroup scope enums.
   std::string getGroupBuiltinPrefix(CallInst *CI);
 
+  /// Transform __spirv_OpAtomicCompareExchange and
+  /// __spirv_OpAtomicCompareExchangeWeak
+  virtual Instruction *visitCallSPIRVAtomicCmpExchg(CallInst *CI, Op OC) = 0;
+
+  /// Transform __spirv_OpAtomicIIncrement/OpAtomicIDecrement to:
+  /// - OCL2.0: atomic_fetch_add_explicit/atomic_fetch_sub_explicit
+  /// - OCL1.2: atomic_inc/atomic_dec
+  virtual Instruction *visitCallSPIRVAtomicIncDec(CallInst *CI, Op OC) = 0;
+
   /// Transform __spirv_Atomic* to atomic_*.
   ///   __spirv_Atomic*(atomic_op, scope, sema, ops, ...) =>
   ///      atomic_*(atomic_op, ops, ..., order(sema), map(scope))
-  virtual Instruction *visitCallSPIRVAtomicBuiltin(CallInst *CI, Op OC);
+  virtual Instruction *visitCallSPIRVAtomicBuiltin(CallInst *CI, Op OC) = 0;
 
   /// Transform __spirv_MemoryBarrier to:
   /// - OCL2.0: atomic_work_item_fence.__spirv_MemoryBarrier(scope, sema) =>
@@ -117,6 +126,13 @@ class SPIRVToOCL : public ModulePass, public InstVisitor<SPIRVToOCL> {
   /// - OCL1.2: barrier
   virtual void visitCallSPIRVControlBarrier(CallInst *CI) = 0;
 
+  /// Conduct generic mutations for all atomic builtins
+  virtual CallInst *mutateCommonAtomicArguments(CallInst *CI, Op OC) = 0;
+
+  /// Transform __spirv_Opcode to ocl-version specific builtin name
+  /// using separate maps for OpenCL 1.2 and OpenCL 2.0
+  virtual Instruction *mutateAtomicName(CallInst *CI, Op OC) = 0;
+
   static char ID;
 
 protected: