Re-implement Intel cache controls decoration (#2587)

The patch adds CacheControls(Load/Store)INTEL decorations representation
as metadata placed on memory accessing instruction with the following form:
!spirv.DecorationCacheControlINTEL !X
!X = !{i32 %decoration_kind%, i32 %level%, i32 %control%,
       i32 %operand of the instruction to decorate%}

Also patch creates a dummy GEP accessing pointer operand of the
instruction to perform SSA copy of the pointer and attaches !spirv.Decorations
metadata to the GEP.

Few notes about this implementation and other options. It is sub-optimal
as it requires iteration over all instructions in the module.
Alternatives are:
1. In SPIRVWriter during transDecorations rewrite already translated
   instruction (lets say load) with the new one, but with GEP operand.
   But while the translator provides API to erase instructions and
   rewriting SPIRVBasicBlocks, yet unfortunately it's not really usable
   outside of of SPIRVModule context;
2. In SPIRVWriter during transValueWithoutDecoration add special
   handling of the instructions with spirv.DecorationCacheControlINTEL
   metadata. And it would work with one exception - it also would
   require to unprivate some methods of SPIRVScavenger to create GEP
   with the correct type, which I'm not sure if it's a good idea.
   Note, in this implementation in the worst case scenario
   SPIRVScavenger would create a bitcast later during translation.

Signed-off-by: Sidorov, Dmitry <[email protected]>

Author: MrSidims

lib/SPIRV/SPIRVInternal.h

@@ -271,6 +271,7 @@ typedef SPIRVMap<SPIRVExtInstSetKind, std::string, SPIRVExtSetShortName>
 
 #define SPIRV_MD_PARAMETER_DECORATIONS "spirv.ParameterDecorations"
 #define SPIRV_MD_DECORATIONS "spirv.Decorations"
+#define SPIRV_MD_INTEL_CACHE_DECORATIONS "spirv.DecorationCacheControlINTEL"
 
 #define OCL_TYPE_NAME_SAMPLER_T "sampler_t"
 #define SPIR_TYPE_NAME_EVENT_T "opencl.event_t"

lib/SPIRV/SPIRVRegularizeLLVM.cpp

@@ -495,6 +495,65 @@ void regularizeWithOverflowInstrinsics(StringRef MangledName, CallInst *Call,
   }
   ToErase.push_back(Call);
 }
+
+// CacheControls(Load/Store)INTEL decorations can be represented as metadata
+// placed on memory accessing instruction with the following form:
+// !spirv.DecorationCacheControlINTEL !X
+// !X = !{i32 %decoration_kind%, i32 %level%, i32 %control%,
+//        i32 %operand of the instruction to decorate%}
+// This function creates a dummy GEP accessing pointer operand of the
+// instruction and creates !spirv.Decorations metadata attached to it.
+void prepareCacheControlsTranslation(Metadata *MD, Instruction *Inst) {
+  if (!Inst->mayReadOrWriteMemory())
+    return;
+  auto *ArgDecoMD = dyn_cast<MDNode>(MD);
+  assert(ArgDecoMD && "Decoration list must be a metadata node");
+  for (unsigned I = 0, E = ArgDecoMD->getNumOperands(); I != E; ++I) {
+    auto *DecoMD = dyn_cast<MDNode>(ArgDecoMD->getOperand(I));
+    if (!DecoMD) {
+      assert(!"Decoration does not name metadata");
+      return;
+    }
+
+    constexpr size_t CacheControlsNumOps = 4;
+    if (DecoMD->getNumOperands() != CacheControlsNumOps) {
+      assert(!"Cache controls metadata on instruction must have 4 operands");
+      return;
+    }
+
+    auto *const KindMD = cast<ConstantAsMetadata>(DecoMD->getOperand(0));
+    auto *const LevelMD = cast<ConstantAsMetadata>(DecoMD->getOperand(1));
+    auto *const ControlMD = cast<ConstantAsMetadata>(DecoMD->getOperand(2));
+
+    const size_t TargetArgNo =
+        mdconst::dyn_extract<ConstantInt>(DecoMD->getOperand(3))
+            ->getZExtValue();
+    Value *PtrInstOp = Inst->getOperand(TargetArgNo);
+    if (!PtrInstOp->getType()->isPointerTy()) {
+      assert(!"Cache controls must decorate a pointer");
+      return;
+    }
+
+    // Create dummy GEP for SSA copy of the pointer operand. Lets do our best
+    // to guess pointee type here, but if we won't - just pointer is also fine,
+    // if necessary TypeScavenger will adjust types and create bitcasts.
+    IRBuilder Builder(Inst);
+    Type *GEPTy = PtrInstOp->getType();
+    if (auto *LI = dyn_cast<LoadInst>(Inst))
+      GEPTy = LI->getType();
+    else if (auto *SI = dyn_cast<StoreInst>(Inst))
+      GEPTy = SI->getValueOperand()->getType();
+    auto *GEP =
+        cast<Instruction>(Builder.CreateConstGEP1_32(GEPTy, PtrInstOp, 0));
+    Inst->setOperand(TargetArgNo, GEP);
+
+    SmallVector<Metadata *, 4> MDs;
+    std::vector<Metadata *> OPs = {KindMD, LevelMD, ControlMD};
+    MDs.push_back(MDNode::get(Inst->getContext(), OPs));
+    MDNode *MDList = MDNode::get(Inst->getContext(), MDs);
+    GEP->setMetadata(SPIRV_MD_DECORATIONS, MDList);
+  }
+}
 } // namespace
 
 /// Remove entities not representable by SPIR-V
@@ -511,9 +570,12 @@ bool SPIRVRegularizeLLVMBase::regularize() {
       continue;
     }
 
+    // TODO: query intrinsic calls from their declarations
     std::vector<Instruction *> ToErase;
     for (BasicBlock &BB : *F) {
       for (Instruction &II : BB) {
+        if (auto *MD = II.getMetadata(SPIRV_MD_INTEL_CACHE_DECORATIONS))
+          prepareCacheControlsTranslation(MD, &II);
         if (auto *Call = dyn_cast<CallInst>(&II)) {
           Call->setTailCall(false);
           Function *CF = Call->getCalledFunction();

test/extensions/INTEL/SPV_INTEL_cache_controls/basic_load_store_arg_dec.ll

@@ -0,0 +1,66 @@
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-spirv --spirv-ext=+SPV_INTEL_cache_controls -spirv-text %t.bc -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: llvm-spirv --spirv-ext=+SPV_INTEL_cache_controls %t.bc -o %t.spv
+; RUN: llvm-spirv -r %t.spv --spirv-target-env=SPV-IR -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK-LLVM
+
+; CHECK-SPIRV-DAG: TypeInt [[#Int32:]] 32 0
+; CHECK-SPIRV-DAG: Constant [[#Int32]] [[#Zero:]] 0
+; CHECK-SPIRV-DAG: Decorate [[#Load1GEPPtr:]] CacheControlLoadINTEL 0 1
+; CHECK-SPIRV-DAG: Decorate [[#Load2GEPPtr:]] CacheControlLoadINTEL 1 1
+; CHECK-SPIRV-DAG: Decorate [[#Store1GEPPtr:]] CacheControlStoreINTEL 0 1
+; CHECK-SPIRV-DAG: Decorate [[#Store2GEPPtr:]] CacheControlStoreINTEL 1 1
+
+; CHECK-SPIRV: FunctionParameter [[#]] [[#Buffer:]]
+; CHECK-SPIRV: PtrAccessChain [[#]] [[#Load1GEPPtr:]] [[#Buffer]] [[#Zero]]
+; CHECK-SPIRV: Load [[#]] [[#]] [[#Load1GEPPtr]]
+; CHECK-SPIRV: PtrAccessChain [[#]] [[#Load2GEPPtr:]] [[#]] [[#Zero]]
+; CHECK-SPIRV: Load [[#]] [[#]] [[#Load2GEPPtr]]
+; CHECK-SPIRV: PtrAccessChain [[#]] [[#Store1GEPPtr:]] [[#]] [[#Zero]]
+; CHECK-SPIRV: Store [[#Store1GEPPtr]]
+; CHECK-SPIRV: PtrAccessChain [[#]] [[#Store2GEPPtr:]] [[#]] [[#Zero]]
+; CHECK-SPIRV: Store [[#Store2GEPPtr]]
+
+; CHECK-LLVM: %[[#GEPLoad1:]] = getelementptr i32, ptr addrspace(1) %{{.*}}, i32 0, !spirv.Decorations ![[#Cache1:]]
+; CHECK-LLVM: load i32, ptr addrspace(1) %[[#GEPLoad1]], align 4
+; CHECK-LLVM: %[[#GEPLoad2:]] = getelementptr i32, ptr addrspace(1) %{{.*}}, i32 0, !spirv.Decorations ![[#Cache2:]]
+; CHECK-LLVM: load i32, ptr addrspace(1) %[[#GEPLoad2]], align 4
+; CHECK-LLVM: %[[#GEPStore1:]] = getelementptr i32, ptr addrspace(1) %{{.*}}, i32 0, !spirv.Decorations ![[#Cache3:]]
+; CHECK-LLVM: store i32 %[[#]], ptr addrspace(1) %[[#GEPStore1]], align 4
+; CHECK-LLVM: %[[#GEPStore2:]] = getelementptr i32, ptr addrspace(1) %{{.*}}, i32 0, !spirv.Decorations ![[#Cache4:]]
+; CHECK-LLVM: store i32 %[[#]], ptr addrspace(1) %[[#GEPStore2]], align 4
+; CHECK-LLVM: ![[#Cache1]] = !{![[#DecLoad1:]]}
+; CHECK-LLVM: ![[#DecLoad1]] = !{i32 6442, i32 0, i32 1}
+; CHECK-LLVM: ![[#Cache2]] = !{![[#DecLoad2:]]}
+; CHECK-LLVM: ![[#DecLoad2]] = !{i32 6442, i32 1, i32 1}
+; CHECK-LLVM: ![[#Cache3:]] = !{![[#DecStore1:]]}
+; CHECK-LLVM: ![[#DecStore1]] = !{i32 6443, i32 0, i32 1}
+; CHECK-LLVM: ![[#Cache4:]] = !{![[#DecStore2:]]}
+; CHECK-LLVM: ![[#DecStore2]] = !{i32 6443, i32 1, i32 1}
+
+target triple = "spir64-unknown-unknown"
+
+define spir_kernel void @test(ptr addrspace(1) %buffer) {
+entry:
+  %0 = load i32, ptr addrspace(1) %buffer, align 4, !spirv.DecorationCacheControlINTEL !3
+  %1 = load i32, ptr addrspace(1) %buffer, align 4, !spirv.DecorationCacheControlINTEL !5
+  store i32 %0, ptr addrspace(1) %buffer, align 4, !spirv.DecorationCacheControlINTEL !7
+  store i32 %1, ptr addrspace(1) %buffer, align 4, !spirv.DecorationCacheControlINTEL !9
+  ret void
+}
+
+!spirv.MemoryModel = !{!0}
+!spirv.Source = !{!1}
+!opencl.spir.version = !{!2}
+!opencl.ocl.version = !{!2}
+
+!0 = !{i32 2, i32 2}
+!1 = !{i32 3, i32 102000}
+!2 = !{i32 1, i32 2}
+!3 = !{!4}
+!4 = !{i32 6442, i32 0, i32 1, i32 0}
+!5 = !{!6}
+!6 = !{i32 6442, i32 1, i32 1, i32 0}
+!7 = !{!8}
+!8 = !{i32 6443, i32 0, i32 1, i32 1}
+!9 = !{!10}
+!10 = !{i32 6443, i32 1, i32 1, i32 1}

test/extensions/INTEL/SPV_INTEL_cache_controls/call_inst_arg_dec.ll

@@ -0,0 +1,43 @@
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-spirv --spirv-ext=+SPV_INTEL_cache_controls -spirv-text %t.bc -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: llvm-spirv --spirv-ext=+SPV_INTEL_cache_controls %t.bc -o %t.spv
+; RUN: llvm-spirv -r %t.spv --spirv-target-env=SPV-IR -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK-LLVM
+
+; CHECK-SPIRV: Decorate [[#GEP1:]] CacheControlLoadINTEL 0 1
+; CHECK-SPIRV: Decorate [[#GEP2:]] CacheControlStoreINTEL 0 1
+; CHECK-SPIRV: TypeInt [[#Int32Ty:]] 32 0
+; CHECK-SPIRV: Constant [[#Int32Ty]] [[#Zero:]] 0
+; CHECK-SPIRV: PtrAccessChain [[#]] [[#GEP1]] [[#]] [[#Zero]]
+; CHECK-SPIRV: PtrAccessChain [[#]] [[#GEP2]] [[#]] [[#Zero]]
+; CHECK-SPIRV: FunctionCall [[#]] [[#]] [[#]] [[#GEP1]] [[#GEP2]]
+
+; CHECK-LLVM: %[[#GEP1:]] = getelementptr ptr addrspace(1), ptr addrspace(1) %{{.*}}, i32 0, !spirv.Decorations ![[#Cache1:]]
+; CHECK-LLVM: %[[#GEP2:]] = getelementptr ptr addrspace(1), ptr addrspace(1) %{{.*}}, i32 0, !spirv.Decorations ![[#Cache2:]]
+; CHECK-LLVM: call spir_func void @foo(ptr addrspace(1) %[[#GEP1]], ptr addrspace(1) %[[#GEP2]])
+; CHECK-LLVM: ![[#Cache1]] = !{![[#LoadCache:]]}
+; CHECK-LLVM: ![[#LoadCache]] = !{i32 6442, i32 0, i32 1}
+; CHECK-LLVM: ![[#Cache2]] = !{![[#StoreCache:]]}
+; CHECK-LLVM: ![[#StoreCache]] = !{i32 6443, i32 0, i32 1}
+
+target triple = "spir64-unknown-unknown"
+
+define spir_kernel void @test(ptr addrspace(1) %buffer1, ptr addrspace(1) %buffer2) {
+entry:
+  call void @foo(ptr addrspace(1) %buffer1, ptr addrspace(1) %buffer2), !spirv.DecorationCacheControlINTEL !3
+  ret void
+}
+
+declare void @foo(ptr addrspace(1), ptr addrspace(1))
+
+!spirv.MemoryModel = !{!0}
+!spirv.Source = !{!1}
+!opencl.spir.version = !{!2}
+!opencl.ocl.version = !{!2}
+
+!0 = !{i32 2, i32 2}
+!1 = !{i32 3, i32 102000}
+!2 = !{i32 1, i32 2}
+!3 = !{!4, !5}
+!4 = !{i32 6442, i32 0, i32 1, i32 0}
+!5 = !{i32 6443, i32 0, i32 1, i32 1}
+