Add support for fpga latency control extension (#1893)

This change adds SPIRV-LLVM-Translator changes for SPV_INTEL_fpga_latency_control extension.
The extension can be found here: KhronosGroup/SPIRV-Registry#179
The SPIR-V headers changes are here: KhronosGroup/SPIRV-Headers#321

Annotated pipes are not supported yet.

Thanks

Signed-off-by: Arvind Sudarsanam <[email protected]>

Original commit:
KhronosGroup/SPIRV-LLVM-Translator@0933ebb

Author: asudarsa

llvm-spirv/include/LLVMSPIRVExtensions.inc

@@ -58,3 +58,4 @@ EXT(SPV_INTEL_masked_gather_scatter)
 EXT(SPV_INTEL_tensor_float32_conversion)
 EXT(SPV_EXT_relaxed_printf_string_address_space)
 EXT(SPV_INTEL_fpga_argument_interfaces)
+EXT(SPV_INTEL_fpga_latency_control)

llvm-spirv/lib/SPIRV/SPIRVReader.cpp

@@ -3377,6 +3377,16 @@ void generateIntelFPGAAnnotation(
     Out << "{force_pow2_depth:" << Result << '}';
   if (E->hasDecorate(DecorationBufferLocationINTEL, 0, &Result))
     Out << "{sycl-buffer-location:" << Result << '}';
+  if (E->hasDecorate(DecorationLatencyControlLabelINTEL, 0, &Result))
+    Out << "{sycl-latency-anchor-id:" << Result << '}';
+  if (E->hasDecorate(DecorationLatencyControlConstraintINTEL)) {
+    auto Literals =
+        E->getDecorationLiterals(DecorationLatencyControlConstraintINTEL);
+    assert(Literals.size() == 3 &&
+           "Latency Control Constraint decoration shall have 3 extra operands");
+    Out << "{sycl-latency-constraint:" << Literals[0] << "," << Literals[1]
+        << "," << Literals[2] << '}';
+  }
 
   unsigned LSUParamsBitmask = 0;
   llvm::SmallString<32> AdditionalParamsStr;

llvm-spirv/lib/SPIRV/SPIRVWriter.cpp

@@ -2958,10 +2958,11 @@ struct AnnotationDecorations {
   DecorationsInfoVec MemoryAttributesVec;
   DecorationsInfoVec MemoryAccessesVec;
   DecorationsInfoVec BufferLocationVec;
+  DecorationsInfoVec LatencyControlVec;
 
   bool empty() {
     return (MemoryAttributesVec.empty() && MemoryAccessesVec.empty() &&
-            BufferLocationVec.empty());
+            BufferLocationVec.empty() && LatencyControlVec.empty());
   }
 };
 
@@ -3095,8 +3096,8 @@ static bool tryParseAnnotationDecoValues(StringRef ValueStr,
           return false;
         // Skip the , delimiter and go directly to the start of next value.
         ValueStart = (++I) + 1;
+        continue;
       }
-      continue;
     }
     if (CurrentC == ',') {
       // Since we are not currently in a string literal, comma denotes a
@@ -3131,7 +3132,6 @@ static bool tryParseAnnotationDecoValues(StringRef ValueStr,
 AnnotationDecorations tryParseAnnotationString(SPIRVModule *BM,
                                                StringRef AnnotatedCode) {
   AnnotationDecorations Decorates;
-
   // Annotation string decorations are separated into {word} OR
   // {word:value,value,...} blocks, where value is either a word (including
   // numbers) or a quotation mark enclosed string.
@@ -3155,6 +3155,8 @@ AnnotationDecorations tryParseAnnotationString(SPIRVModule *BM,
       ExtensionID::SPV_INTEL_fpga_memory_attributes);
   const bool AllowFPGABufLoc =
       BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_fpga_buffer_location);
+  const bool AllowFPGALatencyControl =
+      BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_fpga_latency_control);
 
   bool ValidDecorationFound = false;
   DecorationsInfoVec DecorationsVec;
@@ -3178,6 +3180,12 @@ AnnotationDecorations tryParseAnnotationString(SPIRVModule *BM,
             DecorationKind == DecorationBufferLocationINTEL) {
           Decorates.BufferLocationVec.emplace_back(
               static_cast<Decoration>(DecorationKind), std::move(DecValues));
+        } else if (AllowFPGALatencyControl &&
+                   (DecorationKind == DecorationLatencyControlLabelINTEL ||
+                    DecorationKind ==
+                        DecorationLatencyControlConstraintINTEL)) {
+          Decorates.LatencyControlVec.emplace_back(
+              static_cast<Decoration>(DecorationKind), std::move(DecValues));
         } else {
           DecorationsVec.emplace_back(static_cast<Decoration>(DecorationKind),
                                       std::move(DecValues));
@@ -3270,12 +3278,12 @@ AnnotationDecorations tryParseAnnotationString(SPIRVModule *BM,
 }
 
 std::vector<SPIRVWord>
-getBankBitsFromStrings(const std::vector<std::string> &BitsStrings) {
-  std::vector<SPIRVWord> Bits(BitsStrings.size());
-  for (size_t J = 0; J < BitsStrings.size(); ++J)
-    if (StringRef(BitsStrings[J]).getAsInteger(10, Bits[J]))
+getLiteralsFromStrings(const std::vector<std::string> &Strings) {
+  std::vector<SPIRVWord> Literals(Strings.size());
+  for (size_t J = 0; J < Strings.size(); ++J)
+    if (StringRef(Strings[J]).getAsInteger(10, Literals[J]))
       return {};
-  return Bits;
+  return Literals;
 }
 
 void addAnnotationDecorations(SPIRVEntry *E, DecorationsInfoVec &Decorations) {
@@ -3321,7 +3329,7 @@ void addAnnotationDecorations(SPIRVEntry *E, DecorationsInfoVec &Decorations) {
             I.second.size() > 0, SPIRVEC_InvalidLlvmModule,
             "BankBitsINTEL requires at least one argument.");
         E->addDecorate(new SPIRVDecorateBankBitsINTELAttr(
-            E, getBankBitsFromStrings(I.second)));
+            E, getLiteralsFromStrings(I.second)));
       }
     } break;
     case DecorationRegisterINTEL:
@@ -3382,7 +3390,32 @@ void addAnnotationDecorations(SPIRVEntry *E, DecorationsInfoVec &Decorations) {
         E->addDecorate(I.first, Result);
       }
     } break;
-
+    case DecorationLatencyControlLabelINTEL: {
+      if (M->isAllowedToUseExtension(
+              ExtensionID::SPV_INTEL_fpga_latency_control)) {
+        M->getErrorLog().checkError(
+            I.second.size() == 1, SPIRVEC_InvalidLlvmModule,
+            "LatencyControlLabelINTEL requires exactly 1 extra operand");
+        SPIRVWord Label = 0;
+        StringRef(I.second[0]).getAsInteger(10, Label);
+        E->addDecorate(
+            new SPIRVDecorate(DecorationLatencyControlLabelINTEL, E, Label));
+      }
+      break;
+    }
+    case DecorationLatencyControlConstraintINTEL: {
+      if (M->isAllowedToUseExtension(
+              ExtensionID::SPV_INTEL_fpga_latency_control)) {
+        M->getErrorLog().checkError(
+            I.second.size() == 3, SPIRVEC_InvalidLlvmModule,
+            "LatencyControlConstraintINTEL requires exactly 3 extra operands");
+        auto Literals = getLiteralsFromStrings(I.second);
+        E->addDecorate(
+            new SPIRVDecorate(DecorationLatencyControlConstraintINTEL, E,
+                              Literals[0], Literals[1], Literals[2]));
+      }
+      break;
+    }
     default:
       // Other decorations are either not supported by the translator or
       // handled in other places.
@@ -3431,7 +3464,7 @@ void addAnnotationDecorationsForStructMember(SPIRVEntry *E,
           I.second.size() > 0, SPIRVEC_InvalidLlvmModule,
           "BankBitsINTEL requires at least one argument.");
       E->addMemberDecorate(new SPIRVMemberDecorateBankBitsINTELAttr(
-          E, MemberNumber, getBankBitsFromStrings(I.second)));
+          E, MemberNumber, getLiteralsFromStrings(I.second)));
       break;
     case DecorationRegisterINTEL:
     case DecorationSinglepumpINTEL:
@@ -3648,7 +3681,7 @@ static bool allowsApproxFunction(IntrinsicInst *II) {
            cast<VectorType>(Ty)->getElementType()->isFloatTy()));
 }
 
-bool allowDecorateWithBufferLocationINTEL(IntrinsicInst *II) {
+bool allowDecorateWithBufferLocationOrLatencyControlINTEL(IntrinsicInst *II) {
   SmallVector<Value *, 8> UserList;
 
   for (auto *Inst : II->users()) {
@@ -4153,15 +4186,18 @@ SPIRVValue *LLVMToSPIRVBase::transIntrinsicInst(IntrinsicInst *II,
         // because multiple accesses to the struct-held memory can require
         // different LSU parameters.
         addAnnotationDecorations(ResPtr, Decorations.MemoryAccessesVec);
-        if (allowDecorateWithBufferLocationINTEL(II))
+        if (allowDecorateWithBufferLocationOrLatencyControlINTEL(II)) {
           addAnnotationDecorations(ResPtr, Decorations.BufferLocationVec);
+          addAnnotationDecorations(ResPtr, Decorations.LatencyControlVec);
+        }
       }
       II->replaceAllUsesWith(II->getOperand(0));
     } else {
       // Memory accesses to a standalone pointer variable
       auto *DecSubj = transValue(II->getArgOperand(0), BB);
       if (Decorations.MemoryAccessesVec.empty() &&
-          Decorations.BufferLocationVec.empty())
+          Decorations.BufferLocationVec.empty() &&
+          Decorations.LatencyControlVec.empty())
         DecSubj->addDecorate(new SPIRVDecorateUserSemanticAttr(
             DecSubj, AnnotationString.c_str()));
       else {
@@ -4170,8 +4206,10 @@ SPIRVValue *LLVMToSPIRVBase::transIntrinsicInst(IntrinsicInst *II,
         // loaded from the original pointer variable, and not the value
         // accessed by the latter.
         addAnnotationDecorations(DecSubj, Decorations.MemoryAccessesVec);
-        if (allowDecorateWithBufferLocationINTEL(II))
+        if (allowDecorateWithBufferLocationOrLatencyControlINTEL(II)) {
           addAnnotationDecorations(DecSubj, Decorations.BufferLocationVec);
+          addAnnotationDecorations(DecSubj, Decorations.LatencyControlVec);
+        }
       }
       II->replaceAllUsesWith(II->getOperand(0));
     }

llvm-spirv/lib/SPIRV/libSPIRV/SPIRVDecorate.cpp

@@ -81,6 +81,16 @@ SPIRVDecorateGeneric::SPIRVDecorateGeneric(Op OC, SPIRVWord WC,
   updateModuleVersion();
 }
 
+SPIRVDecorateGeneric::SPIRVDecorateGeneric(Op OC, SPIRVWord WC,
+                                           Decoration TheDec,
+                                           SPIRVEntry *TheTarget, SPIRVWord V1,
+                                           SPIRVWord V2, SPIRVWord V3)
+    : SPIRVDecorateGeneric(OC, WC, TheDec, TheTarget, V1, V2) {
+  Literals.push_back(V3);
+  validate();
+  updateModuleVersion();
+}
+
 SPIRVDecorateGeneric::SPIRVDecorateGeneric(Op OC)
     : SPIRVAnnotationGeneric(OC), Dec(DecorationRelaxedPrecision),
       Owner(nullptr) {}

llvm-spirv/lib/SPIRV/libSPIRV/SPIRVDecorate.h

@@ -60,6 +60,11 @@ class SPIRVDecorateGeneric : public SPIRVAnnotationGeneric {
   // Complete constructor for decorations with two word literals
   SPIRVDecorateGeneric(Op OC, SPIRVWord WC, Decoration TheDec,
                        SPIRVEntry *TheTarget, SPIRVWord V1, SPIRVWord V2);
+  // Complete constructor for decorations with three word literals
+  SPIRVDecorateGeneric(Op OC, SPIRVWord WC, Decoration TheDec,
+                       SPIRVEntry *TheTarget, SPIRVWord V1, SPIRVWord V2,
+                       SPIRVWord V3);
+
   // Incomplete constructor
   SPIRVDecorateGeneric(Op OC);
 
@@ -124,6 +129,11 @@ class SPIRVDecorate : public SPIRVDecorateGeneric {
   SPIRVDecorate(Decoration TheDec, SPIRVEntry *TheTarget, SPIRVWord V1,
                 SPIRVWord V2)
       : SPIRVDecorateGeneric(OC, 5, TheDec, TheTarget, V1, V2) {}
+  // Complete constructor for decorations with three word literals
+  SPIRVDecorate(Decoration TheDec, SPIRVEntry *TheTarget, SPIRVWord V1,
+                SPIRVWord V2, SPIRVWord V3)
+      : SPIRVDecorateGeneric(OC, 6, TheDec, TheTarget, V1, V2, V3) {}
+
   // Incomplete constructor
   SPIRVDecorate() : SPIRVDecorateGeneric(OC) {}
 
@@ -188,6 +198,9 @@ class SPIRVDecorate : public SPIRVDecorateGeneric {
     case DecorationMMHostInterfaceMaxBurstINTEL:
     case DecorationMMHostInterfaceWaitRequestINTEL:
       return ExtensionID::SPV_INTEL_fpga_argument_interfaces;
+    case DecorationLatencyControlLabelINTEL:
+    case DecorationLatencyControlConstraintINTEL:
+      return ExtensionID::SPV_INTEL_fpga_latency_control;
     default:
       return {};
     }

llvm-spirv/lib/SPIRV/libSPIRV/SPIRVEnum.h

@@ -491,6 +491,10 @@ template <> inline void SPIRVMap<Decoration, SPIRVCapVec>::init() {
                {CapabilityFPGAArgumentInterfacesINTEL});
   ADD_VEC_INIT(DecorationStableKernelArgumentINTEL,
                {CapabilityFPGAArgumentInterfacesINTEL});
+  ADD_VEC_INIT(DecorationLatencyControlLabelINTEL,
+               {CapabilityFPGALatencyControlINTEL});
+  ADD_VEC_INIT(DecorationLatencyControlConstraintINTEL,
+               {CapabilityFPGALatencyControlINTEL});
 }
 
 template <> inline void SPIRVMap<BuiltIn, SPIRVCapVec>::init() {

llvm-spirv/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h

@@ -209,6 +209,8 @@ template <> inline void SPIRVMap<Decoration, std::string>::init() {
   add(DecorationMMHostInterfaceWaitRequestINTEL,
       "MMHostInterfaceWaitRequestINTEL");
   add(DecorationStableKernelArgumentINTEL, "StableKernelArgumentINTEL");
+  add(DecorationLatencyControlLabelINTEL, "LatencyControlLabelINTEL");
+  add(DecorationLatencyControlConstraintINTEL, "LatencyControlConstraintINTEL");
 
   // From spirv_internal.hpp
   add(internal::DecorationFuncParamKindINTEL, "FuncParamKindINTEL");
@@ -633,7 +635,7 @@ template <> inline void SPIRVMap<Capability, std::string>::init() {
   add(CapabilityRuntimeAlignedAttributeINTEL, "RuntimeAlignedAttributeINTEL");
   add(CapabilityMax, "Max");
   add(CapabilityFPGAArgumentInterfacesINTEL, "FPGAArgumentInterfacesINTEL");
-
+  add(CapabilityFPGALatencyControlINTEL, "FPGALatencyControlINTEL");
   // From spirv_internal.hpp
   add(internal::CapabilityFastCompositeINTEL, "FastCompositeINTEL");
   add(internal::CapabilityOptNoneINTEL, "OptNoneINTEL");

llvm-spirv/spirv-headers-tag.conf

@@ -1 +1 @@
-295cf5fb3bfe2454360e82b26bae7fc0de699abe
+1feaf4414eb2b353764d01d88f8aa4bcc67b60db

llvm-spirv/test/extensions/INTEL/SPV_INTEL_fpga_latency_control/IntelFPGALatencyControl.ll

@@ -0,0 +1,74 @@
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_INTEL_fpga_latency_control -o %t.spv
+; RUN: llvm-spirv %t.spv -to-text -o %t.spt
+; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV
+
+; RUN: llvm-spirv -r -emit-opaque-pointers %t.spv -o %t.rev.bc
+; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM
+
+; CHECK-SPIRV: Capability FPGALatencyControlINTEL
+; CHECK-SPIRV: Extension "SPV_INTEL_fpga_latency_control"
+; CHECK-SPIRV: Decorate [[#ARGA:]] LatencyControlLabelINTEL 0
+; CHECK-SPIRV: Decorate [[#ARGB:]] LatencyControlLabelINTEL 1
+; CHECK-SPIRV: Decorate [[#ARGB]] LatencyControlConstraintINTEL 0 1 5
+; CHECK-SPIRV: Bitcast [[#]] [[#OUT1:]] [[#ARGA]]
+; CHECK-SPIRV-DAG: Bitcast [[#]] [[#OUT2:]] [[#OUT1]]
+; CHECK-SPIRV-DAG: Load [[#]] [[#]] [[#OUT2]] [[#]] [[#]]
+; CHECK-SPIRV: Bitcast [[#]] [[#OUT3:]] [[#ARGB]]
+; CHECK-SPIRV-DAG: Bitcast [[#]] [[#OUT4:]] [[#OUT3]]
+; CHECK-SPIRV-DAG: Load [[#]] [[#]] [[#OUT4]] [[#]] [[#]]
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
+target triple = "spir64-unknown-unknown"
+
+%struct.__spirv_Something = type { i32, i32 }
+
+$_ZTSZ4fooEUlvE_ = comdat any
+
+@.str = private unnamed_addr constant [16 x i8] c"sycl-properties\00", section "llvm.metadata"
+@.str.1 = private unnamed_addr constant [19 x i8] c"inc/fpga_utils.hpp\00", section "llvm.metadata"
+@.str.9 = private unnamed_addr constant [11 x i8] c"{6172:\220\22}\00", section "llvm.metadata"
+@.str.10 = private unnamed_addr constant [25 x i8] c"{6172:\221\22}{6173:\220,1,5\22}\00", section "llvm.metadata"
+
+; CHECK-LLVM: @[[#ANN_STR1:]] = private unnamed_addr constant [27 x i8] c"{sycl-latency-anchor-id:0}\00"
+; CHECK-LLVM: @[[#ANN_STR2:]] = private unnamed_addr constant [58 x i8] c"{sycl-latency-anchor-id:1}{sycl-latency-constraint:0,1,5}\00"
+
+; Function Attrs: mustprogress norecurse
+define weak_odr dso_local spir_kernel void @_ZTSZ4fooEUlvE_(ptr %0) local_unnamed_addr #0 comdat !kernel_arg_buffer_location !5 !sycl_kernel_omit_args !5 {
+entry:
+  %1 = alloca ptr, align 8
+  store ptr %0, ptr %1, align 8
+  %2 = load ptr, ptr %1, align 8
+  %3 = getelementptr inbounds %struct.__spirv_Something, ptr %2, i32 0, i32 0
+  %4 = bitcast ptr %3 to ptr
+; CHECK-LLVM: %[[#ANN_PTR1:]] = getelementptr inbounds %struct.__spirv_Something, ptr %[[#]], i32 0, i32 0
+  %5 = call ptr @llvm.ptr.annotation.p0.p0(ptr %4, ptr @.str.9, ptr @.str.1, i32 5, ptr null)
+; CHECK-LLVM: call ptr @llvm.ptr.annotation.p0.p0(ptr %[[#ANN_PTR1]], ptr @[[#ANN_STR1]], ptr undef, i32 undef, ptr undef)
+  %6 = load i32, ptr %5, align 8
+  %7 = load ptr, ptr %1, align 8
+  %8 = getelementptr inbounds %struct.__spirv_Something, ptr %7, i32 0, i32 1
+  %9 = bitcast ptr %8 to ptr
+; CHECK-LLVM: %[[#ANN_PTR2:]] = getelementptr inbounds %struct.__spirv_Something, ptr %[[#]], i32 0, i32 1
+  %10 = call ptr @llvm.ptr.annotation.p0.p0(ptr %9, ptr @.str.10, ptr @.str.1, i32 5, ptr null)
+; CHECK-LLVM: call ptr @llvm.ptr.annotation.p0.p0(ptr %[[#ANN_PTR2]], ptr @[[#ANN_STR2]], ptr undef, i32 undef, ptr undef)
+  %11 = load i32, ptr %10, align 8
+  ret void
+}
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite)
+declare ptr @llvm.ptr.annotation.p0.p0(ptr, ptr, ptr, i32, ptr) #1
+
+attributes #0 = { mustprogress norecurse "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="sycl-properties-ptr-annotations.cpp" "uniform-work-group-size"="true" }
+attributes #1 = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite) }
+
+!opencl.spir.version = !{!0, !0, !0, !0, !0, !0}
+!spirv.Source = !{!1, !1, !1, !1, !1, !1}
+!llvm.ident = !{!2, !2, !2, !2, !2, !2}
+!llvm.module.flags = !{!3, !4}
+
+!0 = !{i32 1, i32 2}
+!1 = !{i32 4, i32 100000}
+!2 = !{!"clang version 15.0.0"}
+!3 = !{i32 1, !"wchar_size", i32 4}
+!4 = !{i32 7, !"frame-pointer", i32 2}
+!5 = !{}