Skip to content

Add support for split barriers extension SPV_INTEL_split_barrier #332

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 1, 2022
Merged

Add support for split barriers extension SPV_INTEL_split_barrier #332

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
375 changes: 375 additions & 0 deletions patches/spirv/0002-Add-support-for-split-barriers-extension-SPV_INTEL_s.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,375 @@
From d231d4aea8c48e0c5d15cf247980361c1d6c57c5 Mon Sep 17 00:00:00 2001
From: haonanya <[email protected]>
Date: Mon, 28 Feb 2022 18:20:31 +0800
Subject: [PATCH] Add support for split barriers extension
SPV_INTEL_split_barrier

Signed-off-by: haonanya <[email protected]>
---
include/LLVMSPIRVExtensions.inc | 1 +
lib/SPIRV/OCL20ToSPIRV.cpp | 38 +++++++++++++++++++++++++++
lib/SPIRV/OCLUtil.cpp | 23 +++++++++++++++-
lib/SPIRV/OCLUtil.h | 20 ++++++++++++++
lib/SPIRV/SPIRVReader.cpp | 3 ++-
lib/SPIRV/SPIRVToOCL.cpp | 4 +++
lib/SPIRV/SPIRVToOCL.h | 5 ++++
lib/SPIRV/SPIRVToOCL12.cpp | 17 ++++++++++++
lib/SPIRV/SPIRVToOCL20.cpp | 26 ++++++++++++++++++
lib/SPIRV/libSPIRV/SPIRVInstruction.h | 18 +++++++++++++
lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h | 1 +
lib/SPIRV/libSPIRV/SPIRVOpCode.h | 5 ++++
lib/SPIRV/libSPIRV/SPIRVOpCodeEnum.h | 2 ++
13 files changed, 161 insertions(+), 2 deletions(-)

diff --git a/include/LLVMSPIRVExtensions.inc b/include/LLVMSPIRVExtensions.inc
index 7580e1f8..aa56b29b 100644
--- a/include/LLVMSPIRVExtensions.inc
+++ b/include/LLVMSPIRVExtensions.inc
@@ -28,3 +28,4 @@ EXT(SPV_INTEL_optnone)
EXT(SPV_INTEL_arbitrary_precision_integers)
EXT(SPV_INTEL_variable_length_array)
EXT(SPV_INTEL_memory_access_aliasing)
+EXT(SPV_INTEL_split_barrier)
diff --git a/lib/SPIRV/OCL20ToSPIRV.cpp b/lib/SPIRV/OCL20ToSPIRV.cpp
index b62b433c..cda026ae 100644
--- a/lib/SPIRV/OCL20ToSPIRV.cpp
+++ b/lib/SPIRV/OCL20ToSPIRV.cpp
@@ -279,6 +279,9 @@ public:
StringRef MangledName,
const std::string &DemangledName);

+ /// For cl_intel_split_work_group_barrier built-ins:
+ void visitCallSplitBarrierINTEL(CallInst *CI, StringRef DemangledName);
+
void visitCallLdexp(CallInst *CI, StringRef MangledName,
StringRef DemangledName);

@@ -546,6 +549,10 @@ void OCL20ToSPIRV::visitCallInst(CallInst &CI) {
visitSubgroupImageMediaBlockINTEL(&CI, DemangledName);
return;
}
+ if (DemangledName.find(kOCLBuiltinName::SplitBarrierINTELPrefix) == 0) {
+ visitCallSplitBarrierINTEL(&CI, DemangledName);
+ return;
+ }
// Handle 'cl_intel_device_side_avc_motion_estimation' extension built-ins
if (DemangledName.find(kOCLSubgroupsAVCIntel::Prefix) == 0 ||
// Workaround for a bug in the extension specification
@@ -563,6 +570,37 @@ void OCL20ToSPIRV::visitCallInst(CallInst &CI) {
visitCallBuiltinSimple(&CI, MangledName, DemangledName);
}

+void OCL20ToSPIRV::visitCallSplitBarrierINTEL(CallInst *CI,
+ StringRef DemangledName) {
+ auto Lit = getBarrierLiterals(CI);
+ AttributeList Attrs = CI->getCalledFunction()->getAttributes();
+ Op OpCode =
+ StringSwitch<Op>(DemangledName)
+ .Case("intel_work_group_barrier_arrive", OpControlBarrierArriveINTEL)
+ .Case("intel_work_group_barrier_wait", OpControlBarrierWaitINTEL)
+ .Default(OpNop);
+
+ mutateCallInstSPIRV(
+ M, CI,
+ [=](CallInst *, std::vector<Value *> &Args) {
+ Args.resize(3);
+ // Execution scope
+ Args[0] = addInt32(map<Scope>(std::get<2>(Lit)));
+ // Memory scope
+ Args[1] = addInt32(map<Scope>(std::get<1>(Lit)));
+ // Memory semantics
+ // OpControlBarrierArriveINTEL -> Release,
+ // OpControlBarrierWaitINTEL -> Acquire
+ unsigned MemFenceFlag = std::get<0>(Lit);
+ OCLMemOrderKind MemOrder = OpCode == OpControlBarrierArriveINTEL
+ ? OCLMO_release
+ : OCLMO_acquire;
+ Args[2] = addInt32(mapOCLMemSemanticToSPIRV(MemFenceFlag, MemOrder));
+ return getSPIRVFuncName(OpCode);
+ },
+ &Attrs);
+}
+
void OCL20ToSPIRV::visitCallNDRange(CallInst *CI,
const std::string &DemangledName) {
assert(DemangledName.find(kOCLBuiltinName::NDRangePrefix) == 0);
diff --git a/lib/SPIRV/OCLUtil.cpp b/lib/SPIRV/OCLUtil.cpp
index fcdd16eb..f93338eb 100644
--- a/lib/SPIRV/OCLUtil.cpp
+++ b/lib/SPIRV/OCLUtil.cpp
@@ -471,7 +471,9 @@ public:
} else if (UnmangledName.find("barrier") != std::string::npos) {
addUnsignedArg(0);
if (UnmangledName == "work_group_barrier" ||
- UnmangledName == "sub_group_barrier")
+ UnmangledName == "sub_group_barrier" ||
+ UnmangledName == "intel_work_group_barrier_arrive" ||
+ UnmangledName == "intel_work_group_barrier_wait")
setEnumArg(1, SPIR::PRIMITIVE_MEMORY_SCOPE);
} else if (UnmangledName.find("atomic_work_item_fence") == 0) {
addUnsignedArg(0);
@@ -985,6 +987,25 @@ void insertImageNameAccessQualifier(SPIRVAccessQualifierKind Acc,
}
} // namespace OCLUtil

+Value *SPIRV::transSPIRVMemorySemanticsIntoOCLMemFenceFlags(
+ Value *MemorySemantics, Instruction *InsertBefore) {
+ if (auto *C = dyn_cast<ConstantInt>(MemorySemantics)) {
+ return ConstantInt::get(C->getType(),
+ mapSPIRVMemSemanticToOCL(C->getZExtValue()).first);
+ }
+
+ // TODO: any possible optimizations?
+ // SPIR-V MemorySemantics contains both OCL mem_fence_flags and mem_order and
+ // therefore, we need to apply mask
+ int Mask = MemorySemanticsWorkgroupMemoryMask |
+ MemorySemanticsCrossWorkgroupMemoryMask |
+ MemorySemanticsImageMemoryMask;
+ return getOrCreateSwitchFunc(kSPIRVName::TranslateSPIRVMemFence,
+ MemorySemantics,
+ OCLMemFenceExtendedMap::getRMap(),
+ /* IsReverse */ true, None, InsertBefore, InsertBefore->getModule(), Mask);
+}
+
void llvm::mangleOpenClBuiltin(const std::string &UniqName,
ArrayRef<Type *> ArgTypes,
std::string &MangledName) {
diff --git a/lib/SPIRV/OCLUtil.h b/lib/SPIRV/OCLUtil.h
index 9100b5d2..4e585cee 100644
--- a/lib/SPIRV/OCLUtil.h
+++ b/lib/SPIRV/OCLUtil.h
@@ -238,6 +238,7 @@ const static char SubgroupBlockWriteINTELPrefix[] =
"intel_sub_group_block_write";
const static char SubgroupImageMediaBlockINTELPrefix[] =
"intel_sub_group_media_block";
+const static char SplitBarrierINTELPrefix[] = "intel_work_group_barrier_";
const static char LDEXP[] = "ldexp";
} // namespace kOCLBuiltinName

@@ -545,6 +546,22 @@ getOrCreateSwitchFunc(StringRef MapName, Value *V,
return addCallInst(M, MapName, Ty, V, nullptr, InsertPoint);
}

+/// Performs conversion from SPIR-V Memory Semantics into OpenCL
+/// mem_fence_flags.
+///
+/// Supports both constant and non-constant values. To handle the latter case,
+/// function with switch..case statement will be inserted into module which
+/// \arg InsertBefore belongs to (in order to perform mapping at runtime)
+///
+/// \param [in] MemorySemantics Memory Semantics value which needs to be
+/// translated
+/// \param [in] InsertBefore insertion point for call into conversion function
+/// which is generated if \arg MemorySemantics is not a constant
+/// \returns \c Value corresponding to OpenCL mem_fence_flags equivalent to
+/// SPIR-V Memory Semantics passed in \arg MemorySemantics
+Value *transSPIRVMemorySemanticsIntoOCLMemFenceFlags(Value *MemorySemantics,
+ Instruction *InsertBefore);
+
template <> inline void SPIRVMap<std::string, SPIRVGroupOperationKind>::init() {
add("reduce", GroupOperationReduce);
add("scan_inclusive", GroupOperationInclusiveScan);
@@ -815,6 +832,9 @@ template <> inline void SPIRVMap<std::string, Op, SPIRVInstruction>::init() {
// cl_khr_subgroup_shuffle_relative
_SPIRV_OP(group_shuffle_up, GroupNonUniformShuffleUp)
_SPIRV_OP(group_shuffle_down, GroupNonUniformShuffleDown)
+ // cl_khr_split_work_group_barrier
+ _SPIRV_OP(intel_work_group_barrier_arrive, ControlBarrierArriveINTEL)
+ _SPIRV_OP(intel_work_group_barrier_wait, ControlBarrierWaitINTEL)
#undef _SPIRV_OP
}

diff --git a/lib/SPIRV/SPIRVReader.cpp b/lib/SPIRV/SPIRVReader.cpp
index cd90d979..d26411a9 100644
--- a/lib/SPIRV/SPIRVReader.cpp
+++ b/lib/SPIRV/SPIRVReader.cpp
@@ -2059,7 +2059,8 @@ Instruction *SPIRVToLLVM::transBuiltinFromInst(const std::string &FuncName,
if (isFuncNoUnwind())
Func->addFnAttr(Attribute::NoUnwind);
auto OC = BI->getOpCode();
- if (isGroupOpCode(OC) || isIntelSubgroupOpCode(OC))
+ if (isGroupOpCode(OC) || isIntelSubgroupOpCode(OC) ||
+ isSplitBarrierINTELOpCode(OC))
Func->addFnAttr(Attribute::Convergent);
}
auto Call =
diff --git a/lib/SPIRV/SPIRVToOCL.cpp b/lib/SPIRV/SPIRVToOCL.cpp
index 42b539c7..c485fb0d 100644
--- a/lib/SPIRV/SPIRVToOCL.cpp
+++ b/lib/SPIRV/SPIRVToOCL.cpp
@@ -106,6 +106,10 @@ void SPIRVToOCL::visitCallInst(CallInst &CI) {
if (OC == OpControlBarrier) {
visitCallSPIRVControlBarrier(&CI);
}
+ if (isSplitBarrierINTELOpCode(OC)) {
+ visitCallSPIRVSplitBarrierINTEL(&CI, OC);
+ return;
+ }
if (isAtomicOpCode(OC)) {
visitCallSPIRVAtomicBuiltin(&CI, OC);
return;
diff --git a/lib/SPIRV/SPIRVToOCL.h b/lib/SPIRV/SPIRVToOCL.h
index 127e8dd8..3577496b 100644
--- a/lib/SPIRV/SPIRVToOCL.h
+++ b/lib/SPIRV/SPIRVToOCL.h
@@ -211,6 +211,11 @@ public:
/// - OCL1.2: barrier
virtual void visitCallSPIRVControlBarrier(CallInst *CI) = 0;

+ /// Transform split __spirv_ControlBarrier barrier to:
+ /// - OCL2.0: overload with a memory_scope argument
+ /// - OCL1.2: overload with no memory_scope argument
+ virtual void visitCallSPIRVSplitBarrierINTEL(CallInst *CI, Op OC) = 0;
+
/// Transform __spirv_EnqueueKernel to __enqueue_kernel
virtual void visitCallSPIRVEnqueueKernel(CallInst *CI, Op OC) = 0;

diff --git a/lib/SPIRV/SPIRVToOCL12.cpp b/lib/SPIRV/SPIRVToOCL12.cpp
index 2b8b2967..f8084901 100644
--- a/lib/SPIRV/SPIRVToOCL12.cpp
+++ b/lib/SPIRV/SPIRVToOCL12.cpp
@@ -60,6 +60,10 @@ public:
/// barrier(flag(sema))
void visitCallSPIRVControlBarrier(CallInst *CI) override;

+ /// Transform split __spirv_ControlBarrier barrier to overloads without a
+ /// memory_scope argument.
+ void visitCallSPIRVSplitBarrierINTEL(CallInst *CI, Op OC) override;
+
/// Transform __spirv_OpAtomic functions. It firstly conduct generic
/// mutations for all builtins and then mutate some of them seperately
Instruction *visitCallSPIRVAtomicBuiltin(CallInst *CI, Op OC) override;
@@ -202,6 +206,19 @@ void SPIRVToOCL12::visitCallSPIRVControlBarrier(CallInst *CI) {
&Attrs);
}

+void SPIRVToOCL12::visitCallSPIRVSplitBarrierINTEL(CallInst *CI, Op OC) {
+ AttributeList Attrs = CI->getCalledFunction()->getAttributes();
+ mutateCallInstOCL(
+ M, CI,
+ [=](CallInst *, std::vector<Value *> &Args) {
+ Value *MemFenceFlags =
+ SPIRV::transSPIRVMemorySemanticsIntoOCLMemFenceFlags(Args[2], CI);
+ Args.assign(1, MemFenceFlags);
+ return OCLSPIRVBuiltinMap::rmap(OC);
+ },
+ &Attrs);
+}
+
Instruction *SPIRVToOCL12::visitCallSPIRVAtomicIncDec(CallInst *CI, Op OC) {
AttributeList Attrs = CI->getCalledFunction()->getAttributes();
return mutateCallInstOCL(
diff --git a/lib/SPIRV/SPIRVToOCL20.cpp b/lib/SPIRV/SPIRVToOCL20.cpp
index b7050246..fbe632ea 100644
--- a/lib/SPIRV/SPIRVToOCL20.cpp
+++ b/lib/SPIRV/SPIRVToOCL20.cpp
@@ -63,6 +63,10 @@ public:
/// sub_group_barrier(flag(sema), map(memScope))
void visitCallSPIRVControlBarrier(CallInst *CI) override;

+ /// Transform split __spirv_ControlBarrier barrier to overloads with a
+ /// memory_scope argument.
+ void visitCallSPIRVSplitBarrierINTEL(CallInst *CI, Op OC) override;
+
/// Transform __spirv_Atomic* to atomic_*.
/// __spirv_Atomic*(atomic_op, scope, sema, ops, ...) =>
/// atomic_*(generic atomic_op, ops, ..., order(sema), map(scope))
@@ -158,6 +162,28 @@ void SPIRVToOCL20::visitCallSPIRVControlBarrier(CallInst *CI) {
&Attrs);
}

+void SPIRVToOCL20::visitCallSPIRVSplitBarrierINTEL(CallInst *CI, Op OC) {
+ AttributeList Attrs = CI->getCalledFunction()->getAttributes();
+ mutateCallInstOCL(
+ M, CI,
+ [=](CallInst *, std::vector<Value *> &Args) {
+ auto GetArg = [=](unsigned I) {
+ return cast<ConstantInt>(Args[I])->getZExtValue();
+ };
+ Value *MemScope =
+ getInt32(M, rmap<OCLScopeKind>(static_cast<Scope>(GetArg(1))));
+ Value *MemFenceFlags =
+ SPIRV::transSPIRVMemorySemanticsIntoOCLMemFenceFlags(Args[2], CI);
+
+ Args.resize(2);
+ Args[0] = MemFenceFlags;
+ Args[1] = MemScope;
+
+ return OCLSPIRVBuiltinMap::rmap(OC);
+ },
+ &Attrs);
+}
+
std::string SPIRVToOCL20::mapFPAtomicName(Op OC) {
assert(isFPAtomicOpCode(OC) && "Not intended to handle other opcodes than "
"AtomicF{Add/Min/Max}EXT!");
diff --git a/lib/SPIRV/libSPIRV/SPIRVInstruction.h b/lib/SPIRV/libSPIRV/SPIRVInstruction.h
index 7ae7c96d..2e60dc0e 100644
--- a/lib/SPIRV/libSPIRV/SPIRVInstruction.h
+++ b/lib/SPIRV/libSPIRV/SPIRVInstruction.h
@@ -2864,6 +2864,24 @@ _SPIRV_OP(VariableLengthArray, true, 4)
_SPIRV_OP(SaveMemory, true, 3)
_SPIRV_OP(RestoreMemory, false, 2)
#undef _SPIRV_OP
+
+class SPIRVSplitBarrierINTELBase : public SPIRVInstTemplateBase {
+protected:
+ SPIRVCapVec getRequiredCapability() const override {
+ return getVec(CapabilitySplitBarrierINTEL);
+ }
+
+ SPIRVExtSet getRequiredExtensions() const override {
+ return getSet(ExtensionID::SPV_INTEL_split_barrier);
+ }
+};
+
+#define _SPIRV_OP(x, ...) \
+ typedef SPIRVInstTemplate<SPIRVSplitBarrierINTELBase, Op##x, __VA_ARGS__> \
+ SPIRV##x;
+_SPIRV_OP(ControlBarrierArriveINTEL, false, 4)
+_SPIRV_OP(ControlBarrierWaitINTEL, false, 4)
+#undef _SPIRV_OP
} // namespace SPIRV

#endif // SPIRV_LIBSPIRV_SPIRVINSTRUCTION_H
diff --git a/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h b/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h
index 3509bc34..3a391362 100644
--- a/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h
+++ b/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h
@@ -568,6 +568,7 @@ template <> inline void SPIRVMap<Capability, std::string>::init() {
"ArbitraryPrecisionIntegersINTEL");
add(internal::CapabilityMemoryAccessAliasingINTEL,
"MemoryAccessAliasingINTEL");
+ add(CapabilitySplitBarrierINTEL, "SplitBarrierINTEL");

}
SPIRV_DEF_NAMEMAP(Capability, SPIRVCapabilityNameMap)
diff --git a/lib/SPIRV/libSPIRV/SPIRVOpCode.h b/lib/SPIRV/libSPIRV/SPIRVOpCode.h
index c0f2b650..2a4012db 100644
--- a/lib/SPIRV/libSPIRV/SPIRVOpCode.h
+++ b/lib/SPIRV/libSPIRV/SPIRVOpCode.h
@@ -242,6 +242,11 @@ inline bool isEventOpCode(Op OpCode) {
return OpRetainEvent <= OpCode && OpCode <= OpCaptureEventProfilingInfo;
}

+inline bool isSplitBarrierINTELOpCode(Op OpCode) {
+ return OpCode == OpControlBarrierArriveINTEL ||
+ OpCode == OpControlBarrierWaitINTEL;
+}
+
} // namespace SPIRV

#endif // SPIRV_LIBSPIRV_SPIRVOPCODE_H
diff --git a/lib/SPIRV/libSPIRV/SPIRVOpCodeEnum.h b/lib/SPIRV/libSPIRV/SPIRVOpCodeEnum.h
index d2bf96cb..e9897dce 100644
--- a/lib/SPIRV/libSPIRV/SPIRVOpCodeEnum.h
+++ b/lib/SPIRV/libSPIRV/SPIRVOpCodeEnum.h
@@ -486,3 +486,5 @@ _SPIRV_OP(WritePipeBlockingINTEL, 5947)
_SPIRV_OP(FPGARegINTEL, 5949)
_SPIRV_OP(AtomicFAddEXT, 6035)
_SPIRV_OP(TypeBufferSurfaceINTEL, 6086)
+_SPIRV_OP(ControlBarrierArriveINTEL, 6142)
+_SPIRV_OP(ControlBarrierWaitINTEL, 6143)
--
2.17.1