Skip to content

Commit db2baab

Browse files
t4c1t4c1
committed
[SYCL][CUDA][libclc] Added atomic loads and stores with various memory orders and scopes
1 parent 57255ce commit db2baab

File tree

9 files changed

+1729
-2
lines changed

9 files changed

+1729
-2
lines changed

‎clang/include/clang/Basic/BuiltinsNVPTX.def

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2037,6 +2037,76 @@ TARGET_BUILTIN(__nvvm_atom_acq_rel_cas_shared_ll, "LLiLLiD*LLiLLi", "n", SM_70)
20372037
TARGET_BUILTIN(__nvvm_atom_acq_rel_cta_cas_shared_ll, "LLiLLiD*LLiLLi", "n", SM_70)
20382038
TARGET_BUILTIN(__nvvm_atom_acq_rel_sys_cas_shared_ll, "LLiLLiD*LLiLLi", "n", SM_70)
20392039

2040+
#define LD_VOLATILE_BUILTIN_TYPES(ADDR_SPACE) \
2041+
BUILTIN(__nvvm_volatile_ld##ADDR_SPACE##_i, "iiD*", "n") \
2042+
BUILTIN(__nvvm_volatile_ld##ADDR_SPACE##_l, "LiLiD*", "n") \
2043+
BUILTIN(__nvvm_volatile_ld##ADDR_SPACE##_ll, "LLiLLiD*", "n") \
2044+
BUILTIN(__nvvm_volatile_ld##ADDR_SPACE##_f, "ffD*", "n") \
2045+
BUILTIN(__nvvm_volatile_ld##ADDR_SPACE##_d, "ddD*", "n")
2046+
2047+
#define LD_BUILTIN_TYPES(ORDER, SCOPE, ADDR_SPACE) \
2048+
TARGET_BUILTIN(__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_i, "iiD*", "n", SM_70) \
2049+
TARGET_BUILTIN(__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_l, "LiLiD*", "n", SM_70) \
2050+
TARGET_BUILTIN(__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_ll, "LLiLLiD*", "n", SM_70) \
2051+
TARGET_BUILTIN(__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_f, "ffD*", "n", SM_70) \
2052+
TARGET_BUILTIN(__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_d, "ddD*", "n", SM_70)
2053+
2054+
#define LD_BUILTIN_AS_TYPES(ORDER, SCOPE) \
2055+
LD_BUILTIN_TYPES(ORDER, SCOPE, _gen) \
2056+
LD_BUILTIN_TYPES(ORDER, SCOPE, _global) \
2057+
LD_BUILTIN_TYPES(ORDER, SCOPE, _shared)
2058+
2059+
#define LD_BUILTIN_SCOPES_AS_TYPES(ORDER) \
2060+
LD_BUILTIN_AS_TYPES(ORDER, ) \
2061+
LD_BUILTIN_AS_TYPES(ORDER, _cta) \
2062+
LD_BUILTIN_AS_TYPES(ORDER, _sys)
2063+
2064+
LD_BUILTIN_SCOPES_AS_TYPES()
2065+
LD_BUILTIN_SCOPES_AS_TYPES(_acquire)
2066+
LD_VOLATILE_BUILTIN_TYPES(_gen)
2067+
LD_VOLATILE_BUILTIN_TYPES(_global)
2068+
LD_VOLATILE_BUILTIN_TYPES(_shared)
2069+
2070+
#undef LD_VOLATILE_BUILTIN_TYPES
2071+
#undef LD_BUILTIN_TYPES
2072+
#undef LD_BUILTIN_AS_TYPES
2073+
#undef LD_BUILTIN_SCOPES_AS_TYPES
2074+
2075+
#define ST_VOLATILE_BUILTIN_TYPES(ADDR_SPACE) \
2076+
BUILTIN(__nvvm_volatile_st##ADDR_SPACE##_i, "viD*i", "n") \
2077+
BUILTIN(__nvvm_volatile_st##ADDR_SPACE##_l, "vLiD*Li", "n") \
2078+
BUILTIN(__nvvm_volatile_st##ADDR_SPACE##_ll, "vLLiD*LLi", "n") \
2079+
BUILTIN(__nvvm_volatile_st##ADDR_SPACE##_f, "vfD*f", "n") \
2080+
BUILTIN(__nvvm_volatile_st##ADDR_SPACE##_d, "vdD*d", "n")
2081+
2082+
#define ST_BUILTIN_TYPES(ORDER, SCOPE, ADDR_SPACE) \
2083+
TARGET_BUILTIN(__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_i, "viD*i", "n", SM_70) \
2084+
TARGET_BUILTIN(__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_l, "vLiD*Li", "n", SM_70) \
2085+
TARGET_BUILTIN(__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_ll, "vLLiD*LLi", "n", SM_70) \
2086+
TARGET_BUILTIN(__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_f, "vfD*f", "n", SM_70) \
2087+
TARGET_BUILTIN(__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_d, "vdD*d", "n", SM_70)
2088+
2089+
#define ST_BUILTIN_AS_TYPES(ORDER, SCOPE) \
2090+
ST_BUILTIN_TYPES(ORDER, SCOPE, _gen) \
2091+
ST_BUILTIN_TYPES(ORDER, SCOPE, _global) \
2092+
ST_BUILTIN_TYPES(ORDER, SCOPE, _shared)
2093+
2094+
#define ST_BUILTIN_SCOPES_AS_TYPES(ORDER) \
2095+
ST_BUILTIN_AS_TYPES(ORDER, ) \
2096+
ST_BUILTIN_AS_TYPES(ORDER, _cta) \
2097+
ST_BUILTIN_AS_TYPES(ORDER, _sys)
2098+
2099+
ST_BUILTIN_SCOPES_AS_TYPES()
2100+
ST_BUILTIN_SCOPES_AS_TYPES(_release)
2101+
ST_VOLATILE_BUILTIN_TYPES(_gen)
2102+
ST_VOLATILE_BUILTIN_TYPES(_global)
2103+
ST_VOLATILE_BUILTIN_TYPES(_shared)
2104+
2105+
#undef ST_VOLATILE_BUILTIN_TYPES
2106+
#undef ST_BUILTIN_TYPES
2107+
#undef ST_BUILTIN_AS_TYPES
2108+
#undef ST_BUILTIN_SCOPES_AS_TYPES
2109+
20402110
// Compiler Error Warn
20412111
BUILTIN(__nvvm_compiler_error, "vcC*4", "n")
20422112
BUILTIN(__nvvm_compiler_warn, "vcC*4", "n")

‎clang/lib/CodeGen/CGBuiltin.cpp

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@
5454
#include "llvm/Support/X86TargetParser.h"
5555
#include <sstream>
5656

57+
#include <iostream>
58+
5759
using namespace clang;
5860
using namespace CodeGen;
5961
using namespace llvm;
@@ -17423,6 +17425,22 @@ CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E) {
1742317425
Ptr->getType()}),
1742417426
{Ptr, ConstantInt::get(Builder.getInt32Ty(), Align.getQuantity())});
1742517427
};
17428+
auto MakeScopedLd = [&](unsigned IntrinsicID) {
17429+
Value *Ptr = EmitScalarExpr(E->getArg(0));
17430+
auto tmp = Builder.CreateCall(
17431+
CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(),
17432+
Ptr->getType()}),
17433+
{Ptr});
17434+
return tmp;
17435+
};
17436+
auto MakeScopedSt = [&](unsigned IntrinsicID) {
17437+
Value *Ptr = EmitScalarExpr(E->getArg(0));
17438+
return Builder.CreateCall(
17439+
CGM.getIntrinsic(
17440+
IntrinsicID,
17441+
{Ptr->getType(), Ptr->getType()->getPointerElementType()}),
17442+
{Ptr, EmitScalarExpr(E->getArg(1))});
17443+
};
1742617444
auto MakeScopedAtomic = [&](unsigned IntrinsicID) {
1742717445
Value *Ptr = EmitScalarExpr(E->getArg(0));
1742817446
return Builder.CreateCall(
@@ -17438,6 +17456,85 @@ CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E) {
1743817456
{Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
1743917457
};
1744017458
switch (BuiltinID) {
17459+
17460+
#define LD_VOLATILE_CASES(ADDR_SPACE) \
17461+
case NVPTX::BI__nvvm_volatile_ld##ADDR_SPACE##_i: \
17462+
case NVPTX::BI__nvvm_volatile_ld##ADDR_SPACE##_l: \
17463+
case NVPTX::BI__nvvm_volatile_ld##ADDR_SPACE##_ll: \
17464+
return MakeScopedLd(Intrinsic::nvvm_ld##ADDR_SPACE##_i_volatile); \
17465+
case NVPTX::BI__nvvm_volatile_ld##ADDR_SPACE##_f: \
17466+
case NVPTX::BI__nvvm_volatile_ld##ADDR_SPACE##_d: \
17467+
return MakeScopedLd(Intrinsic::nvvm_ld##ADDR_SPACE##_f_volatile);
17468+
17469+
#define LD_CASES(ORDER, SCOPE, ADDR_SPACE) \
17470+
case NVPTX::BI__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_i: \
17471+
case NVPTX::BI__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_l: \
17472+
case NVPTX::BI__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_ll: \
17473+
return MakeScopedLd(Intrinsic::nvvm_ld##ADDR_SPACE##_i##ORDER##SCOPE); \
17474+
case NVPTX::BI__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_f: \
17475+
case NVPTX::BI__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_d: \
17476+
return MakeScopedLd(Intrinsic::nvvm_ld##ADDR_SPACE##_f##ORDER##SCOPE);
17477+
17478+
#define LD_CASES_AS(ORDER, SCOPE) \
17479+
LD_CASES(ORDER, SCOPE, _gen) \
17480+
LD_CASES(ORDER, SCOPE, _global) \
17481+
LD_CASES(ORDER, SCOPE, _shared)
17482+
17483+
#define LD_CASES_AS_SCOPES(ORDER) \
17484+
LD_CASES_AS(ORDER, ) \
17485+
LD_CASES_AS(ORDER, _cta) \
17486+
LD_CASES_AS(ORDER, _sys)
17487+
17488+
LD_CASES_AS_SCOPES()
17489+
LD_CASES_AS_SCOPES(_acquire)
17490+
LD_VOLATILE_CASES(_gen)
17491+
LD_VOLATILE_CASES(_global)
17492+
LD_VOLATILE_CASES(_shared)
17493+
17494+
#undef LD_VOLATILE_CASES
17495+
#undef LD_CASES
17496+
#undef LD_CASES_AS
17497+
#undef LD_CASES_AS_SCOPES
17498+
17499+
#define ST_VOLATILE_CASES(ADDR_SPACE) \
17500+
case NVPTX::BI__nvvm_volatile_st##ADDR_SPACE##_i: \
17501+
case NVPTX::BI__nvvm_volatile_st##ADDR_SPACE##_l: \
17502+
case NVPTX::BI__nvvm_volatile_st##ADDR_SPACE##_ll: \
17503+
return MakeScopedSt(Intrinsic::nvvm_st##ADDR_SPACE##_i_volatile); \
17504+
case NVPTX::BI__nvvm_volatile_st##ADDR_SPACE##_f: \
17505+
case NVPTX::BI__nvvm_volatile_st##ADDR_SPACE##_d: \
17506+
return MakeScopedSt(Intrinsic::nvvm_st##ADDR_SPACE##_f_volatile);
17507+
17508+
#define ST_CASES(ORDER, SCOPE, ADDR_SPACE) \
17509+
case NVPTX::BI__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_i: \
17510+
case NVPTX::BI__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_l: \
17511+
case NVPTX::BI__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_ll: \
17512+
return MakeScopedSt(Intrinsic::nvvm_st##ADDR_SPACE##_i##ORDER##SCOPE); \
17513+
case NVPTX::BI__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_f: \
17514+
case NVPTX::BI__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_d: \
17515+
return MakeScopedSt(Intrinsic::nvvm_st##ADDR_SPACE##_f##ORDER##SCOPE);
17516+
17517+
#define ST_CASES_AS(ORDER, SCOPE) \
17518+
ST_CASES(ORDER, SCOPE, _gen) \
17519+
ST_CASES(ORDER, SCOPE, _global) \
17520+
ST_CASES(ORDER, SCOPE, _shared)
17521+
17522+
#define ST_CASES_AS_SCOPES(ORDER) \
17523+
ST_CASES_AS(ORDER, ) \
17524+
ST_CASES_AS(ORDER, _cta) \
17525+
ST_CASES_AS(ORDER, _sys)
17526+
17527+
ST_CASES_AS_SCOPES()
17528+
ST_CASES_AS_SCOPES(_release)
17529+
ST_VOLATILE_CASES(_gen)
17530+
ST_VOLATILE_CASES(_global)
17531+
ST_VOLATILE_CASES(_shared)
17532+
17533+
#undef ST_VOLATILE_CASES
17534+
#undef ST_CASES
17535+
#undef ST_CASES_AS
17536+
#undef ST_CASES_AS_SCOPES
17537+
1744117538
case NVPTX::BI__nvvm_atom_add_gen_i:
1744217539
case NVPTX::BI__nvvm_atom_add_gen_l:
1744317540
case NVPTX::BI__nvvm_atom_add_gen_ll:

0 commit comments

Comments
 (0)