Skip to content

Commit fda9905

Browse files
Artem-BArtem-B
committed
[CUDA] added __nvvm_atom_{sys|cta}_* builtins.
These builtins are available on sm_60+ GPU only. Differential Revision: https://reviews.llvm.org/D24944 llvm-svn: 282609
1 parent 16b2ace commit fda9905

File tree

4 files changed

+457
-8
lines changed

4 files changed

+457
-8
lines changed

‎clang/include/clang/Basic/BuiltinsNVPTX.def

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@
1414

1515
// The format of this database matches clang/Basic/Builtins.def.
1616

17+
#if defined(BUILTIN) && !defined(TARGET_BUILTIN)
18+
# define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BUILTIN(ID, TYPE, ATTRS)
19+
#endif
20+
1721
// Special Registers
1822

1923
BUILTIN(__nvvm_read_ptx_sreg_tid_x, "i", "nc")
@@ -452,18 +456,28 @@ BUILTIN(__builtin_ptx_get_image_channel_orderi_, "ii", "")
452456
BUILTIN(__nvvm_atom_add_g_i, "iiD*1i", "n")
453457
BUILTIN(__nvvm_atom_add_s_i, "iiD*3i", "n")
454458
BUILTIN(__nvvm_atom_add_gen_i, "iiD*i", "n")
459+
TARGET_BUILTIN(__nvvm_atom_cta_add_gen_i, "iiD*i", "n", "satom")
460+
TARGET_BUILTIN(__nvvm_atom_sys_add_gen_i, "iiD*i", "n", "satom")
455461
BUILTIN(__nvvm_atom_add_g_l, "LiLiD*1Li", "n")
456462
BUILTIN(__nvvm_atom_add_s_l, "LiLiD*3Li", "n")
457463
BUILTIN(__nvvm_atom_add_gen_l, "LiLiD*Li", "n")
464+
TARGET_BUILTIN(__nvvm_atom_cta_add_gen_l, "LiLiD*Li", "n", "satom")
465+
TARGET_BUILTIN(__nvvm_atom_sys_add_gen_l, "LiLiD*Li", "n", "satom")
458466
BUILTIN(__nvvm_atom_add_g_ll, "LLiLLiD*1LLi", "n")
459467
BUILTIN(__nvvm_atom_add_s_ll, "LLiLLiD*3LLi", "n")
460468
BUILTIN(__nvvm_atom_add_gen_ll, "LLiLLiD*LLi", "n")
469+
TARGET_BUILTIN(__nvvm_atom_cta_add_gen_ll, "LLiLLiD*LLi", "n", "satom")
470+
TARGET_BUILTIN(__nvvm_atom_sys_add_gen_ll, "LLiLLiD*LLi", "n", "satom")
461471
BUILTIN(__nvvm_atom_add_g_f, "ffD*1f", "n")
462472
BUILTIN(__nvvm_atom_add_s_f, "ffD*3f", "n")
463473
BUILTIN(__nvvm_atom_add_gen_f, "ffD*f", "n")
474+
TARGET_BUILTIN(__nvvm_atom_cta_add_gen_f, "ffD*f", "n", "satom")
475+
TARGET_BUILTIN(__nvvm_atom_sys_add_gen_f, "ffD*f", "n", "satom")
464476
BUILTIN(__nvvm_atom_add_g_d, "ddD*1d", "n")
465477
BUILTIN(__nvvm_atom_add_s_d, "ddD*3d", "n")
466478
BUILTIN(__nvvm_atom_add_gen_d, "ddD*d", "n")
479+
TARGET_BUILTIN(__nvvm_atom_cta_add_gen_d, "ddD*d", "n", "satom")
480+
TARGET_BUILTIN(__nvvm_atom_sys_add_gen_d, "ddD*d", "n", "satom")
467481

468482
BUILTIN(__nvvm_atom_sub_g_i, "iiD*1i", "n")
469483
BUILTIN(__nvvm_atom_sub_s_i, "iiD*3i", "n")
@@ -478,97 +492,155 @@ BUILTIN(__nvvm_atom_sub_gen_ll, "LLiLLiD*LLi", "n")
478492
BUILTIN(__nvvm_atom_xchg_g_i, "iiD*1i", "n")
479493
BUILTIN(__nvvm_atom_xchg_s_i, "iiD*3i", "n")
480494
BUILTIN(__nvvm_atom_xchg_gen_i, "iiD*i", "n")
495+
TARGET_BUILTIN(__nvvm_atom_cta_xchg_gen_i, "iiD*i", "n", "satom")
496+
TARGET_BUILTIN(__nvvm_atom_sys_xchg_gen_i, "iiD*i", "n", "satom")
481497
BUILTIN(__nvvm_atom_xchg_g_l, "LiLiD*1Li", "n")
482498
BUILTIN(__nvvm_atom_xchg_s_l, "LiLiD*3Li", "n")
483499
BUILTIN(__nvvm_atom_xchg_gen_l, "LiLiD*Li", "n")
500+
TARGET_BUILTIN(__nvvm_atom_cta_xchg_gen_l, "LiLiD*Li", "n", "satom")
501+
TARGET_BUILTIN(__nvvm_atom_sys_xchg_gen_l, "LiLiD*Li", "n", "satom")
484502
BUILTIN(__nvvm_atom_xchg_g_ll, "LLiLLiD*1LLi", "n")
485503
BUILTIN(__nvvm_atom_xchg_s_ll, "LLiLLiD*3LLi", "n")
486504
BUILTIN(__nvvm_atom_xchg_gen_ll, "LLiLLiD*LLi", "n")
505+
TARGET_BUILTIN(__nvvm_atom_cta_xchg_gen_ll, "LLiLLiD*LLi", "n", "satom")
506+
TARGET_BUILTIN(__nvvm_atom_sys_xchg_gen_ll, "LLiLLiD*LLi", "n", "satom")
487507

488508
BUILTIN(__nvvm_atom_max_g_i, "iiD*1i", "n")
489509
BUILTIN(__nvvm_atom_max_s_i, "iiD*3i", "n")
490510
BUILTIN(__nvvm_atom_max_gen_i, "iiD*i", "n")
511+
TARGET_BUILTIN(__nvvm_atom_cta_max_gen_i, "iiD*i", "n", "satom")
512+
TARGET_BUILTIN(__nvvm_atom_sys_max_gen_i, "iiD*i", "n", "satom")
491513
BUILTIN(__nvvm_atom_max_g_ui, "UiUiD*1Ui", "n")
492514
BUILTIN(__nvvm_atom_max_s_ui, "UiUiD*3Ui", "n")
493515
BUILTIN(__nvvm_atom_max_gen_ui, "UiUiD*Ui", "n")
516+
TARGET_BUILTIN(__nvvm_atom_cta_max_gen_ui, "UiUiD*Ui", "n", "satom")
517+
TARGET_BUILTIN(__nvvm_atom_sys_max_gen_ui, "UiUiD*Ui", "n", "satom")
494518
BUILTIN(__nvvm_atom_max_g_l, "LiLiD*1Li", "n")
495519
BUILTIN(__nvvm_atom_max_s_l, "LiLiD*3Li", "n")
496520
BUILTIN(__nvvm_atom_max_gen_l, "LiLiD*Li", "n")
521+
TARGET_BUILTIN(__nvvm_atom_cta_max_gen_l, "LiLiD*Li", "n", "satom")
522+
TARGET_BUILTIN(__nvvm_atom_sys_max_gen_l, "LiLiD*Li", "n", "satom")
497523
BUILTIN(__nvvm_atom_max_g_ul, "ULiULiD*1ULi", "n")
498524
BUILTIN(__nvvm_atom_max_s_ul, "ULiULiD*3ULi", "n")
499525
BUILTIN(__nvvm_atom_max_gen_ul, "ULiULiD*ULi", "n")
526+
TARGET_BUILTIN(__nvvm_atom_cta_max_gen_ul, "ULiULiD*ULi", "n", "satom")
527+
TARGET_BUILTIN(__nvvm_atom_sys_max_gen_ul, "ULiULiD*ULi", "n", "satom")
500528
BUILTIN(__nvvm_atom_max_g_ll, "LLiLLiD*1LLi", "n")
501529
BUILTIN(__nvvm_atom_max_s_ll, "LLiLLiD*3LLi", "n")
502530
BUILTIN(__nvvm_atom_max_gen_ll, "LLiLLiD*LLi", "n")
531+
TARGET_BUILTIN(__nvvm_atom_cta_max_gen_ll, "LLiLLiD*LLi", "n", "satom")
532+
TARGET_BUILTIN(__nvvm_atom_sys_max_gen_ll, "LLiLLiD*LLi", "n", "satom")
503533
BUILTIN(__nvvm_atom_max_g_ull, "ULLiULLiD*1ULLi", "n")
504534
BUILTIN(__nvvm_atom_max_s_ull, "ULLiULLiD*3ULLi", "n")
505535
BUILTIN(__nvvm_atom_max_gen_ull, "ULLiULLiD*ULLi", "n")
536+
TARGET_BUILTIN(__nvvm_atom_cta_max_gen_ull, "ULLiULLiD*ULLi", "n", "satom")
537+
TARGET_BUILTIN(__nvvm_atom_sys_max_gen_ull, "ULLiULLiD*ULLi", "n", "satom")
506538

507539
BUILTIN(__nvvm_atom_min_g_i, "iiD*1i", "n")
508540
BUILTIN(__nvvm_atom_min_s_i, "iiD*3i", "n")
509541
BUILTIN(__nvvm_atom_min_gen_i, "iiD*i", "n")
542+
TARGET_BUILTIN(__nvvm_atom_cta_min_gen_i, "iiD*i", "n", "satom")
543+
TARGET_BUILTIN(__nvvm_atom_sys_min_gen_i, "iiD*i", "n", "satom")
510544
BUILTIN(__nvvm_atom_min_g_ui, "UiUiD*1Ui", "n")
511545
BUILTIN(__nvvm_atom_min_s_ui, "UiUiD*3Ui", "n")
512546
BUILTIN(__nvvm_atom_min_gen_ui, "UiUiD*Ui", "n")
547+
TARGET_BUILTIN(__nvvm_atom_cta_min_gen_ui, "UiUiD*Ui", "n", "satom")
548+
TARGET_BUILTIN(__nvvm_atom_sys_min_gen_ui, "UiUiD*Ui", "n", "satom")
513549
BUILTIN(__nvvm_atom_min_g_l, "LiLiD*1Li", "n")
514550
BUILTIN(__nvvm_atom_min_s_l, "LiLiD*3Li", "n")
515551
BUILTIN(__nvvm_atom_min_gen_l, "LiLiD*Li", "n")
552+
TARGET_BUILTIN(__nvvm_atom_cta_min_gen_l, "LiLiD*Li", "n", "satom")
553+
TARGET_BUILTIN(__nvvm_atom_sys_min_gen_l, "LiLiD*Li", "n", "satom")
516554
BUILTIN(__nvvm_atom_min_g_ul, "ULiULiD*1ULi", "n")
517555
BUILTIN(__nvvm_atom_min_s_ul, "ULiULiD*3ULi", "n")
518556
BUILTIN(__nvvm_atom_min_gen_ul, "ULiULiD*ULi", "n")
557+
TARGET_BUILTIN(__nvvm_atom_cta_min_gen_ul, "ULiULiD*ULi", "n", "satom")
558+
TARGET_BUILTIN(__nvvm_atom_sys_min_gen_ul, "ULiULiD*ULi", "n", "satom")
519559
BUILTIN(__nvvm_atom_min_g_ll, "LLiLLiD*1LLi", "n")
520560
BUILTIN(__nvvm_atom_min_s_ll, "LLiLLiD*3LLi", "n")
521561
BUILTIN(__nvvm_atom_min_gen_ll, "LLiLLiD*LLi", "n")
562+
TARGET_BUILTIN(__nvvm_atom_cta_min_gen_ll, "LLiLLiD*LLi", "n", "satom")
563+
TARGET_BUILTIN(__nvvm_atom_sys_min_gen_ll, "LLiLLiD*LLi", "n", "satom")
522564
BUILTIN(__nvvm_atom_min_g_ull, "ULLiULLiD*1ULLi", "n")
523565
BUILTIN(__nvvm_atom_min_s_ull, "ULLiULLiD*3ULLi", "n")
524566
BUILTIN(__nvvm_atom_min_gen_ull, "ULLiULLiD*ULLi", "n")
567+
TARGET_BUILTIN(__nvvm_atom_cta_min_gen_ull, "ULLiULLiD*ULLi", "n", "satom")
568+
TARGET_BUILTIN(__nvvm_atom_sys_min_gen_ull, "ULLiULLiD*ULLi", "n", "satom")
525569

526570
BUILTIN(__nvvm_atom_inc_g_ui, "UiUiD*1Ui", "n")
527571
BUILTIN(__nvvm_atom_inc_s_ui, "UiUiD*3Ui", "n")
528572
BUILTIN(__nvvm_atom_inc_gen_ui, "UiUiD*Ui", "n")
573+
TARGET_BUILTIN(__nvvm_atom_cta_inc_gen_ui, "UiUiD*Ui", "n", "satom")
574+
TARGET_BUILTIN(__nvvm_atom_sys_inc_gen_ui, "UiUiD*Ui", "n", "satom")
529575
BUILTIN(__nvvm_atom_dec_g_ui, "UiUiD*1Ui", "n")
530576
BUILTIN(__nvvm_atom_dec_s_ui, "UiUiD*3Ui", "n")
531577
BUILTIN(__nvvm_atom_dec_gen_ui, "UiUiD*Ui", "n")
578+
TARGET_BUILTIN(__nvvm_atom_cta_dec_gen_ui, "UiUiD*Ui", "n", "satom")
579+
TARGET_BUILTIN(__nvvm_atom_sys_dec_gen_ui, "UiUiD*Ui", "n", "satom")
532580

533581
BUILTIN(__nvvm_atom_and_g_i, "iiD*1i", "n")
534582
BUILTIN(__nvvm_atom_and_s_i, "iiD*3i", "n")
535583
BUILTIN(__nvvm_atom_and_gen_i, "iiD*i", "n")
584+
TARGET_BUILTIN(__nvvm_atom_cta_and_gen_i, "iiD*i", "n", "satom")
585+
TARGET_BUILTIN(__nvvm_atom_sys_and_gen_i, "iiD*i", "n", "satom")
536586
BUILTIN(__nvvm_atom_and_g_l, "LiLiD*1Li", "n")
537587
BUILTIN(__nvvm_atom_and_s_l, "LiLiD*3Li", "n")
538588
BUILTIN(__nvvm_atom_and_gen_l, "LiLiD*Li", "n")
589+
TARGET_BUILTIN(__nvvm_atom_cta_and_gen_l, "LiLiD*Li", "n", "satom")
590+
TARGET_BUILTIN(__nvvm_atom_sys_and_gen_l, "LiLiD*Li", "n", "satom")
539591
BUILTIN(__nvvm_atom_and_g_ll, "LLiLLiD*1LLi", "n")
540592
BUILTIN(__nvvm_atom_and_s_ll, "LLiLLiD*3LLi", "n")
541593
BUILTIN(__nvvm_atom_and_gen_ll, "LLiLLiD*LLi", "n")
594+
TARGET_BUILTIN(__nvvm_atom_cta_and_gen_ll, "LLiLLiD*LLi", "n", "satom")
595+
TARGET_BUILTIN(__nvvm_atom_sys_and_gen_ll, "LLiLLiD*LLi", "n", "satom")
542596

543597
BUILTIN(__nvvm_atom_or_g_i, "iiD*1i", "n")
544598
BUILTIN(__nvvm_atom_or_s_i, "iiD*3i", "n")
545599
BUILTIN(__nvvm_atom_or_gen_i, "iiD*i", "n")
600+
TARGET_BUILTIN(__nvvm_atom_cta_or_gen_i, "iiD*i", "n", "satom")
601+
TARGET_BUILTIN(__nvvm_atom_sys_or_gen_i, "iiD*i", "n", "satom")
546602
BUILTIN(__nvvm_atom_or_g_l, "LiLiD*1Li", "n")
547603
BUILTIN(__nvvm_atom_or_s_l, "LiLiD*3Li", "n")
548604
BUILTIN(__nvvm_atom_or_gen_l, "LiLiD*Li", "n")
605+
TARGET_BUILTIN(__nvvm_atom_cta_or_gen_l, "LiLiD*Li", "n", "satom")
606+
TARGET_BUILTIN(__nvvm_atom_sys_or_gen_l, "LiLiD*Li", "n", "satom")
549607
BUILTIN(__nvvm_atom_or_g_ll, "LLiLLiD*1LLi", "n")
550608
BUILTIN(__nvvm_atom_or_s_ll, "LLiLLiD*3LLi", "n")
551609
BUILTIN(__nvvm_atom_or_gen_ll, "LLiLLiD*LLi", "n")
610+
TARGET_BUILTIN(__nvvm_atom_cta_or_gen_ll, "LLiLLiD*LLi", "n", "satom")
611+
TARGET_BUILTIN(__nvvm_atom_sys_or_gen_ll, "LLiLLiD*LLi", "n", "satom")
552612

553613
BUILTIN(__nvvm_atom_xor_g_i, "iiD*1i", "n")
554614
BUILTIN(__nvvm_atom_xor_s_i, "iiD*3i", "n")
555615
BUILTIN(__nvvm_atom_xor_gen_i, "iiD*i", "n")
616+
TARGET_BUILTIN(__nvvm_atom_cta_xor_gen_i, "iiD*i", "n", "satom")
617+
TARGET_BUILTIN(__nvvm_atom_sys_xor_gen_i, "iiD*i", "n", "satom")
556618
BUILTIN(__nvvm_atom_xor_g_l, "LiLiD*1Li", "n")
557619
BUILTIN(__nvvm_atom_xor_s_l, "LiLiD*3Li", "n")
558620
BUILTIN(__nvvm_atom_xor_gen_l, "LiLiD*Li", "n")
621+
TARGET_BUILTIN(__nvvm_atom_cta_xor_gen_l, "LiLiD*Li", "n", "satom")
622+
TARGET_BUILTIN(__nvvm_atom_sys_xor_gen_l, "LiLiD*Li", "n", "satom")
559623
BUILTIN(__nvvm_atom_xor_g_ll, "LLiLLiD*1LLi", "n")
560624
BUILTIN(__nvvm_atom_xor_s_ll, "LLiLLiD*3LLi", "n")
561625
BUILTIN(__nvvm_atom_xor_gen_ll, "LLiLLiD*LLi", "n")
626+
TARGET_BUILTIN(__nvvm_atom_cta_xor_gen_ll, "LLiLLiD*LLi", "n", "satom")
627+
TARGET_BUILTIN(__nvvm_atom_sys_xor_gen_ll, "LLiLLiD*LLi", "n", "satom")
562628

563629
BUILTIN(__nvvm_atom_cas_g_i, "iiD*1ii", "n")
564630
BUILTIN(__nvvm_atom_cas_s_i, "iiD*3ii", "n")
565631
BUILTIN(__nvvm_atom_cas_gen_i, "iiD*ii", "n")
632+
TARGET_BUILTIN(__nvvm_atom_cta_cas_gen_i, "iiD*ii", "n", "satom")
633+
TARGET_BUILTIN(__nvvm_atom_sys_cas_gen_i, "iiD*ii", "n", "satom")
566634
BUILTIN(__nvvm_atom_cas_g_l, "LiLiD*1LiLi", "n")
567635
BUILTIN(__nvvm_atom_cas_s_l, "LiLiD*3LiLi", "n")
568636
BUILTIN(__nvvm_atom_cas_gen_l, "LiLiD*LiLi", "n")
637+
TARGET_BUILTIN(__nvvm_atom_cta_cas_gen_l, "LiLiD*LiLi", "n", "satom")
638+
TARGET_BUILTIN(__nvvm_atom_sys_cas_gen_l, "LiLiD*LiLi", "n", "satom")
569639
BUILTIN(__nvvm_atom_cas_g_ll, "LLiLLiD*1LLiLLi", "n")
570640
BUILTIN(__nvvm_atom_cas_s_ll, "LLiLLiD*3LLiLLi", "n")
571641
BUILTIN(__nvvm_atom_cas_gen_ll, "LLiLLiD*LLiLLi", "n")
642+
TARGET_BUILTIN(__nvvm_atom_cta_cas_gen_ll, "LLiLLiD*LLiLLi", "n", "satom")
643+
TARGET_BUILTIN(__nvvm_atom_sys_cas_gen_ll, "LLiLLiD*LLiLLi", "n", "satom")
572644

573645
// Compiler Error Warn
574646
BUILTIN(__nvvm_compiler_error, "vcC*4", "n")
@@ -611,3 +683,4 @@ BUILTIN(__nvvm_ldg_f4, "E4fE4fC*", "")
611683
BUILTIN(__nvvm_ldg_d2, "E2dE2dC*", "")
612684

613685
#undef BUILTIN
686+
#undef TARGET_BUILTIN

‎clang/lib/Basic/Targets.cpp

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1850,8 +1850,19 @@ class NVPTXTargetInfo : public TargetInfo {
18501850
return llvm::makeArrayRef(BuiltinInfo,
18511851
clang::NVPTX::LastTSBuiltin - Builtin::FirstTSBuiltin);
18521852
}
1853+
bool
1854+
initFeatureMap(llvm::StringMap<bool> &Features, DiagnosticsEngine &Diags,
1855+
StringRef CPU,
1856+
const std::vector<std::string> &FeaturesVec) const override {
1857+
Features["satom"] = GPU >= CudaArch::SM_60;
1858+
return TargetInfo::initFeatureMap(Features, Diags, CPU, FeaturesVec);
1859+
}
1860+
18531861
bool hasFeature(StringRef Feature) const override {
1854-
return Feature == "ptx" || Feature == "nvptx";
1862+
return llvm::StringSwitch<bool>(Feature)
1863+
.Cases("ptx", "nvptx", true)
1864+
.Case("satom", GPU >= CudaArch::SM_60) // Atomics w/ scope.
1865+
.Default(false);
18551866
}
18561867

18571868
ArrayRef<const char *> getGCCRegNames() const override;
@@ -1906,6 +1917,8 @@ const Builtin::Info NVPTXTargetInfo::BuiltinInfo[] = {
19061917
{ #ID, TYPE, ATTRS, nullptr, ALL_LANGUAGES, nullptr },
19071918
#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER) \
19081919
{ #ID, TYPE, ATTRS, HEADER, ALL_LANGUAGES, nullptr },
1920+
#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) \
1921+
{ #ID, TYPE, ATTRS, nullptr, ALL_LANGUAGES, FEATURE },
19091922
#include "clang/Basic/BuiltinsNVPTX.def"
19101923
};
19111924

‎clang/lib/CodeGen/CGBuiltin.cpp

Lines changed: 110 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8124,7 +8124,13 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
81248124
Ptr->getType()}),
81258125
{Ptr, ConstantInt::get(Builder.getInt32Ty(), Align.getQuantity())});
81268126
};
8127-
8127+
auto MakeScopedAtomic = [&](unsigned IntrinsicID) {
8128+
Value *Ptr = EmitScalarExpr(E->getArg(0));
8129+
return Builder.CreateCall(
8130+
CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(),
8131+
Ptr->getType()}),
8132+
{Ptr, EmitScalarExpr(E->getArg(1))});
8133+
};
81288134
switch (BuiltinID) {
81298135
case NVPTX::BI__nvvm_atom_add_gen_i:
81308136
case NVPTX::BI__nvvm_atom_add_gen_l:
@@ -8243,6 +8249,109 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
82438249
case NVPTX::BI__nvvm_ldg_d:
82448250
case NVPTX::BI__nvvm_ldg_d2:
82458251
return MakeLdg(Intrinsic::nvvm_ldg_global_f);
8252+
8253+
case NVPTX::BI__nvvm_atom_cta_add_gen_i:
8254+
case NVPTX::BI__nvvm_atom_cta_add_gen_l:
8255+
case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
8256+
return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_cta);
8257+
case NVPTX::BI__nvvm_atom_sys_add_gen_i:
8258+
case NVPTX::BI__nvvm_atom_sys_add_gen_l:
8259+
case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
8260+
return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_sys);
8261+
case NVPTX::BI__nvvm_atom_cta_add_gen_f:
8262+
case NVPTX::BI__nvvm_atom_cta_add_gen_d:
8263+
return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_cta);
8264+
case NVPTX::BI__nvvm_atom_sys_add_gen_f:
8265+
case NVPTX::BI__nvvm_atom_sys_add_gen_d:
8266+
return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_sys);
8267+
case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
8268+
case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
8269+
case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
8270+
return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_cta);
8271+
case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
8272+
case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
8273+
case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
8274+
return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_sys);
8275+
case NVPTX::BI__nvvm_atom_cta_max_gen_i:
8276+
case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
8277+
case NVPTX::BI__nvvm_atom_cta_max_gen_l:
8278+
case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
8279+
case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
8280+
case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
8281+
return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_cta);
8282+
case NVPTX::BI__nvvm_atom_sys_max_gen_i:
8283+
case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
8284+
case NVPTX::BI__nvvm_atom_sys_max_gen_l:
8285+
case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
8286+
case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
8287+
case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
8288+
return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_sys);
8289+
case NVPTX::BI__nvvm_atom_cta_min_gen_i:
8290+
case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
8291+
case NVPTX::BI__nvvm_atom_cta_min_gen_l:
8292+
case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
8293+
case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
8294+
case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
8295+
return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_cta);
8296+
case NVPTX::BI__nvvm_atom_sys_min_gen_i:
8297+
case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
8298+
case NVPTX::BI__nvvm_atom_sys_min_gen_l:
8299+
case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
8300+
case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
8301+
case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
8302+
return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_sys);
8303+
case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
8304+
return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_cta);
8305+
case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
8306+
return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_cta);
8307+
case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
8308+
return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_sys);
8309+
case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
8310+
return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_sys);
8311+
case NVPTX::BI__nvvm_atom_cta_and_gen_i:
8312+
case NVPTX::BI__nvvm_atom_cta_and_gen_l:
8313+
case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
8314+
return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_cta);
8315+
case NVPTX::BI__nvvm_atom_sys_and_gen_i:
8316+
case NVPTX::BI__nvvm_atom_sys_and_gen_l:
8317+
case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
8318+
return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_sys);
8319+
case NVPTX::BI__nvvm_atom_cta_or_gen_i:
8320+
case NVPTX::BI__nvvm_atom_cta_or_gen_l:
8321+
case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
8322+
return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_cta);
8323+
case NVPTX::BI__nvvm_atom_sys_or_gen_i:
8324+
case NVPTX::BI__nvvm_atom_sys_or_gen_l:
8325+
case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
8326+
return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_sys);
8327+
case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
8328+
case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
8329+
case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
8330+
return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_cta);
8331+
case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
8332+
case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
8333+
case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
8334+
return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_sys);
8335+
case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
8336+
case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
8337+
case NVPTX::BI__nvvm_atom_cta_cas_gen_ll: {
8338+
Value *Ptr = EmitScalarExpr(E->getArg(0));
8339+
return Builder.CreateCall(
8340+
CGM.getIntrinsic(
8341+
Intrinsic::nvvm_atomic_cas_gen_i_cta,
8342+
{Ptr->getType()->getPointerElementType(), Ptr->getType()}),
8343+
{Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
8344+
}
8345+
case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
8346+
case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
8347+
case NVPTX::BI__nvvm_atom_sys_cas_gen_ll: {
8348+
Value *Ptr = EmitScalarExpr(E->getArg(0));
8349+
return Builder.CreateCall(
8350+
CGM.getIntrinsic(
8351+
Intrinsic::nvvm_atomic_cas_gen_i_sys,
8352+
{Ptr->getType()->getPointerElementType(), Ptr->getType()}),
8353+
{Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
8354+
}
82468355
default:
82478356
return nullptr;
82488357
}

0 commit comments

Comments
 (0)