merge main into amd-staging (llvm#2493)

Author: ronlieb

clang/include/clang/Basic/AddressSpaces.h

@@ -61,6 +61,7 @@ enum class LangAS : unsigned {
   hlsl_constant,
   hlsl_private,
   hlsl_device,
+  hlsl_input,
 
   // Wasm specific address spaces.
   wasm_funcref,

clang/include/clang/Basic/Attr.td

@@ -140,6 +140,12 @@ def SharedVar : SubsetSubject<Var,
                               [{S->hasGlobalStorage() && !S->getTLSKind()}],
                               "global variables">;
 
+def HLSLInputBuiltin
+    : SubsetSubject<Var, [{S->hasGlobalStorage() &&
+                           S->getStorageClass() == StorageClass::SC_Static &&
+                           S->getType().isConstQualified()}],
+                    "static const globals">;
+
 def GlobalVar : SubsetSubject<Var,
                              [{S->hasGlobalStorage()}], "global variables">;
 
@@ -4951,6 +4957,14 @@ def HLSLWaveSize: InheritableAttr {
   let Documentation = [WaveSizeDocs];
 }
 
+def HLSLVkExtBuiltinInput : InheritableAttr {
+  let Spellings = [CXX11<"vk", "ext_builtin_input">];
+  let Args = [UnsignedArgument<"BuiltIn">];
+  let Subjects = SubjectList<[HLSLInputBuiltin], ErrorDiag>;
+  let LangOpts = [HLSL];
+  let Documentation = [HLSLVkExtBuiltinInputDocs];
+}
+
 def RandomizeLayout : InheritableAttr {
   let Spellings = [GCC<"randomize_layout">];
   let Subjects = SubjectList<[Record]>;

clang/include/clang/Basic/AttrDocs.td

@@ -8508,6 +8508,29 @@ and copied back to the argument after the callee returns.
   }];
 }
 
+def HLSLVkExtBuiltinInputDocs : Documentation {
+  let Category = DocCatVariable;
+  let Content = [{
+Vulkan shaders have `Input` builtins. Those variables are externally
+initialized by the driver/pipeline, but each copy is private to the current
+lane.
+
+Those builtins can be declared using the `[[vk::ext_builtin_input]]` attribute
+like follows:
+
+.. code-block:: c++
+
+  [[vk::ext_builtin_input(/* WorkgroupId */ 26)]]
+  static const uint3 groupid;
+
+This variable will be lowered into a module-level variable, with the `Input`
+storage class, and the `BuiltIn 26` decoration.
+
+The full documentation for this inline SPIR-V attribute can be found here:
+https://github.com/microsoft/hlsl-specs/blob/main/proposals/0011-inline-spirv.md
+  }];
+}
+
 def AnnotateTypeDocs : Documentation {
   let Category = DocCatType;
   let Heading = "annotate_type";

clang/include/clang/Basic/AttributeCommonInfo.h

@@ -69,7 +69,7 @@ class AttributeCommonInfo {
     IgnoredAttribute,
     UnknownAttribute,
   };
-  enum class Scope { NONE, CLANG, GNU, MSVC, OMP, HLSL, GSL, RISCV };
+  enum class Scope { NONE, CLANG, GNU, MSVC, OMP, HLSL, VK, GSL, RISCV };
   enum class AttrArgsInfo {
     None,
     Optional,

clang/include/clang/Basic/BuiltinHeaders.def

@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 HEADER(NO_HEADER, nullptr)
+HEADER(ARMACLE_H, "arm_acle.h")
 HEADER(BLOCKS_H, "Blocks.h")
 HEADER(COMPLEX_H, "complex.h")
 HEADER(CTYPE_H, "ctype.h")

clang/include/clang/Basic/BuiltinsAArch64.def

@@ -50,6 +50,11 @@ BUILTIN(__builtin_arm_wfi, "v", "")
 BUILTIN(__builtin_arm_sev, "v", "")
 BUILTIN(__builtin_arm_sevl, "v", "")
 BUILTIN(__builtin_arm_chkfeat, "WUiWUi", "")
+TARGET_HEADER_BUILTIN(__yield, "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__wfe,   "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__wfi,   "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__sev,   "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__sevl,  "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
 
 // Like __builtin_trap but provide an 16-bit immediate reason code (which goes into `brk #N`).
 BUILTIN(__builtin_arm_trap, "vUIs", "nr")
@@ -82,6 +87,9 @@ TARGET_BUILTIN(__builtin_arm_mops_memset_tag, "v*v*iz", "", "mte,mops")
 BUILTIN(__builtin_arm_dmb, "vUi", "nc")
 BUILTIN(__builtin_arm_dsb, "vUi", "nc")
 BUILTIN(__builtin_arm_isb, "vUi", "nc")
+TARGET_HEADER_BUILTIN(__dmb, "vUi", "nch", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__dsb, "vUi", "nch", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__isb, "vUi", "nch", ARMACLE_H, ALL_LANGUAGES, "")
 
 TARGET_BUILTIN(__builtin_arm_jcvt, "Zid", "nc", "v8.3a")
 
@@ -98,16 +106,6 @@ BUILTIN(__builtin_arm_wsr64, "vcC*WUi", "nc")
 TARGET_BUILTIN(__builtin_arm_wsr128, "vcC*LLLUi", "nc", "d128")
 BUILTIN(__builtin_arm_wsrp, "vcC*vC*", "nc")
 
-// MSVC
-LANGBUILTIN(__dmb, "vUi", "nc", ALL_MS_LANGUAGES)
-LANGBUILTIN(__dsb, "vUi", "nc", ALL_MS_LANGUAGES)
-LANGBUILTIN(__isb, "vUi", "nc", ALL_MS_LANGUAGES)
-LANGBUILTIN(__yield, "v", "",   ALL_MS_LANGUAGES)
-LANGBUILTIN(__wfe,   "v", "",   ALL_MS_LANGUAGES)
-LANGBUILTIN(__wfi,   "v", "",   ALL_MS_LANGUAGES)
-LANGBUILTIN(__sev,   "v", "",   ALL_MS_LANGUAGES)
-LANGBUILTIN(__sevl,  "v", "",   ALL_MS_LANGUAGES)
-
 // Misc
 BUILTIN(__builtin_sponentry, "v*", "c")
 

clang/include/clang/Basic/BuiltinsARM.def

@@ -186,11 +186,19 @@ BUILTIN(__builtin_arm_wfi, "v", "")
 BUILTIN(__builtin_arm_sev, "v", "")
 BUILTIN(__builtin_arm_sevl, "v", "")
 BUILTIN(__builtin_arm_dbg, "vUi", "")
+TARGET_HEADER_BUILTIN(__yield, "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__wfe, "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__wfi, "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__sev, "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__sevl, "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
 
 // Data barrier
 BUILTIN(__builtin_arm_dmb, "vUi", "nc")
 BUILTIN(__builtin_arm_dsb, "vUi", "nc")
 BUILTIN(__builtin_arm_isb, "vUi", "nc")
+TARGET_HEADER_BUILTIN(__dmb, "vUi", "nch", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__dsb, "vUi", "nch", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__isb, "vUi", "nch", ARMACLE_H, ALL_LANGUAGES, "")
 
 // Prefetch
 BUILTIN(__builtin_arm_prefetch, "vvC*UiUi", "nc")
@@ -209,15 +217,6 @@ BUILTIN(__builtin_sponentry, "v*", "c")
 // MSVC
 LANGBUILTIN(__emit, "vIUiC", "", ALL_MS_LANGUAGES)
 
-LANGBUILTIN(__yield, "v", "", ALL_MS_LANGUAGES)
-LANGBUILTIN(__wfe, "v", "", ALL_MS_LANGUAGES)
-LANGBUILTIN(__wfi, "v", "", ALL_MS_LANGUAGES)
-LANGBUILTIN(__sev, "v", "", ALL_MS_LANGUAGES)
-LANGBUILTIN(__sevl, "v", "", ALL_MS_LANGUAGES)
-
-LANGBUILTIN(__dmb, "vUi", "nc", ALL_MS_LANGUAGES)
-LANGBUILTIN(__dsb, "vUi", "nc", ALL_MS_LANGUAGES)
-LANGBUILTIN(__isb, "vUi", "nc", ALL_MS_LANGUAGES)
 LANGBUILTIN(__ldrexd, "WiWiCD*", "", ALL_MS_LANGUAGES)
 LANGBUILTIN(_MoveFromCoprocessor, "UiIUiIUiIUiIUiIUi", "", ALL_MS_LANGUAGES)
 LANGBUILTIN(_MoveFromCoprocessor2, "UiIUiIUiIUiIUiIUi", "", ALL_MS_LANGUAGES)

clang/include/clang/Sema/SemaHLSL.h

@@ -131,6 +131,8 @@ class SemaHLSL : public SemaBase {
   void handleParamModifierAttr(Decl *D, const ParsedAttr &AL);
   bool handleResourceTypeAttr(QualType T, const ParsedAttr &AL);
 
+  void handleVkExtBuiltinInputAttr(Decl *D, const ParsedAttr &AL);
+
   bool CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall);
   QualType ProcessResourceTypeAttributes(QualType Wrapped);
   HLSLAttributedResourceLocInfo

clang/lib/AST/Type.cpp

@@ -100,6 +100,7 @@ bool Qualifiers::isTargetAddressSpaceSupersetOf(LangAS A, LangAS B,
          // address spaces to default to work around this problem.
          (A == LangAS::Default && B == LangAS::hlsl_private) ||
          (A == LangAS::Default && B == LangAS::hlsl_device) ||
+         (A == LangAS::Default && B == LangAS::hlsl_input) ||
          // Conversions from target specific address spaces may be legal
          // depending on the target information.
          Ctx.getTargetInfo().isAddressSpaceSupersetOf(A, B);

clang/lib/AST/TypePrinter.cpp

@@ -2667,6 +2667,8 @@ std::string Qualifiers::getAddrSpaceAsString(LangAS AS) {
     return "hlsl_private";
   case LangAS::hlsl_device:
     return "hlsl_device";
+  case LangAS::hlsl_input:
+    return "hlsl_input";
   case LangAS::wasm_funcref:
     return "__funcref";
   default:

clang/lib/Basic/Attributes.cpp

@@ -210,6 +210,7 @@ getScopeFromNormalizedScopeName(StringRef ScopeName) {
       .Case("gnu", AttributeCommonInfo::Scope::GNU)
       .Case("gsl", AttributeCommonInfo::Scope::GSL)
       .Case("hlsl", AttributeCommonInfo::Scope::HLSL)
+      .Case("vk", AttributeCommonInfo::Scope::VK)
       .Case("msvc", AttributeCommonInfo::Scope::MSVC)
       .Case("omp", AttributeCommonInfo::Scope::OMP)
       .Case("riscv", AttributeCommonInfo::Scope::RISCV);

clang/lib/Basic/TargetInfo.cpp

@@ -49,6 +49,8 @@ static const LangASMap FakeAddrSpaceMap = {
     13, // hlsl_groupshared
     14, // hlsl_constant
     15, // hlsl_private
+    16, // hlsl_device
+    17, // hlsl_input
     20, // wasm_funcref
 };
 

clang/lib/Basic/Targets/AArch64.h

@@ -47,6 +47,7 @@ static const unsigned ARM64AddrSpaceMap[] = {
     0, // hlsl_constant
     0, // hlsl_private
     0, // hlsl_device
+    0, // hlsl_input
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref

clang/lib/Basic/Targets/AMDGPU.cpp

@@ -62,6 +62,7 @@ const LangASMap AMDGPUTargetInfo::AMDGPUDefIsGenMap = {
     // will break loudly.
     llvm::AMDGPUAS::PRIVATE_ADDRESS, // hlsl_private
     llvm::AMDGPUAS::GLOBAL_ADDRESS,  // hlsl_device
+    llvm::AMDGPUAS::PRIVATE_ADDRESS, // hlsl_input
 };
 
 const LangASMap AMDGPUTargetInfo::AMDGPUDefIsPrivMap = {
@@ -89,6 +90,7 @@ const LangASMap AMDGPUTargetInfo::AMDGPUDefIsPrivMap = {
     llvm::AMDGPUAS::CONSTANT_ADDRESS, // hlsl_constant
     llvm::AMDGPUAS::PRIVATE_ADDRESS,  // hlsl_private
     llvm::AMDGPUAS::GLOBAL_ADDRESS,   // hlsl_device
+    llvm::AMDGPUAS::PRIVATE_ADDRESS,  // hlsl_input
 };
 } // namespace targets
 } // namespace clang

clang/lib/Basic/Targets/DirectX.h

@@ -45,6 +45,7 @@ static const unsigned DirectXAddrSpaceMap[] = {
     2, // hlsl_constant
     0, // hlsl_private
     0, // hlsl_device
+    0, // hlsl_input
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref

clang/lib/Basic/Targets/NVPTX.h

@@ -49,6 +49,7 @@ static const unsigned NVPTXAddrSpaceMap[] = {
     0, // hlsl_constant
     0, // hlsl_private
     0, // hlsl_device
+    0, // hlsl_input
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref

clang/lib/Basic/Targets/SPIR.h

@@ -50,6 +50,7 @@ static const unsigned SPIRDefIsPrivMap[] = {
     12, // hlsl_constant
     10, // hlsl_private
     11, // hlsl_device
+    7,  // hlsl_input
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
@@ -85,6 +86,7 @@ static const unsigned SPIRDefIsGenMap[] = {
     0,  // hlsl_constant
     10, // hlsl_private
     11, // hlsl_device
+    7,  // hlsl_input
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref

clang/lib/Basic/Targets/SystemZ.h

@@ -45,6 +45,7 @@ static const unsigned ZOSAddressMap[] = {
     0, // hlsl_constant
     0, // hlsl_private
     0, // hlsl_device
+    0, // hlsl_input
     0  // wasm_funcref
 };
 

clang/lib/Basic/Targets/TCE.h

@@ -54,6 +54,7 @@ static const unsigned TCEOpenCLAddrSpaceMap[] = {
     0, // hlsl_constant
     0, // hlsl_private
     0, // hlsl_device
+    0, // hlsl_input
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref

clang/lib/Basic/Targets/WebAssembly.h

@@ -45,6 +45,7 @@ static const unsigned WebAssemblyAddrSpaceMap[] = {
     0,  // hlsl_constant
     0,  // hlsl_private
     0,  // hlsl_device
+    0,  // hlsl_input
     20, // wasm_funcref
 };
 

clang/lib/Basic/Targets/X86.h

@@ -49,6 +49,7 @@ static const unsigned X86AddrSpaceMap[] = {
     0,   // hlsl_constant
     0,   // hlsl_private
     0,   // hlsl_device
+    0,   // hlsl_input
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref

clang/lib/CodeGen/CGCall.cpp

@@ -3941,9 +3941,9 @@ llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
   return R;
 }
 
-void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
-                                         bool EmitRetDbgLoc,
-                                         SourceLocation EndLoc) {
+void CodeGenFunction::EmitFunctionEpilog(
+    const CGFunctionInfo &FI, bool EmitRetDbgLoc, SourceLocation EndLoc,
+    uint64_t RetKeyInstructionsSourceAtom) {
   if (FI.isNoReturn()) {
     // Noreturn functions don't return.
     EmitUnreachable(EndLoc);
@@ -3958,7 +3958,11 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
 
   // Functions with no result always return void.
   if (!ReturnValue.isValid()) {
-    Builder.CreateRetVoid();
+    auto *I = Builder.CreateRetVoid();
+    if (RetKeyInstructionsSourceAtom)
+      addInstToSpecificSourceAtom(I, nullptr, RetKeyInstructionsSourceAtom);
+    else
+      addInstToNewSourceAtom(I, nullptr);
     return;
   }
 
@@ -4138,6 +4142,12 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
 
   if (RetDbgLoc)
     Ret->setDebugLoc(std::move(RetDbgLoc));
+
+  llvm::Value *Backup = RV ? Ret->getOperand(0) : nullptr;
+  if (RetKeyInstructionsSourceAtom)
+    addInstToSpecificSourceAtom(Ret, Backup, RetKeyInstructionsSourceAtom);
+  else
+    addInstToNewSourceAtom(Ret, Backup);
 }
 
 void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {

clang/lib/CodeGen/CGHLSLRuntime.cpp

@@ -624,6 +624,20 @@ void CGHLSLRuntime::initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
   }
 }
 
+void CGHLSLRuntime::handleGlobalVarDefinition(const VarDecl *VD,
+                                              llvm::GlobalVariable *GV) {
+  if (auto Attr = VD->getAttr<HLSLVkExtBuiltinInputAttr>()) {
+    LLVMContext &Ctx = GV->getContext();
+    IRBuilder<> B(GV->getContext());
+    MDNode *Operands = MDNode::get(
+        Ctx, {ConstantAsMetadata::get(
+                  B.getInt32(/* Spirv::Decoration::BuiltIn */ 11)),
+              ConstantAsMetadata::get(B.getInt32(Attr->getBuiltIn()))});
+    MDNode *Decoration = MDNode::get(Ctx, {Operands});
+    GV->addMetadata("spirv.Decorations", *Decoration);
+  }
+}
+
 llvm::Instruction *CGHLSLRuntime::getConvergenceToken(BasicBlock &BB) {
   if (!CGM.shouldEmitConvergenceTokens())
     return nullptr;

clang/lib/CodeGen/CGHLSLRuntime.h

@@ -157,6 +157,7 @@ class CGHLSLRuntime {
 
   void emitEntryFunction(const FunctionDecl *FD, llvm::Function *Fn);
   void setHLSLFunctionAttributes(const FunctionDecl *FD, llvm::Function *Fn);
+  void handleGlobalVarDefinition(const VarDecl *VD, llvm::GlobalVariable *Var);
 
   llvm::Instruction *getConvergenceToken(llvm::BasicBlock &BB);
 

clang/lib/CodeGen/CGStmt.cpp

@@ -2869,6 +2869,7 @@ static bool isSwiftAsyncCallee(const CallExpr *CE) {
 /// if the function returns void, or may be missing one if the function returns
 /// non-void.  Fun stuff :).
 void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
+  ApplyAtomGroup Grp(getDebugInfo());
   if (requiresReturnValueCheck()) {
     llvm::Constant *SLoc = EmitCheckSourceLocation(S.getBeginLoc());
     auto *SLocPtr =
@@ -2944,16 +2945,19 @@ void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
     // If this function returns a reference, take the address of the expression
     // rather than the value.
     RValue Result = EmitReferenceBindingToExpr(RV);
-    Builder.CreateStore(Result.getScalarVal(), ReturnValue);
+    auto *I = Builder.CreateStore(Result.getScalarVal(), ReturnValue);
+    addInstToCurrentSourceAtom(I, I->getValueOperand());
   } else {
     switch (getEvaluationKind(RV->getType())) {
     case TEK_Scalar: {
       llvm::Value *Ret = EmitScalarExpr(RV);
-      if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect)
+      if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
         EmitStoreOfScalar(Ret, MakeAddrLValue(ReturnValue, RV->getType()),
                           /*isInit*/ true);
-      else
-        Builder.CreateStore(Ret, ReturnValue);
+      } else {
+        auto *I = Builder.CreateStore(Ret, ReturnValue);
+        addInstToCurrentSourceAtom(I, I->getValueOperand());
+      }
       break;
     }
     case TEK_Complex: