[SIL] Introduce the increment_profiler_counter instruction

docs/SIL.rst

@@ -3898,6 +3898,22 @@ It is worth noting that a SIL DIExpression is similar to
 info metadata. While LLVM represents ``!DIExpression`` are a list of 64-bit integers,
 SIL DIExpression can have elements with various types, like AST nodes or strings.
 
+Profiling
+~~~~~~~~~
+
+increment_profiler_counter
+``````````````````````````
+::
+
+  sil-instruction ::= 'increment_profiler_counter' int-literal ',' string-literal ',' 'num_counters' int-literal ',' 'hash' int-literal
+
+  increment_profiler_counter 1, "$foo", num_counters 3, hash 0
+
+Increments a given profiler counter for a given PGO function name. This is
+lowered to the ``llvm.instrprof.increment`` LLVM intrinsic. This instruction
+is emitted when profiling is enabled, and enables features such as code coverage
+and profile-guided optimization.
+
 Accessing Memory
 ~~~~~~~~~~~~~~~~
 

include/swift/AST/Builtins.def

@@ -738,9 +738,6 @@ BUILTIN_MISC_OPERATION(PoundAssert, "poundAssert", "", Special)
 // TypePtrAuthDiscriminator has type <T> (T.Type) -> Int64
 BUILTIN_MISC_OPERATION(TypePtrAuthDiscriminator, "typePtrAuthDiscriminator", "n", Special)
 
-//  int_instrprof_increment has type (Builtin.RawPointer, Builtin.Int64, Builtin.Int32, Builtin.Int32) -> ().
-BUILTIN_MISC_OPERATION(IntInstrprofIncrement, "int_instrprof_increment", "", Special)
-
 /// Initialize the default-actor instance in a default actor object.
 BUILTIN_MISC_OPERATION(InitializeDefaultActor, "initializeDefaultActor", "", Special)
 

include/swift/AST/DiagnosticsParse.def

@@ -531,6 +531,14 @@ ERROR(sil_invalid_attribute_for_instruction,none,
 ERROR(sil_invalid_attribute_for_expected,none,
       "Invalid attribute '%0' (expected '%1').", 
       (StringRef, StringRef))
+ERROR(expected_sil_profiler_counter_idx,none,
+      "expected profiler counter index", ())
+ERROR(expected_sil_profiler_counter_pgo_func_name,none,
+      "expected profiler counter PGO function name", ())
+ERROR(expected_sil_profiler_counter_total,none,
+      "expected profiler counter total", ())
+ERROR(expected_sil_profiler_counter_hash,none,
+      "expected profiler counter hash", ())
 
 // SIL Values
 ERROR(sil_value_redefinition,none,

include/swift/SIL/SILBuilder.h

@@ -2131,6 +2131,19 @@ class SILBuilder {
     return createCondFail(loc, one, message);
   }
 
+  //===--------------------------------------------------------------------===//
+  // Profiler
+  //===--------------------------------------------------------------------===//
+
+  IncrementProfilerCounterInst *
+  createIncrementProfilerCounter(SILLocation Loc, unsigned CounterIdx,
+                                 StringRef PGOFuncName, unsigned NumCounters,
+                                 uint64_t PGOFuncHash) {
+    return insert(IncrementProfilerCounterInst::create(
+        getSILDebugLocation(Loc), CounterIdx, PGOFuncName, NumCounters,
+        PGOFuncHash, getModule()));
+  }
+
   //===--------------------------------------------------------------------===//
   // Array indexing instructions
   //===--------------------------------------------------------------------===//

include/swift/SIL/SILCloner.h

@@ -2620,6 +2620,17 @@ SILCloner<ImplClass>::visitCondFailInst(CondFailInst *Inst) {
                                         Inst->getMessage()));
 }
 
+template <typename ImplClass>
+void SILCloner<ImplClass>::visitIncrementProfilerCounterInst(
+    IncrementProfilerCounterInst *Inst) {
+  getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope()));
+  recordClonedInstruction(Inst,
+                          getBuilder().createIncrementProfilerCounter(
+                              getOpLocation(Inst->getLoc()),
+                              Inst->getCounterIndex(), Inst->getPGOFuncName(),
+                              Inst->getNumCounters(), Inst->getPGOFuncHash()));
+}
+
 template<typename ImplClass>
 void
 SILCloner<ImplClass>::visitIndexAddrInst(IndexAddrInst *Inst) {

include/swift/SIL/SILInstruction.h

@@ -3858,7 +3858,53 @@ class BuiltinInst final
     return OperandValueArrayRef(getAllOperands());
   }
 };
-
+
+/// Increments a given profiler counter for a given PGO function name. This is
+/// lowered to the \c llvm.instrprof.increment LLVM intrinsic.
+class IncrementProfilerCounterInst final
+    : public InstructionBase<SILInstructionKind::IncrementProfilerCounterInst,
+                             NonValueInstruction>,
+      private llvm::TrailingObjects<IncrementProfilerCounterInst, char> {
+  friend TrailingObjects;
+  friend SILBuilder;
+
+  unsigned CounterIdx;
+  unsigned PGOFuncNameLength;
+  unsigned NumCounters;
+  uint64_t PGOFuncHash;
+
+  IncrementProfilerCounterInst(SILDebugLocation Loc, unsigned CounterIdx,
+                               unsigned PGOFuncNameLength, unsigned NumCounters,
+                               uint64_t PGOFuncHash)
+      : InstructionBase(Loc), CounterIdx(CounterIdx),
+        PGOFuncNameLength(PGOFuncNameLength), NumCounters(NumCounters),
+        PGOFuncHash(PGOFuncHash) {}
+
+  static IncrementProfilerCounterInst *
+  create(SILDebugLocation Loc, unsigned CounterIdx, StringRef PGOFuncName,
+         unsigned NumCounters, uint64_t PGOFuncHash, SILModule &M);
+
+public:
+  /// The index of the counter to be incremented.
+  unsigned getCounterIndex() const { return CounterIdx; }
+
+  /// The PGO function name for the function in which the counter resides.
+  StringRef getPGOFuncName() const {
+    return StringRef(getTrailingObjects<char>(), PGOFuncNameLength);
+  }
+
+  /// The total number of counters within the function.
+  unsigned getNumCounters() const { return NumCounters; }
+
+  /// A hash value for the function used to determine whether the profile is
+  /// outdated.
+  /// FIXME: This is currently always 0.
+  uint64_t getPGOFuncHash() const { return PGOFuncHash; }
+
+  ArrayRef<Operand> getAllOperands() const { return {}; }
+  MutableArrayRef<Operand> getAllOperands() { return {}; }
+};
+
 /// Initializes a SIL global variable. Only valid once, before any
 /// usages of the global via GlobalAddrInst.
 class AllocGlobalInst

include/swift/SIL/SILNodes.def

@@ -810,7 +810,10 @@ NON_VALUE_INST(AbortApplyInst, abort_apply,
 BRIDGED_NON_VALUE_INST(CondFailInst, cond_fail,
                SILInstruction, MayHaveSideEffects, DoesNotRelease)
 
-NODE_RANGE(NonValueInstruction, UnreachableInst, CondFailInst)
+NON_VALUE_INST(IncrementProfilerCounterInst, increment_profiler_counter,
+               SILInstruction, MayReadWrite, DoesNotRelease)
+
+NODE_RANGE(NonValueInstruction, UnreachableInst, IncrementProfilerCounterInst)
 
 ABSTRACT_INST(MultipleValueInstruction, SILInstruction)
 MULTIPLE_VALUE_INST(BeginApplyInst, begin_apply,

lib/AST/Builtins.cpp

@@ -1190,15 +1190,6 @@ static ValueDecl *getCOWBufferForReading(ASTContext &C, Identifier Id) {
   return builder.build(Id);
 }
 
-static ValueDecl *getIntInstrprofIncrement(ASTContext &C, Identifier Id) {
-  // (Builtin.RawPointer, Builtin.Int64, Builtin.Int32, Builtin.Int32) -> ()
-  Type Int64Ty = BuiltinIntegerType::get(64, C);
-  Type Int32Ty = BuiltinIntegerType::get(32, C);
-  return getBuiltinFunction(Id,
-                            {C.TheRawPointerType, Int64Ty, Int32Ty, Int32Ty},
-                            TupleType::getEmpty(C));
-}
-
 static ValueDecl *getTypePtrAuthDiscriminator(ASTContext &C, Identifier Id) {
   // <T : AnyObject> (T.Type) -> Int64
   BuiltinFunctionBuilder builder(C);
@@ -2844,9 +2835,6 @@ ValueDecl *swift::getBuiltinValueDecl(ASTContext &Context, Identifier Id) {
                               {},
                               TupleType::getEmpty(Context));
 
-  case BuiltinValueKind::IntInstrprofIncrement:
-    return getIntInstrprofIncrement(Context, Id);
-
   case BuiltinValueKind::TypePtrAuthDiscriminator:
     return getTypePtrAuthDiscriminator(Context, Id);
 

lib/IRGen/GenBuiltin.cpp

@@ -400,56 +400,6 @@ void irgen::emitBuiltinCall(IRGenFunction &IGF, const BuiltinInfo &Builtin,
     return;
   }
 
-  // Calls to the int_instrprof_increment intrinsic are emitted during SILGen.
-  // At that stage, the function name GV used by the profiling pass is hidden.
-  // Fix the intrinsic call here by pointing it to the correct GV.
-  if (IID == llvm::Intrinsic::instrprof_increment) {
-    // If we import profiling intrinsics from a swift module but profiling is
-    // not enabled, ignore the increment.
-    SILModule &SILMod = IGF.getSILModule();
-    const auto &Opts = SILMod.getOptions();
-    if (!Opts.GenerateProfile) {
-      (void)args.claimAll();
-      return;
-    }
-
-    // Extract the PGO function name.
-    auto *NameGEP = cast<llvm::User>(args.claimNext());
-    auto *NameGV = dyn_cast<llvm::GlobalVariable>(NameGEP->stripPointerCasts());
-
-    // TODO: The SIL optimizer may rewrite the name argument in a way that
-    // makes it impossible to lower. Until that issue is fixed, defensively
-    // refuse to lower ill-formed intrinsics (rdar://39146527).
-    if (!NameGV) {
-      (void)args.claimAll();
-      return;
-    }
-
-    auto *NameC = NameGV->getInitializer();
-    StringRef Name = cast<llvm::ConstantDataArray>(NameC)->getRawDataValues();
-    StringRef PGOFuncName = Name.rtrim(StringRef("\0", 1));
-
-    // Point the increment call to the right function name variable.
-    std::string PGOFuncNameVar = llvm::getPGOFuncNameVarName(
-        PGOFuncName, llvm::GlobalValue::LinkOnceAnyLinkage);
-    auto *FuncNamePtr = IGF.IGM.Module.getNamedGlobal(PGOFuncNameVar);
-    if (!FuncNamePtr)
-      FuncNamePtr = llvm::createPGOFuncNameVar(
-          *IGF.IGM.getModule(), llvm::GlobalValue::LinkOnceAnyLinkage,
-          PGOFuncName);
-
-    llvm::SmallVector<llvm::Value *, 2> Indices(2, NameGEP->getOperand(1));
-    NameGEP = llvm::ConstantExpr::getGetElementPtr(
-        ((llvm::PointerType *)FuncNamePtr->getType())->getPointerElementType(),
-        FuncNamePtr, makeArrayRef(Indices));
-
-    // Replace the placeholder value with the new GEP.
-    Explosion replacement;
-    replacement.add(NameGEP);
-    replacement.add(args.claimAll());
-    args = std::move(replacement);
-  }
-
   // Implement the ptrauth builtins as no-ops when the Clang
   // intrinsics are disabled.
   if ((IID == llvm::Intrinsic::ptrauth_sign ||

lib/IRGen/IRGenSIL.cpp

@@ -1315,7 +1315,9 @@ class IRGenSILFunction :
   void visitRebindMemoryInst(RebindMemoryInst *i);
 
   void visitCondFailInst(CondFailInst *i);
-
+
+  void visitIncrementProfilerCounterInst(IncrementProfilerCounterInst *I);
+
   void visitConvertFunctionInst(ConvertFunctionInst *i);
   void visitConvertEscapeToNoEscapeInst(ConvertEscapeToNoEscapeInst *i);
   void visitUpcastInst(UpcastInst *i);
@@ -6956,6 +6958,39 @@ void IRGenSILFunction::visitCondFailInst(swift::CondFailInst *i) {
   FailBBs.push_back(failBB);
 }
 
+void IRGenSILFunction::visitIncrementProfilerCounterInst(
+    IncrementProfilerCounterInst *i) {
+  // If we import profiling intrinsics from a swift module but profiling is
+  // not enabled, ignore the increment.
+  if (!getSILModule().getOptions().GenerateProfile)
+    return;
+
+  // Retrieve the global variable that stores the PGO function name, creating it
+  // if needed.
+  auto funcName = i->getPGOFuncName();
+  auto varLinkage = llvm::GlobalValue::LinkOnceAnyLinkage;
+  auto *nameVar = IGM.Module.getNamedGlobal(
+      llvm::getPGOFuncNameVarName(funcName, varLinkage));
+  if (!nameVar)
+    nameVar = llvm::createPGOFuncNameVar(IGM.Module, varLinkage, funcName);
+
+  // We need to GEP the function name global to point to the first character of
+  // the string.
+  llvm::SmallVector<llvm::Value *, 2> indices;
+  indices.append(2, llvm::ConstantInt::get(IGM.SizeTy, 0));
+  auto *nameGEP = llvm::ConstantExpr::getGetElementPtr(
+      nameVar->getValueType(), nameVar, makeArrayRef(indices));
+
+  // Emit the call to the 'llvm.instrprof.increment' LLVM intrinsic.
+  llvm::Value *args[] = {
+    nameGEP,
+    llvm::ConstantInt::get(IGM.Int64Ty, i->getPGOFuncHash()),
+    llvm::ConstantInt::get(IGM.Int32Ty, i->getNumCounters()),
+    llvm::ConstantInt::get(IGM.Int32Ty, i->getCounterIndex())
+  };
+  Builder.CreateIntrinsicCall(llvm::Intrinsic::instrprof_increment, args);
+}
+
 void IRGenSILFunction::visitSuperMethodInst(swift::SuperMethodInst *i) {
   assert(!i->getMember().isForeign);
 

lib/SIL/IR/OperandOwnership.cpp

@@ -119,6 +119,7 @@ SHOULD_NEVER_VISIT_INST(ReleaseValue)
 SHOULD_NEVER_VISIT_INST(ReleaseValueAddr)
 SHOULD_NEVER_VISIT_INST(StrongRelease)
 SHOULD_NEVER_VISIT_INST(GetAsyncContinuation)
+SHOULD_NEVER_VISIT_INST(IncrementProfilerCounter)
 
 #define ALWAYS_OR_SOMETIMES_LOADABLE_CHECKED_REF_STORAGE(Name, ...)            \
   SHOULD_NEVER_VISIT_INST(StrongRetain##Name)                                  \
@@ -794,7 +795,6 @@ BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, PoundAssert)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, GlobalStringTablePointer)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, TypePtrAuthDiscriminator)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, TargetOSVersionAtLeast)
-BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, IntInstrprofIncrement)
 BUILTIN_OPERAND_OWNERSHIP(UnownedInstantaneousUse, Copy)
 BUILTIN_OPERAND_OWNERSHIP(DestroyingConsume, StartAsyncLet)
 BUILTIN_OPERAND_OWNERSHIP(DestroyingConsume, EndAsyncLet)

lib/SIL/IR/SILInstructions.cpp

@@ -438,6 +438,22 @@ BuiltinInst::BuiltinInst(SILDebugLocation Loc, Identifier Name,
       Substitutions(Subs) {
 }
 
+IncrementProfilerCounterInst *IncrementProfilerCounterInst::create(
+    SILDebugLocation Loc, unsigned CounterIdx, StringRef PGOFuncName,
+    unsigned NumCounters, uint64_t PGOFuncHash, SILModule &M) {
+
+  auto PGOFuncNameLength = PGOFuncName.size();
+  auto Size = totalSizeToAlloc<char>(PGOFuncNameLength);
+  auto Buffer = M.allocateInst(Size, alignof(IncrementProfilerCounterInst));
+
+  auto *Inst = ::new (Buffer) IncrementProfilerCounterInst(
+      Loc, CounterIdx, PGOFuncNameLength, NumCounters, PGOFuncHash);
+
+  std::uninitialized_copy(PGOFuncName.begin(), PGOFuncName.end(),
+                          Inst->getTrailingObjects<char>());
+  return Inst;
+}
+
 InitBlockStorageHeaderInst *
 InitBlockStorageHeaderInst::create(SILFunction &F,
                                SILDebugLocation DebugLoc, SILValue BlockStorage,

lib/SIL/IR/SILPrinter.cpp

@@ -2359,7 +2359,14 @@ class SILPrinter : public SILInstructionVisitor<SILPrinter> {
     *this << getIDAndType(FI->getOperand()) << ", "
           << QuotedString(FI->getMessage());
   }
-
+
+  void visitIncrementProfilerCounterInst(IncrementProfilerCounterInst *IPCI) {
+    *this << IPCI->getCounterIndex() << ", "
+          << QuotedString(IPCI->getPGOFuncName()) << ", "
+          << "num_counters " << IPCI->getNumCounters() << ", "
+          << "hash " << IPCI->getPGOFuncHash();
+  }
+
   void visitIndexAddrInst(IndexAddrInst *IAI) {
     *this << getIDAndType(IAI->getBase()) << ", "
           << getIDAndType(IAI->getIndex());

lib/SIL/IR/ValueOwnership.cpp

@@ -532,7 +532,6 @@ CONSTANT_OWNERSHIP_BUILTIN(None, TSanInoutAccess)
 CONSTANT_OWNERSHIP_BUILTIN(None, Swift3ImplicitObjCEntrypoint)
 CONSTANT_OWNERSHIP_BUILTIN(None, PoundAssert)
 CONSTANT_OWNERSHIP_BUILTIN(None, TypePtrAuthDiscriminator)
-CONSTANT_OWNERSHIP_BUILTIN(None, IntInstrprofIncrement)
 CONSTANT_OWNERSHIP_BUILTIN(None, TargetOSVersionAtLeast)
 CONSTANT_OWNERSHIP_BUILTIN(None, GlobalStringTablePointer)
 CONSTANT_OWNERSHIP_BUILTIN(None, GetCurrentAsyncTask)