[SIL] Introduce the increment_profiler_counter instruction

This is a dedicated instruction for incrementing a
profiler counter, which lowers to the
`llvm.instrprof.increment` intrinsic. This
replaces the builtin instruction that was
previously used, and ensures that its arguments
are statically known. This ensures that SIL
optimization passes do not invalidate the
instruction, fixing some code coverage cases in
`-O`.

rdar://39146527

Author: hamishknight

docs/SIL.rst

@@ -3898,6 +3898,22 @@ It is worth noting that a SIL DIExpression is similar to
 info metadata. While LLVM represents ``!DIExpression`` are a list of 64-bit integers,
 SIL DIExpression can have elements with various types, like AST nodes or strings.
 
+Profiling
+~~~~~~~~~
+
+increment_profiler_counter
+``````````````````````````
+::
+
+  sil-instruction ::= 'increment_profiler_counter' int-literal ',' string-literal ',' 'num_counters' int-literal ',' 'hash' int-literal
+
+  increment_profiler_counter 1, "$foo", num_counters 3, hash 0
+
+Increments a given profiler counter for a given PGO function name. This is
+lowered to the ``llvm.instrprof.increment`` LLVM intrinsic. This instruction
+is emitted when profiling is enabled, and enables features such as code coverage
+and profile-guided optimization.
+
 Accessing Memory
 ~~~~~~~~~~~~~~~~
 

include/swift/AST/Builtins.def

@@ -738,9 +738,6 @@ BUILTIN_MISC_OPERATION(PoundAssert, "poundAssert", "", Special)
 // TypePtrAuthDiscriminator has type <T> (T.Type) -> Int64
 BUILTIN_MISC_OPERATION(TypePtrAuthDiscriminator, "typePtrAuthDiscriminator", "n", Special)
 
-//  int_instrprof_increment has type (Builtin.RawPointer, Builtin.Int64, Builtin.Int32, Builtin.Int32) -> ().
-BUILTIN_MISC_OPERATION(IntInstrprofIncrement, "int_instrprof_increment", "", Special)
-
 /// Initialize the default-actor instance in a default actor object.
 BUILTIN_MISC_OPERATION(InitializeDefaultActor, "initializeDefaultActor", "", Special)
 

include/swift/AST/DiagnosticsParse.def

@@ -531,6 +531,14 @@ ERROR(sil_invalid_attribute_for_instruction,none,
 ERROR(sil_invalid_attribute_for_expected,none,
       "Invalid attribute '%0' (expected '%1').", 
       (StringRef, StringRef))
+ERROR(expected_sil_profiler_counter_idx,none,
+      "expected profiler counter index", ())
+ERROR(expected_sil_profiler_counter_pgo_func_name,none,
+      "expected profiler counter PGO function name", ())
+ERROR(expected_sil_profiler_counter_total,none,
+      "expected profiler counter total", ())
+ERROR(expected_sil_profiler_counter_hash,none,
+      "expected profiler counter hash", ())
 
 // SIL Values
 ERROR(sil_value_redefinition,none,

include/swift/SIL/SILBuilder.h

@@ -2131,6 +2131,19 @@ class SILBuilder {
     return createCondFail(loc, one, message);
   }
 
+  //===--------------------------------------------------------------------===//
+  // Profiler
+  //===--------------------------------------------------------------------===//
+
+  IncrementProfilerCounterInst *
+  createIncrementProfilerCounter(SILLocation Loc, unsigned CounterIdx,
+                                 StringRef PGOFuncName, unsigned NumCounters,
+                                 uint64_t PGOFuncHash) {
+    return insert(IncrementProfilerCounterInst::create(
+        getSILDebugLocation(Loc), CounterIdx, PGOFuncName, NumCounters,
+        PGOFuncHash, getModule()));
+  }
+
   //===--------------------------------------------------------------------===//
   // Array indexing instructions
   //===--------------------------------------------------------------------===//

include/swift/SIL/SILCloner.h

@@ -2620,6 +2620,17 @@ SILCloner<ImplClass>::visitCondFailInst(CondFailInst *Inst) {
                                         Inst->getMessage()));
 }
 
+template <typename ImplClass>
+void SILCloner<ImplClass>::visitIncrementProfilerCounterInst(
+    IncrementProfilerCounterInst *Inst) {
+  getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope()));
+  recordClonedInstruction(Inst,
+                          getBuilder().createIncrementProfilerCounter(
+                              getOpLocation(Inst->getLoc()),
+                              Inst->getCounterIndex(), Inst->getPGOFuncName(),
+                              Inst->getNumCounters(), Inst->getPGOFuncHash()));
+}
+
 template<typename ImplClass>
 void
 SILCloner<ImplClass>::visitIndexAddrInst(IndexAddrInst *Inst) {

include/swift/SIL/SILInstruction.h

@@ -3858,7 +3858,53 @@ class BuiltinInst final
     return OperandValueArrayRef(getAllOperands());
   }
 };
-  
+
+/// Increments a given profiler counter for a given PGO function name. This is
+/// lowered to the \c llvm.instrprof.increment LLVM intrinsic.
+class IncrementProfilerCounterInst final
+    : public InstructionBase<SILInstructionKind::IncrementProfilerCounterInst,
+                             NonValueInstruction>,
+      private llvm::TrailingObjects<IncrementProfilerCounterInst, char> {
+  friend TrailingObjects;
+  friend SILBuilder;
+
+  unsigned CounterIdx;
+  unsigned PGOFuncNameLength;
+  unsigned NumCounters;
+  uint64_t PGOFuncHash;
+
+  IncrementProfilerCounterInst(SILDebugLocation Loc, unsigned CounterIdx,
+                               unsigned PGOFuncNameLength, unsigned NumCounters,
+                               uint64_t PGOFuncHash)
+      : InstructionBase(Loc), CounterIdx(CounterIdx),
+        PGOFuncNameLength(PGOFuncNameLength), NumCounters(NumCounters),
+        PGOFuncHash(PGOFuncHash) {}
+
+  static IncrementProfilerCounterInst *
+  create(SILDebugLocation Loc, unsigned CounterIdx, StringRef PGOFuncName,
+         unsigned NumCounters, uint64_t PGOFuncHash, SILModule &M);
+
+public:
+  /// The index of the counter to be incremented.
+  unsigned getCounterIndex() const { return CounterIdx; }
+
+  /// The PGO function name for the function in which the counter resides.
+  StringRef getPGOFuncName() const {
+    return StringRef(getTrailingObjects<char>(), PGOFuncNameLength);
+  }
+
+  /// The total number of counters within the function.
+  unsigned getNumCounters() const { return NumCounters; }
+
+  /// A hash value for the function used to determine whether the profile is
+  /// outdated.
+  /// FIXME: This is currently always 0.
+  uint64_t getPGOFuncHash() const { return PGOFuncHash; }
+
+  ArrayRef<Operand> getAllOperands() const { return {}; }
+  MutableArrayRef<Operand> getAllOperands() { return {}; }
+};
+
 /// Initializes a SIL global variable. Only valid once, before any
 /// usages of the global via GlobalAddrInst.
 class AllocGlobalInst

include/swift/SIL/SILNodes.def

@@ -810,7 +810,10 @@ NON_VALUE_INST(AbortApplyInst, abort_apply,
 BRIDGED_NON_VALUE_INST(CondFailInst, cond_fail,
                SILInstruction, MayHaveSideEffects, DoesNotRelease)
 
-NODE_RANGE(NonValueInstruction, UnreachableInst, CondFailInst)
+NON_VALUE_INST(IncrementProfilerCounterInst, increment_profiler_counter,
+               SILInstruction, MayReadWrite, DoesNotRelease)
+
+NODE_RANGE(NonValueInstruction, UnreachableInst, IncrementProfilerCounterInst)
 
 ABSTRACT_INST(MultipleValueInstruction, SILInstruction)
 MULTIPLE_VALUE_INST(BeginApplyInst, begin_apply,

lib/AST/Builtins.cpp

@@ -1190,15 +1190,6 @@ static ValueDecl *getCOWBufferForReading(ASTContext &C, Identifier Id) {
   return builder.build(Id);
 }
 
-static ValueDecl *getIntInstrprofIncrement(ASTContext &C, Identifier Id) {
-  // (Builtin.RawPointer, Builtin.Int64, Builtin.Int32, Builtin.Int32) -> ()
-  Type Int64Ty = BuiltinIntegerType::get(64, C);
-  Type Int32Ty = BuiltinIntegerType::get(32, C);
-  return getBuiltinFunction(Id,
-                            {C.TheRawPointerType, Int64Ty, Int32Ty, Int32Ty},
-                            TupleType::getEmpty(C));
-}
-
 static ValueDecl *getTypePtrAuthDiscriminator(ASTContext &C, Identifier Id) {
   // <T : AnyObject> (T.Type) -> Int64
   BuiltinFunctionBuilder builder(C);
@@ -2844,9 +2835,6 @@ ValueDecl *swift::getBuiltinValueDecl(ASTContext &Context, Identifier Id) {
                               {},
                               TupleType::getEmpty(Context));
 
-  case BuiltinValueKind::IntInstrprofIncrement:
-    return getIntInstrprofIncrement(Context, Id);
-
   case BuiltinValueKind::TypePtrAuthDiscriminator:
     return getTypePtrAuthDiscriminator(Context, Id);
 

lib/IRGen/GenBuiltin.cpp

@@ -400,56 +400,6 @@ void irgen::emitBuiltinCall(IRGenFunction &IGF, const BuiltinInfo &Builtin,
     return;
   }
 
-  // Calls to the int_instrprof_increment intrinsic are emitted during SILGen.
-  // At that stage, the function name GV used by the profiling pass is hidden.
-  // Fix the intrinsic call here by pointing it to the correct GV.
-  if (IID == llvm::Intrinsic::instrprof_increment) {
-    // If we import profiling intrinsics from a swift module but profiling is
-    // not enabled, ignore the increment.
-    SILModule &SILMod = IGF.getSILModule();
-    const auto &Opts = SILMod.getOptions();
-    if (!Opts.GenerateProfile) {
-      (void)args.claimAll();
-      return;
-    }
-
-    // Extract the PGO function name.
-    auto *NameGEP = cast<llvm::User>(args.claimNext());
-    auto *NameGV = dyn_cast<llvm::GlobalVariable>(NameGEP->stripPointerCasts());
-
-    // TODO: The SIL optimizer may rewrite the name argument in a way that
-    // makes it impossible to lower. Until that issue is fixed, defensively
-    // refuse to lower ill-formed intrinsics (rdar://39146527).
-    if (!NameGV) {
-      (void)args.claimAll();
-      return;
-    }
-
-    auto *NameC = NameGV->getInitializer();
-    StringRef Name = cast<llvm::ConstantDataArray>(NameC)->getRawDataValues();
-    StringRef PGOFuncName = Name.rtrim(StringRef("\0", 1));
-
-    // Point the increment call to the right function name variable.
-    std::string PGOFuncNameVar = llvm::getPGOFuncNameVarName(
-        PGOFuncName, llvm::GlobalValue::LinkOnceAnyLinkage);
-    auto *FuncNamePtr = IGF.IGM.Module.getNamedGlobal(PGOFuncNameVar);
-    if (!FuncNamePtr)
-      FuncNamePtr = llvm::createPGOFuncNameVar(
-          *IGF.IGM.getModule(), llvm::GlobalValue::LinkOnceAnyLinkage,
-          PGOFuncName);
-
-    llvm::SmallVector<llvm::Value *, 2> Indices(2, NameGEP->getOperand(1));
-    NameGEP = llvm::ConstantExpr::getGetElementPtr(
-        ((llvm::PointerType *)FuncNamePtr->getType())->getPointerElementType(),
-        FuncNamePtr, makeArrayRef(Indices));
-
-    // Replace the placeholder value with the new GEP.
-    Explosion replacement;
-    replacement.add(NameGEP);
-    replacement.add(args.claimAll());
-    args = std::move(replacement);
-  }
-
   // Implement the ptrauth builtins as no-ops when the Clang
   // intrinsics are disabled.
   if ((IID == llvm::Intrinsic::ptrauth_sign ||

lib/IRGen/IRGenSIL.cpp

@@ -1315,7 +1315,9 @@ class IRGenSILFunction :
   void visitRebindMemoryInst(RebindMemoryInst *i);
 
   void visitCondFailInst(CondFailInst *i);
-  
+
+  void visitIncrementProfilerCounterInst(IncrementProfilerCounterInst *I);
+
   void visitConvertFunctionInst(ConvertFunctionInst *i);
   void visitConvertEscapeToNoEscapeInst(ConvertEscapeToNoEscapeInst *i);
   void visitUpcastInst(UpcastInst *i);
@@ -6956,6 +6958,39 @@ void IRGenSILFunction::visitCondFailInst(swift::CondFailInst *i) {
   FailBBs.push_back(failBB);
 }
 
+void IRGenSILFunction::visitIncrementProfilerCounterInst(
+    IncrementProfilerCounterInst *i) {
+  // If we import profiling intrinsics from a swift module but profiling is
+  // not enabled, ignore the increment.
+  if (!getSILModule().getOptions().GenerateProfile)
+    return;
+
+  // Retrieve the global variable that stores the PGO function name, creating it
+  // if needed.
+  auto funcName = i->getPGOFuncName();
+  auto varLinkage = llvm::GlobalValue::LinkOnceAnyLinkage;
+  auto *nameVar = IGM.Module.getNamedGlobal(
+      llvm::getPGOFuncNameVarName(funcName, varLinkage));
+  if (!nameVar)
+    nameVar = llvm::createPGOFuncNameVar(IGM.Module, varLinkage, funcName);
+
+  // We need to GEP the function name global to point to the first character of
+  // the string.
+  llvm::SmallVector<llvm::Value *, 2> indices;
+  indices.append(2, llvm::ConstantInt::get(IGM.SizeTy, 0));
+  auto *nameGEP = llvm::ConstantExpr::getGetElementPtr(
+      nameVar->getValueType(), nameVar, makeArrayRef(indices));
+
+  // Emit the call to the 'llvm.instrprof.increment' LLVM intrinsic.
+  llvm::Value *args[] = {
+    nameGEP,
+    llvm::ConstantInt::get(IGM.Int64Ty, i->getPGOFuncHash()),
+    llvm::ConstantInt::get(IGM.Int32Ty, i->getNumCounters()),
+    llvm::ConstantInt::get(IGM.Int32Ty, i->getCounterIndex())
+  };
+  Builder.CreateIntrinsicCall(llvm::Intrinsic::instrprof_increment, args);
+}
+
 void IRGenSILFunction::visitSuperMethodInst(swift::SuperMethodInst *i) {
   assert(!i->getMember().isForeign);
 

lib/SIL/IR/OperandOwnership.cpp

@@ -119,6 +119,7 @@ SHOULD_NEVER_VISIT_INST(ReleaseValue)
 SHOULD_NEVER_VISIT_INST(ReleaseValueAddr)
 SHOULD_NEVER_VISIT_INST(StrongRelease)
 SHOULD_NEVER_VISIT_INST(GetAsyncContinuation)
+SHOULD_NEVER_VISIT_INST(IncrementProfilerCounter)
 
 #define ALWAYS_OR_SOMETIMES_LOADABLE_CHECKED_REF_STORAGE(Name, ...)            \
   SHOULD_NEVER_VISIT_INST(StrongRetain##Name)                                  \
@@ -794,7 +795,6 @@ BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, PoundAssert)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, GlobalStringTablePointer)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, TypePtrAuthDiscriminator)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, TargetOSVersionAtLeast)
-BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, IntInstrprofIncrement)
 BUILTIN_OPERAND_OWNERSHIP(UnownedInstantaneousUse, Copy)
 BUILTIN_OPERAND_OWNERSHIP(DestroyingConsume, StartAsyncLet)
 BUILTIN_OPERAND_OWNERSHIP(DestroyingConsume, EndAsyncLet)

lib/SIL/IR/SILInstructions.cpp

@@ -438,6 +438,22 @@ BuiltinInst::BuiltinInst(SILDebugLocation Loc, Identifier Name,
       Substitutions(Subs) {
 }
 
+IncrementProfilerCounterInst *IncrementProfilerCounterInst::create(
+    SILDebugLocation Loc, unsigned CounterIdx, StringRef PGOFuncName,
+    unsigned NumCounters, uint64_t PGOFuncHash, SILModule &M) {
+
+  auto PGOFuncNameLength = PGOFuncName.size();
+  auto Size = totalSizeToAlloc<char>(PGOFuncNameLength);
+  auto Buffer = M.allocateInst(Size, alignof(IncrementProfilerCounterInst));
+
+  auto *Inst = ::new (Buffer) IncrementProfilerCounterInst(
+      Loc, CounterIdx, PGOFuncNameLength, NumCounters, PGOFuncHash);
+
+  std::uninitialized_copy(PGOFuncName.begin(), PGOFuncName.end(),
+                          Inst->getTrailingObjects<char>());
+  return Inst;
+}
+
 InitBlockStorageHeaderInst *
 InitBlockStorageHeaderInst::create(SILFunction &F,
                                SILDebugLocation DebugLoc, SILValue BlockStorage,

lib/SIL/IR/SILPrinter.cpp

@@ -2359,7 +2359,14 @@ class SILPrinter : public SILInstructionVisitor<SILPrinter> {
     *this << getIDAndType(FI->getOperand()) << ", "
           << QuotedString(FI->getMessage());
   }
-  
+
+  void visitIncrementProfilerCounterInst(IncrementProfilerCounterInst *IPCI) {
+    *this << IPCI->getCounterIndex() << ", "
+          << QuotedString(IPCI->getPGOFuncName()) << ", "
+          << "num_counters " << IPCI->getNumCounters() << ", "
+          << "hash " << IPCI->getPGOFuncHash();
+  }
+
   void visitIndexAddrInst(IndexAddrInst *IAI) {
     *this << getIDAndType(IAI->getBase()) << ", "
           << getIDAndType(IAI->getIndex());

lib/SIL/IR/ValueOwnership.cpp

@@ -532,7 +532,6 @@ CONSTANT_OWNERSHIP_BUILTIN(None, TSanInoutAccess)
 CONSTANT_OWNERSHIP_BUILTIN(None, Swift3ImplicitObjCEntrypoint)
 CONSTANT_OWNERSHIP_BUILTIN(None, PoundAssert)
 CONSTANT_OWNERSHIP_BUILTIN(None, TypePtrAuthDiscriminator)
-CONSTANT_OWNERSHIP_BUILTIN(None, IntInstrprofIncrement)
 CONSTANT_OWNERSHIP_BUILTIN(None, TargetOSVersionAtLeast)
 CONSTANT_OWNERSHIP_BUILTIN(None, GlobalStringTablePointer)
 CONSTANT_OWNERSHIP_BUILTIN(None, GetCurrentAsyncTask)