Manual merge remote-tracking branch 'llvm.org/main' into amd-gfx

Author: dstutt

bolt/docs/CommandLineArgumentReference.md

@@ -688,6 +688,10 @@
 
   Use a modified clustering algorithm geared towards minimizing branches
 
+- `--name-similarity-function-matching-threshold=<uint>`
+
+  Match functions using namespace and edit distance.
+
 - `--no-inline`
 
   Disable all inlining (overrides other inlining options)

bolt/docs/OptimizingLinux.md

@@ -44,6 +44,11 @@ $ perf2bolt -p perf.data -o perf.fdata vmlinux
 
 Under a high load, `perf.data` should be several gigabytes in size and you should expect the converted `perf.fdata` not to exceed 100 MB.
 
+Profiles collected from multiple workloads could be joined into a single profile using `merge-fdata` utility:
+```bash
+$ merge-fdata perf.1.fdata perf.2.fdata ... perf.<N>.fdata > perf.merged.fdata
+```
+
 Two changes are required for the kernel build. The first one is optional but highly recommended. It introduces a BOLT-reserved space into `vmlinux` code section:
 
 

bolt/include/bolt/Core/DebugData.h

@@ -210,6 +210,15 @@ class DebugRangesSectionWriter {
   static bool classof(const DebugRangesSectionWriter *Writer) {
     return Writer->getKind() == RangesWriterKind::DebugRangesWriter;
   }
+  
+  /// Append a range to the main buffer.
+  void appendToRangeBuffer(const DebugBufferVector &CUBuffer);
+
+  /// Sets Unit DIE to be updated for CU.
+  void setDie(DIE *Die) { this->Die = Die; }
+
+  /// Returns Unit DIE to be updated for CU.
+  DIE *getDie() const { return Die; }
 
   /// Writes out range lists for a current CU being processed.
   void virtual finalizeSection(){};
@@ -232,6 +241,9 @@ class DebugRangesSectionWriter {
   static constexpr uint64_t EmptyRangesOffset{0};
 
 private:
+  /// Stores Unit DIE to be updated for CU.
+  DIE *Die{0};
+
   RangesWriterKind Kind;
 };
 

bolt/include/bolt/Core/DebugNames.h

@@ -91,6 +91,10 @@ class DWARF5AcceleratorTable {
   uint64_t CurrentUnitOffset = 0;
   const DWARFUnit *CurrentUnit = nullptr;
   std::unordered_map<uint32_t, uint32_t> AbbrevTagToIndexMap;
+  /// Contains a map of TU hashes to a Foreign TU indecies.
+  /// This is used to reduce the size of Foreign TU list since there could be
+  /// multiple TUs with the same hash.
+  DenseMap<uint64_t, uint32_t> TUHashToIndexMap;
 
   /// Represents a group of entries with identical name (and hence, hash value).
   struct HashData {

bolt/include/bolt/Profile/YAMLProfileReader.h

@@ -93,6 +93,9 @@ class YAMLProfileReader : public ProfileReaderBase {
     ProfiledFunctions.emplace(&BF);
   }
 
+  /// Matches functions with similarly named profiled functions.
+  uint64_t matchWithNameSimilarity(BinaryContext &BC);
+
   /// Check if the profile uses an event with a given \p Name.
   bool usesEvent(StringRef Name) const;
 };

bolt/include/bolt/Rewrite/DWARFRewriter.h

@@ -89,6 +89,10 @@ class DWARFRewriter {
   /// Store Rangelists writer for each DWO CU.
   RangeListsDWOWriers RangeListsWritersByCU;
 
+  /// Stores ranges writer for each DWO CU.
+  std::unordered_map<uint64_t, std::unique_ptr<DebugRangesSectionWriter>>
+      LegacyRangesWritersByCU;
+
   std::mutex LocListDebugInfoPatchesMutex;
 
   /// Dwo id specific its RangesBase.

bolt/lib/Core/BinaryContext.cpp

@@ -2403,16 +2403,8 @@ BinaryContext::calculateEmittedSize(BinaryFunction &BF, bool FixBranches) {
     Streamer->emitLabel(SplitStartLabel);
     emitFunctionBody(*Streamer, BF, FF, /*EmitCodeOnly=*/true);
     Streamer->emitLabel(SplitEndLabel);
-    // To avoid calling MCObjectStreamer::flushPendingLabels() which is
-    // private
-    Streamer->emitBytes(StringRef(""));
-    Streamer->switchSection(Section);
   }
 
-  // To avoid calling MCObjectStreamer::flushPendingLabels() which is private or
-  // MCStreamer::Finish(), which does more than we want
-  Streamer->emitBytes(StringRef(""));
-
   MCAssembler &Assembler =
       static_cast<MCObjectStreamer *>(Streamer.get())->getAssembler();
   Assembler.layout();

bolt/lib/Core/DebugData.cpp

@@ -177,6 +177,12 @@ uint64_t DebugRangesSectionWriter::getSectionOffset() {
   return SectionOffset;
 }
 
+void DebugRangesSectionWriter::appendToRangeBuffer(
+    const DebugBufferVector &CUBuffer) {
+  *RangesStream << CUBuffer;
+  SectionOffset = RangesBuffer->size();
+}
+
 DebugAddrWriter *DebugRangeListsSectionWriter::AddrWriter = nullptr;
 
 uint64_t DebugRangeListsSectionWriter::addRanges(

bolt/lib/Core/DebugNames.cpp

@@ -90,7 +90,11 @@ void DWARF5AcceleratorTable::addUnit(DWARFUnit &Unit,
       auto Iter = CUOffsetsToPatch.insert({*DWOID, CUList.size()});
       if (Iter.second)
         CUList.push_back(BADCUOFFSET);
-      ForeignTUList.push_back(cast<DWARFTypeUnit>(&Unit)->getTypeHash());
+      const uint64_t TUHash = cast<DWARFTypeUnit>(&Unit)->getTypeHash();
+      if (!TUHashToIndexMap.count(TUHash)) {
+        TUHashToIndexMap.insert({TUHash, ForeignTUList.size()});
+        ForeignTUList.push_back(TUHash);
+      }
     } else {
       LocalTUList.push_back(CurrentUnitOffset);
     }
@@ -231,8 +235,13 @@ DWARF5AcceleratorTable::addAccelTableEntry(
     IsTU = Unit.isTypeUnit();
     DieTag = Die.getTag();
     if (IsTU) {
-      if (DWOID)
-        return ForeignTUList.size() - 1;
+      if (DWOID) {
+        const uint64_t TUHash = cast<DWARFTypeUnit>(&Unit)->getTypeHash();
+        auto Iter = TUHashToIndexMap.find(TUHash);
+        assert(Iter != TUHashToIndexMap.end() &&
+               "Could not find TU hash in map");
+        return Iter->second;
+      }
       return LocalTUList.size() - 1;
     }
     return CUList.size() - 1;

bolt/lib/Profile/CMakeLists.txt

@@ -10,6 +10,7 @@ add_llvm_library(LLVMBOLTProfile
   DISABLE_LLVM_LINK_LLVM_DYLIB
 
   LINK_COMPONENTS
+  Demangle
   Support
   TransformUtils
   )

bolt/lib/Profile/YAMLProfileReader.cpp

@@ -11,8 +11,11 @@
 #include "bolt/Core/BinaryFunction.h"
 #include "bolt/Passes/MCF.h"
 #include "bolt/Profile/ProfileYAMLMapping.h"
+#include "bolt/Utils/NameResolver.h"
 #include "bolt/Utils/Utils.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/edit_distance.h"
+#include "llvm/Demangle/Demangle.h"
 #include "llvm/Support/CommandLine.h"
 
 using namespace llvm;
@@ -24,6 +27,11 @@ extern cl::OptionCategory BoltOptCategory;
 extern cl::opt<bool> InferStaleProfile;
 extern cl::opt<bool> Lite;
 
+cl::opt<unsigned> NameSimilarityFunctionMatchingThreshold(
+    "name-similarity-function-matching-threshold",
+    cl::desc("Match functions using namespace and edit distance"), cl::init(0),
+    cl::Hidden, cl::cat(BoltOptCategory));
+
 static llvm::cl::opt<bool>
     IgnoreHash("profile-ignore-hash",
                cl::desc("ignore hash while reading function profile"),
@@ -350,6 +358,111 @@ bool YAMLProfileReader::mayHaveProfileData(const BinaryFunction &BF) {
   return false;
 }
 
+uint64_t YAMLProfileReader::matchWithNameSimilarity(BinaryContext &BC) {
+  uint64_t MatchedWithNameSimilarity = 0;
+  ItaniumPartialDemangler Demangler;
+
+  // Demangle and derive namespace from function name.
+  auto DemangleName = [&](std::string &FunctionName) {
+    StringRef RestoredName = NameResolver::restore(FunctionName);
+    return demangle(RestoredName);
+  };
+  auto DeriveNameSpace = [&](std::string &DemangledName) {
+    if (Demangler.partialDemangle(DemangledName.c_str()))
+      return std::string("");
+    std::vector<char> Buffer(DemangledName.begin(), DemangledName.end());
+    size_t BufferSize;
+    char *NameSpace =
+        Demangler.getFunctionDeclContextName(&Buffer[0], &BufferSize);
+    return std::string(NameSpace, BufferSize);
+  };
+
+  // Maps namespaces to associated function block counts and gets profile
+  // function names and namespaces to minimize the number of BFs to process and
+  // avoid repeated name demangling/namespace derivation.
+  StringMap<std::set<uint32_t>> NamespaceToProfiledBFSizes;
+  std::vector<std::string> ProfileBFDemangledNames;
+  ProfileBFDemangledNames.reserve(YamlBP.Functions.size());
+  std::vector<std::string> ProfiledBFNamespaces;
+  ProfiledBFNamespaces.reserve(YamlBP.Functions.size());
+
+  for (auto &YamlBF : YamlBP.Functions) {
+    std::string YamlBFDemangledName = DemangleName(YamlBF.Name);
+    ProfileBFDemangledNames.push_back(YamlBFDemangledName);
+    std::string YamlBFNamespace = DeriveNameSpace(YamlBFDemangledName);
+    ProfiledBFNamespaces.push_back(YamlBFNamespace);
+    NamespaceToProfiledBFSizes[YamlBFNamespace].insert(YamlBF.NumBasicBlocks);
+  }
+
+  StringMap<std::vector<BinaryFunction *>> NamespaceToBFs;
+
+  // Maps namespaces to BFs excluding binary functions with no equal sized
+  // profiled functions belonging to the same namespace.
+  for (BinaryFunction *BF : BC.getAllBinaryFunctions()) {
+    std::string DemangledName = BF->getDemangledName();
+    std::string Namespace = DeriveNameSpace(DemangledName);
+
+    auto NamespaceToProfiledBFSizesIt =
+        NamespaceToProfiledBFSizes.find(Namespace);
+    // Skip if there are no ProfileBFs with a given \p Namespace.
+    if (NamespaceToProfiledBFSizesIt == NamespaceToProfiledBFSizes.end())
+      continue;
+    // Skip if there are no ProfileBFs in a given \p Namespace with
+    // equal number of blocks.
+    if (NamespaceToProfiledBFSizesIt->second.count(BF->size()) == 0)
+      continue;
+    auto NamespaceToBFsIt = NamespaceToBFs.find(Namespace);
+    if (NamespaceToBFsIt == NamespaceToBFs.end())
+      NamespaceToBFs[Namespace] = {BF};
+    else
+      NamespaceToBFsIt->second.push_back(BF);
+  }
+
+  // Iterates through all profiled functions and binary functions belonging to
+  // the same namespace and matches based on edit distance threshold.
+  assert(YamlBP.Functions.size() == ProfiledBFNamespaces.size() &&
+         ProfiledBFNamespaces.size() == ProfileBFDemangledNames.size());
+  for (size_t I = 0; I < YamlBP.Functions.size(); ++I) {
+    yaml::bolt::BinaryFunctionProfile &YamlBF = YamlBP.Functions[I];
+    std::string &YamlBFNamespace = ProfiledBFNamespaces[I];
+    if (YamlBF.Used)
+      continue;
+    // Skip if there are no BFs in a given \p Namespace.
+    auto It = NamespaceToBFs.find(YamlBFNamespace);
+    if (It == NamespaceToBFs.end())
+      continue;
+
+    std::string &YamlBFDemangledName = ProfileBFDemangledNames[I];
+    std::vector<BinaryFunction *> BFs = It->second;
+    unsigned MinEditDistance = UINT_MAX;
+    BinaryFunction *ClosestNameBF = nullptr;
+
+    // Determines BF the closest to the profiled function, in the
+    // same namespace.
+    for (BinaryFunction *BF : BFs) {
+      if (ProfiledFunctions.count(BF))
+        continue;
+      if (BF->size() != YamlBF.NumBasicBlocks)
+        continue;
+      std::string BFDemangledName = BF->getDemangledName();
+      unsigned BFEditDistance =
+          StringRef(BFDemangledName).edit_distance(YamlBFDemangledName);
+      if (BFEditDistance < MinEditDistance) {
+        MinEditDistance = BFEditDistance;
+        ClosestNameBF = BF;
+      }
+    }
+
+    if (ClosestNameBF &&
+        MinEditDistance <= opts::NameSimilarityFunctionMatchingThreshold) {
+      matchProfileToFunction(YamlBF, *ClosestNameBF);
+      ++MatchedWithNameSimilarity;
+    }
+  }
+
+  return MatchedWithNameSimilarity;
+}
+
 Error YAMLProfileReader::readProfile(BinaryContext &BC) {
   if (opts::Verbosity >= 1) {
     outs() << "BOLT-INFO: YAML profile with hash: ";
@@ -461,6 +574,12 @@ Error YAMLProfileReader::readProfile(BinaryContext &BC) {
     if (!YamlBF.Used && BF && !ProfiledFunctions.count(BF))
       matchProfileToFunction(YamlBF, *BF);
 
+  // Uses name similarity to match functions that were not matched by name.
+  uint64_t MatchedWithNameSimilarity =
+      opts::NameSimilarityFunctionMatchingThreshold > 0
+          ? matchWithNameSimilarity(BC)
+          : 0;
+
   for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions)
     if (!YamlBF.Used && opts::Verbosity >= 1)
       errs() << "BOLT-WARNING: profile ignored for function " << YamlBF.Name
@@ -473,6 +592,8 @@ Error YAMLProfileReader::readProfile(BinaryContext &BC) {
            << " functions with hash\n";
     outs() << "BOLT-INFO: matched " << MatchedWithLTOCommonName
            << " functions with matching LTO common names\n";
+    outs() << "BOLT-INFO: matched " << MatchedWithNameSimilarity
+           << " functions with similar names\n";
   }
 
   // Set for parseFunctionProfile().