Merge branch 'main' into memrefops/add-memory-effect-to-assume-aligned

Author: shay-kl

.github/workflows/containers/github-action-ci-windows/Dockerfile

@@ -108,7 +108,7 @@ RUN choco install -y handle
 
 RUN pip3 install pywin32 buildbot-worker==2.8.4
 
-ARG RUNNER_VERSION=2.323.0
+ARG RUNNER_VERSION=2.324.0
 ENV RUNNER_VERSION=$RUNNER_VERSION
 
 RUN powershell -Command \

.github/workflows/containers/github-action-ci/Dockerfile

@@ -2,7 +2,7 @@ FROM docker.io/library/ubuntu:24.04 as base
 ENV LLVM_SYSROOT=/opt/llvm
 
 FROM base as stage1-toolchain
-ENV LLVM_VERSION=20.1.1
+ENV LLVM_VERSION=20.1.4
 
 RUN apt-get update && \
     apt-get install -y \
@@ -86,7 +86,7 @@ WORKDIR /home/gha
 
 FROM ci-container as ci-container-agent
 
-ENV GITHUB_RUNNER_VERSION=2.323.0
+ENV GITHUB_RUNNER_VERSION=2.324.0
 
 RUN mkdir actions-runner && \
     cd actions-runner && \

bolt/include/bolt/Core/BinaryFunction.h

@@ -142,8 +142,8 @@ class BinaryFunction {
   /// Types of profile the function can use. Could be a combination.
   enum {
     PF_NONE = 0,     /// No profile.
-    PF_LBR = 1,      /// Profile is based on last branch records.
-    PF_SAMPLE = 2,   /// Non-LBR sample-based profile.
+    PF_BRANCH = 1,   /// Profile is based on branches or branch stacks.
+    PF_BASIC = 2,    /// Non-branch IP sample-based profile.
     PF_MEMEVENT = 4, /// Profile has mem events.
   };
 
@@ -392,7 +392,7 @@ class BinaryFunction {
   float ProfileMatchRatio{0.0f};
 
   /// Raw branch count for this function in the profile.
-  uint64_t RawBranchCount{0};
+  uint64_t RawSampleCount{0};
 
   /// Dynamically executed function bytes, used for density computation.
   uint64_t SampleCountInBytes{0};
@@ -1893,11 +1893,11 @@ class BinaryFunction {
 
   /// Return the raw profile information about the number of branch
   /// executions corresponding to this function.
-  uint64_t getRawBranchCount() const { return RawBranchCount; }
+  uint64_t getRawSampleCount() const { return RawSampleCount; }
 
   /// Set the profile data about the number of branch executions corresponding
   /// to this function.
-  void setRawBranchCount(uint64_t Count) { RawBranchCount = Count; }
+  void setRawSampleCount(uint64_t Count) { RawSampleCount = Count; }
 
   /// Return the number of dynamically executed bytes, from raw perf data.
   uint64_t getSampleCountInBytes() const { return SampleCountInBytes; }

bolt/include/bolt/Profile/DataAggregator.h

@@ -92,16 +92,6 @@ class DataAggregator : public DataReader {
     uint64_t Addr;
   };
 
-  /// Used for parsing specific pre-aggregated input files.
-  struct AggregatedLBREntry {
-    enum Type : char { BRANCH = 0, FT, FT_EXTERNAL_ORIGIN, TRACE };
-    Location From;
-    Location To;
-    uint64_t Count;
-    uint64_t Mispreds;
-    Type EntryType;
-  };
-
   struct Trace {
     uint64_t From;
     uint64_t To;
@@ -131,7 +121,6 @@ class DataAggregator : public DataReader {
   /// and use them later for processing and assigning profile.
   std::unordered_map<Trace, TakenBranchInfo, TraceHash> BranchLBRs;
   std::unordered_map<Trace, FTInfo, TraceHash> FallthroughLBRs;
-  std::vector<AggregatedLBREntry> AggregatedLBRs;
   std::unordered_map<uint64_t, uint64_t> BasicSamples;
   std::vector<PerfMemSample> MemSamples;
 
@@ -223,11 +212,6 @@ class DataAggregator : public DataReader {
   uint64_t NumTraces{0};
   uint64_t NumInvalidTraces{0};
   uint64_t NumLongRangeTraces{0};
-  /// Specifies how many samples were recorded in cold areas if we are dealing
-  /// with profiling data collected in a bolted binary. For LBRs, incremented
-  /// for the source of the branch to avoid counting cold activity twice (one
-  /// for source and another for destination).
-  uint64_t NumColdSamples{0};
   uint64_t NumTotalSamples{0};
 
   /// Looks into system PATH for Linux Perf and set up the aggregator to use it
@@ -257,7 +241,8 @@ class DataAggregator : public DataReader {
 
   /// Semantic actions - parser hooks to interpret parsed perf samples
   /// Register a sample (non-LBR mode), i.e. a new hit at \p Address
-  bool doSample(BinaryFunction &Func, const uint64_t Address, uint64_t Count);
+  bool doBasicSample(BinaryFunction &Func, const uint64_t Address,
+                     uint64_t Count);
 
   /// Register an intraprocedural branch \p Branch.
   bool doIntraBranch(BinaryFunction &Func, uint64_t From, uint64_t To,
@@ -422,9 +407,6 @@ class DataAggregator : public DataReader {
   /// an external tool.
   std::error_code parsePreAggregatedLBRSamples();
 
-  /// Process parsed pre-aggregated data.
-  void processPreAggregated();
-
   /// If \p Address falls into the binary address space based on memory
   /// mapping info \p MMI, then adjust it for further processing by subtracting
   /// the base load address. External addresses, i.e. addresses that do not
@@ -486,7 +468,6 @@ class DataAggregator : public DataReader {
   void dump(const PerfMemSample &Sample) const;
 
   /// Profile diagnostics print methods
-  void printColdSamplesDiagnostic() const;
   void printLongRangeTracesDiagnostic() const;
   void printBranchSamplesDiagnostics() const;
   void printBasicSamplesDiagnostics(uint64_t OutOfRangeSamples) const;

bolt/include/bolt/Profile/DataReader.h

@@ -212,15 +212,16 @@ struct FuncMemData {
 /// Similar to BranchInfo, but instead of recording from-to address (an edge),
 /// it records the address of a perf event and the number of times samples hit
 /// this address.
-struct SampleInfo {
+struct BasicSampleInfo {
   Location Loc;
   int64_t Hits;
 
-  SampleInfo(Location Loc, int64_t Hits) : Loc(std::move(Loc)), Hits(Hits) {}
+  BasicSampleInfo(Location Loc, int64_t Hits)
+      : Loc(std::move(Loc)), Hits(Hits) {}
 
-  bool operator==(const SampleInfo &RHS) const { return Loc == RHS.Loc; }
+  bool operator==(const BasicSampleInfo &RHS) const { return Loc == RHS.Loc; }
 
-  bool operator<(const SampleInfo &RHS) const {
+  bool operator<(const BasicSampleInfo &RHS) const {
     if (Loc < RHS.Loc)
       return true;
 
@@ -229,18 +230,18 @@ struct SampleInfo {
 
   void print(raw_ostream &OS) const;
 
-  void mergeWith(const SampleInfo &SI);
+  void mergeWith(const BasicSampleInfo &SI);
 };
 
 /// Helper class to store samples recorded in the address space of a given
 /// function, analogous to FuncBranchData but for samples instead of branches.
-struct FuncSampleData {
-  typedef std::vector<SampleInfo> ContainerTy;
+struct FuncBasicSampleData {
+  typedef std::vector<BasicSampleInfo> ContainerTy;
 
   StringRef Name;
   ContainerTy Data;
 
-  FuncSampleData(StringRef Name, ContainerTy Data)
+  FuncBasicSampleData(StringRef Name, ContainerTy Data)
       : Name(Name), Data(std::move(Data)) {}
 
   /// Get the number of samples recorded in [Start, End)
@@ -308,7 +309,7 @@ class DataReader : public ProfileReaderBase {
   /// The last step is to infer edge counts based on BB execution count. Note
   /// this is the opposite of the LBR way, where we infer BB execution count
   /// based on edge counts.
-  void readSampleData(BinaryFunction &BF);
+  void readBasicSampleData(BinaryFunction &BF);
 
   /// Convert function-level branch data into instruction annotations.
   void convertBranchData(BinaryFunction &BF) const;
@@ -382,7 +383,8 @@ class DataReader : public ProfileReaderBase {
   /// Return mem data matching one of the names in \p FuncNames.
   FuncMemData *getMemDataForNames(const std::vector<StringRef> &FuncNames);
 
-  FuncSampleData *getFuncSampleData(const std::vector<StringRef> &FuncNames);
+  FuncBasicSampleData *
+  getFuncBasicSampleData(const std::vector<StringRef> &FuncNames);
 
   /// Return a vector of all FuncBranchData matching the list of names.
   /// Internally use fuzzy matching to match special names like LTO-generated
@@ -425,7 +427,7 @@ class DataReader : public ProfileReaderBase {
   }
 
   using NamesToBranchesMapTy = std::map<StringRef, FuncBranchData>;
-  using NamesToSamplesMapTy = std::map<StringRef, FuncSampleData>;
+  using NamesToBasicSamplesMapTy = std::map<StringRef, FuncBasicSampleData>;
   using NamesToMemEventsMapTy = std::map<StringRef, FuncMemData>;
   using FuncsToBranchesMapTy =
       std::unordered_map<const BinaryFunction *, FuncBranchData *>;
@@ -474,7 +476,7 @@ class DataReader : public ProfileReaderBase {
     return parseLocation(EndChar, EndNl, true);
   }
   ErrorOr<BranchInfo> parseBranchInfo();
-  ErrorOr<SampleInfo> parseSampleInfo();
+  ErrorOr<BasicSampleInfo> parseSampleInfo();
   ErrorOr<MemInfo> parseMemInfo();
   ErrorOr<bool> maybeParseNoLBRFlag();
   ErrorOr<bool> maybeParseBATFlag();
@@ -488,7 +490,7 @@ class DataReader : public ProfileReaderBase {
   unsigned Line{0};
   unsigned Col{0};
   NamesToBranchesMapTy NamesToBranches;
-  NamesToSamplesMapTy NamesToSamples;
+  NamesToBasicSamplesMapTy NamesToBasicSamples;
   NamesToMemEventsMapTy NamesToMemEvents;
   FuncsToBranchesMapTy FuncsToBranches;
   FuncsToMemDataMapTy FuncsToMemData;

bolt/include/bolt/Profile/Heatmap.h

@@ -52,14 +52,17 @@ class Heatmap {
       : BucketSize(BucketSize), MinAddress(MinAddress), MaxAddress(MaxAddress),
         TextSections(TextSections) {}
 
+  uint64_t HotStart{0};
+  uint64_t HotEnd{0};
+
   inline bool ignoreAddress(uint64_t Address) const {
     return (Address > MaxAddress) || (Address < MinAddress);
   }
 
   /// Register a single sample at \p Address.
-  void registerAddress(uint64_t Address) {
+  void registerAddress(uint64_t Address, uint64_t Count) {
     if (!ignoreAddress(Address))
-      ++Map[Address / BucketSize];
+      Map[Address / BucketSize] += Count;
   }
 
   /// Register \p Count samples at [\p StartAddress, \p EndAddress ].

bolt/include/bolt/Profile/ProfileYAMLMapping.h

@@ -230,8 +230,8 @@ LLVM_YAML_STRONG_TYPEDEF(uint16_t, PROFILE_PF)
 
 template <> struct ScalarBitSetTraits<PROFILE_PF> {
   static void bitset(IO &io, PROFILE_PF &value) {
-    io.bitSetCase(value, "lbr", BinaryFunction::PF_LBR);
-    io.bitSetCase(value, "sample", BinaryFunction::PF_SAMPLE);
+    io.bitSetCase(value, "lbr", BinaryFunction::PF_BRANCH);
+    io.bitSetCase(value, "sample", BinaryFunction::PF_BASIC);
     io.bitSetCase(value, "memevent", BinaryFunction::PF_MEMEVENT);
   }
 };

bolt/include/bolt/Utils/CommandLineOpts.h

@@ -17,7 +17,13 @@
 
 namespace opts {
 
-extern bool HeatmapMode;
+enum HeatmapModeKind {
+  HM_None = 0,
+  HM_Exclusive, // llvm-bolt-heatmap
+  HM_Optional   // perf2bolt --heatmap
+};
+
+extern HeatmapModeKind HeatmapMode;
 extern bool BinaryAnalysisMode;
 
 extern llvm::cl::OptionCategory BoltCategory;
@@ -45,6 +51,7 @@ extern llvm::cl::opt<unsigned> HeatmapBlock;
 extern llvm::cl::opt<unsigned long long> HeatmapMaxAddress;
 extern llvm::cl::opt<unsigned long long> HeatmapMinAddress;
 extern llvm::cl::opt<bool> HeatmapPrintMappings;
+extern llvm::cl::opt<std::string> HeatmapOutput;
 extern llvm::cl::opt<bool> HotData;
 extern llvm::cl::opt<bool> HotFunctionsAtEnd;
 extern llvm::cl::opt<bool> HotText;

bolt/lib/Core/BinaryFunction.cpp

@@ -66,7 +66,7 @@ extern cl::opt<bool> UpdateDebugSections;
 extern cl::opt<unsigned> Verbosity;
 
 extern bool BinaryAnalysisMode;
-extern bool HeatmapMode;
+extern HeatmapModeKind HeatmapMode;
 extern bool processAllFunctions();
 
 static cl::opt<bool> CheckEncoding(
@@ -473,7 +473,7 @@ void BinaryFunction::print(raw_ostream &OS, std::string Annotation) {
     OS << "\n  Image       : 0x" << Twine::utohexstr(getImageAddress());
   if (ExecutionCount != COUNT_NO_PROFILE) {
     OS << "\n  Exec Count  : " << ExecutionCount;
-    OS << "\n  Branch Count: " << RawBranchCount;
+    OS << "\n  Sample Count: " << RawSampleCount;
     OS << "\n  Profile Acc : " << format("%.1f%%", ProfileMatchRatio * 100.0f);
   }
 

bolt/lib/Core/BinaryFunctionProfile.cpp

@@ -70,7 +70,7 @@ void BinaryFunction::postProcessProfile() {
     return;
   }
 
-  if (!(getProfileFlags() & PF_LBR))
+  if (!(getProfileFlags() & PF_BRANCH))
     return;
 
   // If we have at least some branch data for the function indicate that it

bolt/lib/Passes/BinaryPasses.cpp

@@ -1445,7 +1445,7 @@ Error PrintProgramStats::runOnFunctions(BinaryContext &BC) {
     if (!Function.hasProfile())
       continue;
 
-    uint64_t SampleCount = Function.getRawBranchCount();
+    uint64_t SampleCount = Function.getRawSampleCount();
     TotalSampleCount += SampleCount;
 
     if (Function.hasValidProfile()) {

bolt/lib/Passes/MCF.cpp

@@ -458,15 +458,15 @@ void EstimateEdgeCounts::runOnFunction(BinaryFunction &BF) {
 Error EstimateEdgeCounts::runOnFunctions(BinaryContext &BC) {
   if (llvm::none_of(llvm::make_second_range(BC.getBinaryFunctions()),
                     [](const BinaryFunction &BF) {
-                      return BF.getProfileFlags() == BinaryFunction::PF_SAMPLE;
+                      return BF.getProfileFlags() == BinaryFunction::PF_BASIC;
                     }))
     return Error::success();
 
   ParallelUtilities::WorkFuncTy WorkFun = [&](BinaryFunction &BF) {
     runOnFunction(BF);
   };
   ParallelUtilities::PredicateTy SkipFunc = [&](const BinaryFunction &BF) {
-    return BF.getProfileFlags() != BinaryFunction::PF_SAMPLE;
+    return BF.getProfileFlags() != BinaryFunction::PF_BASIC;
   };
 
   ParallelUtilities::runOnEachFunction(