[𝘀𝗽𝗿] changes to main this commit is based on

Created using spr 1.3.4

[skip ci]

Author: aaupov

bolt/docs/BAT.md

@@ -0,0 +1,102 @@
+# BOLT Address Translation (BAT)
+# Purpose
+A regular profile collection for BOLT involves collecting samples from
+unoptimized binary. BOLT Address Translation allows collecting profile
+from BOLT-optimized binary and using it for optimizing the input (pre-BOLT)
+binary.
+
+# Overview
+BOLT Address Translation is an extra section (`.note.bolt_bat`) inserted by BOLT
+into the output binary containing translation tables and split functions linkage
+information. This information enables mapping the profile back from optimized
+binary onto the original binary.
+
+# Usage
+`--enable-bat` flag controls the generation of BAT section. Sampled profile 
+needs to be passed along with the optimized binary containing BAT section to
+`perf2bolt` which reads BAT section and produces fdata profile for the original
+binary. Note that YAML profile generation is not supported since BAT doesn't
+contain the metadata for input functions.
+
+# Internals
+## Section contents
+The section is organized as follows:
+- Hot functions table
+  - Address translation tables
+- Cold functions table
+
+## Construction and parsing
+BAT section is created from `BoltAddressTranslation` class which captures
+address translation information provided by BOLT linker. It is then encoded as a
+note section in the output binary.
+
+During profile conversion when BAT-enabled binary is passed to perf2bolt, 
+`BoltAddressTranslation` class is populated from BAT section. The class is then
+queried by `DataAggregator` during sample processing to reconstruct addresses/
+offsets in the input binary.
+
+## Encoding format
+The encoding is specified in bolt/include/bolt/Profile/BoltAddressTranslation.h
+and bolt/lib/Profile/BoltAddressTranslation.cpp.
+
+### Layout
+The general layout is as follows:
+```
+Hot functions table header
+|------------------|
+|  Function entry  |
+| |--------------| |
+| | OutOff InOff | |
+| |--------------| |
+~~~~~~~~~~~~~~~~~~~~
+
+Cold functions table header
+|------------------|
+|  Function entry  |
+| |--------------| |
+| | OutOff InOff | |
+| |--------------| |
+~~~~~~~~~~~~~~~~~~~~
+```
+
+### Functions table
+Hot and cold functions tables share the encoding except differences marked below.
+Header:
+| Entry  | Encoding | Description |
+| ------ | ----- | ----------- |
+| `NumFuncs` | ULEB128 | Number of functions in the functions table |
+
+The header is followed by Functions table with `NumFuncs` entries.
+Output binary addresses are delta encoded, meaning that only the difference with
+the last previous output address is stored. Addresses implicitly start at zero.
+Output addresses are continuous through function start addresses and function
+internal offsets, and between hot and cold fragments, to better spread deltas
+and save space.
+
+Hot indices are delta encoded, implicitly starting at zero.
+| Entry  | Encoding | Description |
+| ------ | ------| ----------- |
+| `Address` | Continuous, Delta, ULEB128 | Function address in the output binary |
+| `HotIndex` | Delta, ULEB128 | Cold functions only: index of corresponding hot function in hot functions table |
+| `FuncHash` | 8b | Hot functions only: function hash for input function |
+| `NumEntries` | ULEB128 | Number of address translation entries for a function |
+| `EqualElems` | ULEB128 | Hot functions only: number of equal offsets in the beginning of a function |
+| `BranchEntries` | Bitmask, `alignTo(EqualElems, 8)` bits | Hot functions only: if `EqualElems` is non-zero, bitmask denoting entries with `BRANCHENTRY` bit |
+Function header is followed by `EqualElems` offsets (hot functions only) and
+`NumEntries-EqualElems` (`NumEntries` for cold functions) pairs of offsets for
+current function.
+
+### Address translation table
+Delta encoding means that only the difference with the previous corresponding
+entry is encoded. Input offsets implicitly start at zero.
+| Entry  | Encoding | Description |
+| ------ | ------| ----------- |
+| `OutputAddr` | Continuous, Delta, ULEB128 | Function offset in output binary |
+| `InputAddr` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit |
+| `BBHash` | Optional, 8b | Basic block entries only: basic block hash in input binary |
+
+`BRANCHENTRY` bit denotes whether a given offset pair is a control flow source
+(branch or call instruction). If not set, it signifies a control flow target
+(basic block offset).
+`InputAddr` is omitted for equal offsets in input and output function. In this
+case, `BRANCHENTRY` bits are encoded separately in a `BranchEntries` bitvector.

bolt/include/bolt/Profile/BoltAddressTranslation.h

@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/DataExtractor.h"
 #include <cstdint>
 #include <map>
 #include <optional>
@@ -78,10 +79,21 @@ class BoltAddressTranslation {
 
   BoltAddressTranslation() {}
 
+  /// Write the serialized address translation table for a function.
+  template <bool Cold>
+  void writeMaps(std::map<uint64_t, MapTy> &Maps, uint64_t &PrevAddress,
+                 raw_ostream &OS);
+
   /// Write the serialized address translation tables for each reordered
   /// function
   void write(const BinaryContext &BC, raw_ostream &OS);
 
+  /// Read the serialized address translation table for a function.
+  /// Return a parse error if failed.
+  template <bool Cold>
+  void parseMaps(std::vector<uint64_t> &HotFuncs, uint64_t &PrevAddress,
+                 DataExtractor &DE, uint64_t &Offset, Error &Err);
+
   /// Read the serialized address translation tables and load them internally
   /// in memory. Return a parse error if failed.
   std::error_code parse(StringRef Buf);
@@ -110,22 +122,40 @@ class BoltAddressTranslation {
   /// addresses when aggregating profile
   bool enabledFor(llvm::object::ELFObjectFileBase *InputFile) const;
 
+  /// Save function and basic block hashes used for metadata dump.
+  void saveMetadata(BinaryContext &BC);
+
 private:
   /// Helper to update \p Map by inserting one or more BAT entries reflecting
   /// \p BB for function located at \p FuncAddress. At least one entry will be
   /// emitted for the start of the BB. More entries may be emitted to cover
   /// the location of calls or any instruction that may change control flow.
   void writeEntriesForBB(MapTy &Map, const BinaryBasicBlock &BB,
-                         uint64_t FuncAddress);
+                         uint64_t FuncAddress, uint64_t FuncInputAddress);
+
+  /// Returns the bitmask with set bits corresponding to indices of BRANCHENTRY
+  /// entries in function address translation map.
+  APInt calculateBranchEntriesBitMask(MapTy &Map, size_t EqualElems);
+
+  /// Calculate the number of equal offsets (output = input) in the beginning
+  /// of the function.
+  size_t getNumEqualOffsets(const MapTy &Map) const;
 
   std::map<uint64_t, MapTy> Maps;
+  std::map<uint64_t, MapTy> ColdMaps;
+
+  using BBHashMap = std::unordered_map<uint32_t, size_t>;
+  std::unordered_map<uint64_t, std::pair<size_t, BBHashMap>> FuncHashes;
 
   /// Links outlined cold bocks to their original function
   std::map<uint64_t, uint64_t> ColdPartSource;
 
+  /// Links output address of a main fragment back to input address.
+  std::unordered_map<uint64_t, uint64_t> ReverseMap;
+
   /// Identifies the address of a control-flow changing instructions in a
   /// translation map entry
-  const static uint32_t BRANCHENTRY = 0x80000000;
+  const static uint32_t BRANCHENTRY = 0x1;
 };
 } // namespace bolt
 

bolt/include/bolt/Profile/DataAggregator.h

@@ -225,6 +225,10 @@ class DataAggregator : public DataReader {
   /// Aggregation statistics
   uint64_t NumInvalidTraces{0};
   uint64_t NumLongRangeTraces{0};
+  /// Specifies how many samples were recorded in cold areas if we are dealing
+  /// with profiling data collected in a bolted binary. For LBRs, incremented
+  /// for the source of the branch to avoid counting cold activity twice (one
+  /// for source and another for destination).
   uint64_t NumColdSamples{0};
 
   /// Looks into system PATH for Linux Perf and set up the aggregator to use it
@@ -246,13 +250,9 @@ class DataAggregator : public DataReader {
   BinaryFunction *getBinaryFunctionContainingAddress(uint64_t Address) const;
 
   /// Retrieve the location name to be used for samples recorded in \p Func.
-  /// If doing BAT translation, link cold parts to the hot part  names (used by
-  /// the original binary).  \p Count specifies how many samples were recorded
-  /// at that location, so we can tally total activity in cold areas if we are
-  /// dealing with profiling data collected in a bolted binary. For LBRs,
-  /// \p Count should only be used for the source of the branch to avoid
-  /// counting cold activity twice (one for source and another for destination).
-  StringRef getLocationName(BinaryFunction &Func, uint64_t Count);
+  /// If doing BAT translation, link cold parts to the hot part names (used by
+  /// the original binary) and return true as second member.
+  std::pair<StringRef, bool> getLocationName(const BinaryFunction &Func) const;
 
   /// Semantic actions - parser hooks to interpret parsed perf samples
   /// Register a sample (non-LBR mode), i.e. a new hit at \p Address