LLVM and SPIRV-LLVM-Translator pulldown (WW39)

.github/workflows/version-check.py

@@ -12,7 +12,7 @@ def get_version_from_tag(tag):
             # We have an rc tag.
             return m.group(1,2,3)
         # We have a final release tag.
-        return (m.group(1), m.group(2), int(m.group(3)) + 1)
+        return (m.group(1), m.group(2), str(int(m.group(3)) + 1))
 
     m = re.match('llvmorg-([0-9]+)-init', tag)
     if m:

bolt/include/bolt/Core/BinaryContext.h

@@ -104,12 +104,9 @@ inline int64_t truncateToSize(int64_t Value, unsigned Bytes) {
 /// Filter iterator.
 template <typename ItrType,
           typename PredType = std::function<bool(const ItrType &)>>
-class FilterIterator
-    : public std::iterator<std::bidirectional_iterator_tag,
-                           typename std::iterator_traits<ItrType>::value_type> {
+class FilterIterator {
+  using inner_traits = std::iterator_traits<ItrType>;
   using Iterator = FilterIterator;
-  using T = typename std::iterator_traits<ItrType>::reference;
-  using PointerT = typename std::iterator_traits<ItrType>::pointer;
 
   PredType Pred;
   ItrType Itr, End;
@@ -128,14 +125,20 @@ class FilterIterator
   }
 
 public:
+  using iterator_category = std::bidirectional_iterator_tag;
+  using value_type = typename inner_traits::value_type;
+  using difference_type = typename inner_traits::difference_type;
+  using pointer = typename inner_traits::pointer;
+  using reference = typename inner_traits::reference;
+
   Iterator &operator++() { next(); return *this; }
   Iterator &operator--() { prev(); return *this; }
   Iterator operator++(int) { auto Tmp(Itr); next(); return Tmp; }
   Iterator operator--(int) { auto Tmp(Itr); prev(); return Tmp; }
   bool operator==(const Iterator &Other) const { return Itr == Other.Itr; }
   bool operator!=(const Iterator &Other) const { return !operator==(Other); }
-  T operator*() { return *Itr; }
-  PointerT operator->() { return &operator*(); }
+  reference operator*() { return *Itr; }
+  pointer operator->() { return &operator*(); }
   FilterIterator(PredType Pred, ItrType Itr, ItrType End)
       : Pred(Pred), Itr(Itr), End(End) {
     nextMatching();
@@ -383,6 +386,8 @@ class BinaryContext {
   }
 
   unsigned getDWARFEncodingSize(unsigned Encoding) {
+    if (Encoding == dwarf::DW_EH_PE_omit)
+      return 0;
     switch (Encoding & 0x0f) {
     default:
       llvm_unreachable("unknown encoding");
@@ -666,7 +671,6 @@ class BinaryContext {
 
   /// DWARF encoding. Available encoding types defined in BinaryFormat/Dwarf.h
   /// enum Constants, e.g. DW_EH_PE_omit.
-  unsigned TTypeEncoding = dwarf::DW_EH_PE_omit;
   unsigned LSDAEncoding = dwarf::DW_EH_PE_omit;
 
   BinaryContext(std::unique_ptr<MCContext> Ctx,

bolt/include/bolt/Core/BinaryFunction.h

@@ -34,8 +34,11 @@
 #include "bolt/Core/JumpTable.h"
 #include "bolt/Core/MCPlus.h"
 #include "bolt/Utils/NameResolver.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator.h"
+#include "llvm/ADT/iterator_range.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCDwarf.h"
@@ -44,9 +47,11 @@
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
+#include <iterator>
 #include <limits>
 #include <unordered_map>
 #include <unordered_set>
+#include <utility>
 #include <vector>
 
 using namespace llvm::object;
@@ -142,7 +147,8 @@ class BinaryFunction {
     uint64_t Action;
   };
 
-  using CallSitesType = SmallVector<CallSite, 0>;
+  using CallSitesList = SmallVector<std::pair<FragmentNum, CallSite>, 0>;
+  using CallSitesRange = iterator_range<CallSitesList::const_iterator>;
 
   using IslandProxiesType =
       std::map<BinaryFunction *, std::map<const MCSymbol *, MCSymbol *>>;
@@ -382,6 +388,9 @@ class BinaryFunction {
   /// Original LSDA address for the function.
   uint64_t LSDAAddress{0};
 
+  /// Original LSDA type encoding
+  unsigned LSDATypeEncoding{dwarf::DW_EH_PE_omit};
+
   /// Containing compilation unit for the function.
   DWARFUnit *DwarfUnit{nullptr};
 
@@ -492,8 +501,7 @@ class BinaryFunction {
   DenseMap<int32_t, SmallVector<int32_t, 4>> FrameRestoreEquivalents;
 
   // For tracking exception handling ranges.
-  CallSitesType CallSites;
-  CallSitesType ColdCallSites;
+  CallSitesList CallSites;
 
   /// Binary blobs representing action, type, and type index tables for this
   /// function' LSDA (exception handling).
@@ -507,9 +515,9 @@ class BinaryFunction {
   /// addressing.
   LSDATypeTableTy LSDATypeAddressTable;
 
-  /// Marking for the beginning of language-specific data area for the function.
-  MCSymbol *LSDASymbol{nullptr};
-  MCSymbol *ColdLSDASymbol{nullptr};
+  /// Marking for the beginnings of language-specific data areas for each
+  /// fragment of the function.
+  SmallVector<MCSymbol *, 0> LSDASymbols;
 
   /// Map to discover which CFIs are attached to a given instruction offset.
   /// Maps an instruction offset into a FrameInstructions offset.
@@ -716,7 +724,6 @@ class BinaryFunction {
       BB->releaseCFG();
 
     clearList(CallSites);
-    clearList(ColdCallSites);
     clearList(LSDATypeTable);
     clearList(LSDATypeAddressTable);
 
@@ -1067,6 +1074,14 @@ class BinaryFunction {
     return N;
   }
 
+  /// Return true if function has instructions to emit.
+  bool hasNonPseudoInstructions() const {
+    for (const BinaryBasicBlock &BB : blocks())
+      if (BB.getNumNonPseudos() > 0)
+        return true;
+    return false;
+  }
+
   /// Return MC symbol associated with the function.
   /// All references to the function should use this symbol.
   MCSymbol *getSymbol(const FragmentNum Fragment = FragmentNum::main()) {
@@ -1418,14 +1433,24 @@ class BinaryFunction {
 
   uint8_t getPersonalityEncoding() const { return PersonalityEncoding; }
 
-  const CallSitesType &getCallSites() const { return CallSites; }
+  CallSitesRange getCallSites(const FragmentNum F) const {
+    return make_range(std::equal_range(CallSites.begin(), CallSites.end(),
+                                       std::make_pair(F, CallSite()),
+                                       llvm::less_first()));
+  }
 
-  const CallSitesType &getColdCallSites() const { return ColdCallSites; }
+  void
+  addCallSites(const ArrayRef<std::pair<FragmentNum, CallSite>> NewCallSites) {
+    llvm::copy(NewCallSites, std::back_inserter(CallSites));
+    llvm::stable_sort(CallSites, llvm::less_first());
+  }
 
   ArrayRef<uint8_t> getLSDAActionTable() const { return LSDAActionTable; }
 
   const LSDATypeTableTy &getLSDATypeTable() const { return LSDATypeTable; }
 
+  unsigned getLSDATypeEncoding() const { return LSDATypeEncoding; }
+
   const LSDATypeTableTy &getLSDATypeAddressTable() const {
     return LSDATypeAddressTable;
   }
@@ -1822,9 +1847,11 @@ class BinaryFunction {
     return *this;
   }
 
-  /// Set LSDA symbol for the function.
+  /// Set main LSDA symbol for the function.
   BinaryFunction &setLSDASymbol(MCSymbol *Symbol) {
-    LSDASymbol = Symbol;
+    if (LSDASymbols.empty())
+      LSDASymbols.resize(1);
+    LSDASymbols.front() = Symbol;
     return *this;
   }
 
@@ -1848,30 +1875,25 @@ class BinaryFunction {
   uint64_t getLSDAAddress() const { return LSDAAddress; }
 
   /// Return symbol pointing to function's LSDA.
-  MCSymbol *getLSDASymbol() {
-    if (LSDASymbol)
-      return LSDASymbol;
-    if (CallSites.empty())
+  MCSymbol *getLSDASymbol(const FragmentNum F) {
+    if (F.get() < LSDASymbols.size() && LSDASymbols[F.get()] != nullptr)
+      return LSDASymbols[F.get()];
+    if (getCallSites(F).empty())
       return nullptr;
 
-    LSDASymbol = BC.Ctx->getOrCreateSymbol(
-        Twine("GCC_except_table") + Twine::utohexstr(getFunctionNumber()));
-
-    return LSDASymbol;
-  }
+    if (F.get() >= LSDASymbols.size())
+      LSDASymbols.resize(F.get() + 1);
 
-  /// Return symbol pointing to function's LSDA for the cold part.
-  MCSymbol *getColdLSDASymbol(const FragmentNum Fragment) {
-    if (ColdLSDASymbol)
-      return ColdLSDASymbol;
-    if (ColdCallSites.empty())
-      return nullptr;
+    SmallString<256> SymbolName;
+    if (F == FragmentNum::main())
+      SymbolName = formatv("GCC_except_table{0:x-}", getFunctionNumber());
+    else
+      SymbolName = formatv("GCC_cold_except_table{0:x-}.{1}",
+                           getFunctionNumber(), F.get());
 
-    ColdLSDASymbol =
-        BC.Ctx->getOrCreateSymbol(formatv("GCC_cold_except_table{0:x-}.{1}",
-                                          getFunctionNumber(), Fragment.get()));
+    LSDASymbols[F.get()] = BC.Ctx->getOrCreateSymbol(SymbolName);
 
-    return ColdLSDASymbol;
+    return LSDASymbols[F.get()];
   }
 
   void setOutputDataAddress(uint64_t Address) { OutputDataOffset = Address; }
@@ -2101,6 +2123,12 @@ class BinaryFunction {
   /// cannot be statically evaluated for any given indirect branch.
   bool postProcessIndirectBranches(MCPlusBuilder::AllocatorIdTy AllocId);
 
+  /// Validate that all data references to function offsets are claimed by
+  /// recognized jump tables. Register externally referenced blocks as entry
+  /// points. Returns true if there are no unclaimed externally referenced
+  /// offsets.
+  bool validateExternallyReferencedOffsets();
+
   /// Return all call site profile info for this function.
   IndirectCallSiteProfile &getAllCallSites() { return AllCallSites; }
 

bolt/include/bolt/Core/MCPlusBuilder.h

@@ -164,9 +164,14 @@ class MCPlusBuilder {
   void setTailCall(MCInst &Inst);
 
 public:
-  class InstructionIterator
-      : public std::iterator<std::bidirectional_iterator_tag, MCInst> {
+  class InstructionIterator {
   public:
+    using iterator_category = std::bidirectional_iterator_tag;
+    using value_type = MCInst;
+    using difference_type = std::ptrdiff_t;
+    using pointer = value_type *;
+    using reference = value_type &;
+
     class Impl {
     public:
       virtual Impl *Copy() const = 0;

bolt/include/bolt/Passes/CacheMetrics.h

@@ -21,18 +21,9 @@ namespace bolt {
 class BinaryFunction;
 namespace CacheMetrics {
 
-/// Calculate various metrics related to instruction cache performance.
+/// Calculate and print various metrics related to instruction cache performance
 void printAll(const std::vector<BinaryFunction *> &BinaryFunctions);
 
-/// Calculate Extended-TSP metric, which quantifies the expected number of
-/// i-cache misses for a given pair of basic blocks. The parameters are:
-/// - SrcAddr is the address of the source block;
-/// - SrcSize is the size of the source block;
-/// - DstAddr is the address of the destination block;
-/// - Count is the number of jumps between the pair of blocks.
-double extTSPScore(uint64_t SrcAddr, uint64_t SrcSize, uint64_t DstAddr,
-                   uint64_t Count);
-
 } // namespace CacheMetrics
 } // namespace bolt
 } // namespace llvm

bolt/include/bolt/Passes/DataflowAnalysis.h

@@ -439,13 +439,18 @@ class DataflowAnalysis {
 /// Define an iterator for navigating the expressions calculated by a
 /// dataflow analysis at each program point, when they are backed by a
 /// BitVector.
-class ExprIterator
-    : public std::iterator<std::forward_iterator_tag, const MCInst *> {
+class ExprIterator {
   const BitVector *BV;
   const std::vector<MCInst *> &Expressions;
   int Idx;
 
 public:
+  using iterator_category = std::forward_iterator_tag;
+  using value_type = const MCInst *;
+  using difference_type = std::ptrdiff_t;
+  using pointer = value_type *;
+  using reference = value_type &;
+
   ExprIterator &operator++() {
     assert(Idx != -1 && "Iterator already at the end");
     Idx = BV->find_next(Idx);

bolt/include/bolt/Passes/IndirectCallPromotion.h

@@ -217,6 +217,10 @@ class IndirectCallPromotion : public BinaryFunctionPass {
   bool shouldPrint(const BinaryFunction &BF) const override {
     return BinaryFunctionPass::shouldPrint(BF) && Modified.count(&BF) > 0;
   }
+  bool shouldOptimize(const BinaryFunction &BF) const override {
+    return BF.isSimple() && !BF.isIgnored() && BF.hasProfile() &&
+           !BF.hasUnknownControlFlow();
+  }
   void runOnFunctions(BinaryContext &BC) override;
 };
 

bolt/include/bolt/Passes/SplitFunctions.h

@@ -34,12 +34,21 @@ enum SplitFunctionsStrategy : char {
   All
 };
 
+class SplitStrategy {
+public:
+  using BlockIt = BinaryFunction::BasicBlockOrderType::iterator;
+
+  virtual ~SplitStrategy() = default;
+  virtual bool canSplit(const BinaryFunction &BF) = 0;
+  virtual bool keepEmpty() = 0;
+  virtual void fragment(const BlockIt Start, const BlockIt End) = 0;
+};
+
 /// Split function code in multiple parts.
 class SplitFunctions : public BinaryFunctionPass {
 private:
   /// Split function body into fragments.
-  template <typename Strategy>
-  void splitFunction(BinaryFunction &Function, Strategy S = {});
+  void splitFunction(BinaryFunction &Function, SplitStrategy &Strategy);
 
   struct TrampolineKey {
     FragmentNum SourceFN = FragmentNum::main();

bolt/include/bolt/Rewrite/ExecutableFileMemoryManager.h

@@ -10,7 +10,7 @@
 #define BOLT_REWRITE_EXECUTABLE_FILE_MEMORY_MANAGER_H
 
 #include "llvm/ADT/StringRef.h"
-#include "llvm/ExecutionEngine/SectionMemoryManager.h"
+#include "llvm/ExecutionEngine/RuntimeDyld.h"
 #include <cstdint>
 #include <string>
 
@@ -20,14 +20,21 @@ namespace bolt {
 class BinaryContext;
 
 /// Class responsible for allocating and managing code and data sections.
-class ExecutableFileMemoryManager : public SectionMemoryManager {
+class ExecutableFileMemoryManager : public RuntimeDyld::MemoryManager {
 private:
-  uint8_t *allocateSection(intptr_t Size, unsigned Alignment,
+  uint8_t *allocateSection(uintptr_t Size, unsigned Alignment,
                            unsigned SectionID, StringRef SectionName,
                            bool IsCode, bool IsReadOnly);
   BinaryContext &BC;
   bool AllowStubs;
 
+  struct AllocInfo {
+    uint8_t *Address;
+    size_t Size;
+    size_t Alignment;
+  };
+  SmallVector<AllocInfo, 8> AllocatedSections;
+
 public:
   // Our linker's main purpose is to handle a single object file, created
   // by RewriteInstance after reading the input binary and reordering it.
@@ -69,7 +76,16 @@ class ExecutableFileMemoryManager : public SectionMemoryManager {
 
   bool allowStubAllocation() const override { return AllowStubs; }
 
-  bool finalizeMemory(std::string *ErrMsg = nullptr) override;
+  /// Count processed objects and skip memory finalization.
+  bool finalizeMemory(std::string *ErrMsg) override {
+    ++ObjectsLoaded;
+    return false;
+  }
+
+  /// Ignore EH frames.
+  void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr,
+                        size_t Size) override {}
+  void deregisterEHFrames() override {}
 };
 
 } // namespace bolt