LLVM and SPIRV-LLVM-Translator pulldown (WW44)

bolt/include/bolt/Core/BinaryContext.h

@@ -179,6 +179,10 @@ class BinaryContext {
   using NameToSectionMapType = std::multimap<std::string, BinarySection *>;
   NameToSectionMapType NameToSection;
 
+  /// Map section references to BinarySection for matching sections in the
+  /// input file to internal section representation.
+  DenseMap<SectionRef, BinarySection *> SectionRefToBinarySection;
+
   /// Low level section registration.
   BinarySection &registerSection(BinarySection *Section);
 
@@ -224,6 +228,9 @@ class BinaryContext {
   /// DWARF line info for CUs.
   std::map<unsigned, DwarfLineTable> DwarfLineTablesCUMap;
 
+  /// Internal helper for removing section name from a lookup table.
+  void deregisterSectionName(const BinarySection &Section);
+
 public:
   static Expected<std::unique_ptr<BinaryContext>>
   createBinaryContext(const ObjectFile *File, bool IsPIC,
@@ -951,13 +958,13 @@ class BinaryContext {
   BinarySection &registerSection(SectionRef Section);
 
   /// Register a copy of /p OriginalSection under a different name.
-  BinarySection &registerSection(StringRef SectionName,
+  BinarySection &registerSection(const Twine &SectionName,
                                  const BinarySection &OriginalSection);
 
   /// Register or update the information for the section with the given
   /// /p Name.  If the section already exists, the information in the
   /// section will be updated with the new data.
-  BinarySection &registerOrUpdateSection(StringRef Name, unsigned ELFType,
+  BinarySection &registerOrUpdateSection(const Twine &Name, unsigned ELFType,
                                          unsigned ELFFlags,
                                          uint8_t *Data = nullptr,
                                          uint64_t Size = 0,
@@ -967,7 +974,7 @@ class BinaryContext {
   /// with the given /p Name.  If the section already exists, the
   /// information in the section will be updated with the new data.
   BinarySection &
-  registerOrUpdateNoteSection(StringRef Name, uint8_t *Data = nullptr,
+  registerOrUpdateNoteSection(const Twine &Name, uint8_t *Data = nullptr,
                               uint64_t Size = 0, unsigned Alignment = 1,
                               bool IsReadOnly = true,
                               unsigned ELFType = ELF::SHT_PROGBITS) {
@@ -976,10 +983,16 @@ class BinaryContext {
                                    Size, Alignment);
   }
 
+  /// Remove sections that were preregistered but never used.
+  void deregisterUnusedSections();
+
   /// Remove the given /p Section from the set of all sections.  Return
   /// true if the section was removed (and deleted), otherwise false.
   bool deregisterSection(BinarySection &Section);
 
+  /// Re-register \p Section under the \p NewName.
+  void renameSection(BinarySection &Section, const Twine &NewName);
+
   /// Iterate over all registered sections.
   iterator_range<FilteredSectionIterator> sections() {
     auto notNull = [](const SectionIterator &Itr) { return (bool)*Itr; };
@@ -1073,20 +1086,26 @@ class BinaryContext {
     return const_cast<BinaryContext *>(this)->getSectionForAddress(Address);
   }
 
+  /// Return internal section representation for a section in a file.
+  BinarySection *getSectionForSectionRef(SectionRef Section) const {
+    return SectionRefToBinarySection.lookup(Section);
+  }
+
   /// Return section(s) associated with given \p Name.
   iterator_range<NameToSectionMapType::iterator>
-  getSectionByName(StringRef Name) {
-    return make_range(NameToSection.equal_range(std::string(Name)));
+  getSectionByName(const Twine &Name) {
+    return make_range(NameToSection.equal_range(Name.str()));
   }
   iterator_range<NameToSectionMapType::const_iterator>
-  getSectionByName(StringRef Name) const {
-    return make_range(NameToSection.equal_range(std::string(Name)));
+  getSectionByName(const Twine &Name) const {
+    return make_range(NameToSection.equal_range(Name.str()));
   }
 
   /// Return the unique section associated with given \p Name.
   /// If there is more than one section with the same name, return an error
   /// object.
-  ErrorOr<BinarySection &> getUniqueSectionByName(StringRef SectionName) const {
+  ErrorOr<BinarySection &>
+  getUniqueSectionByName(const Twine &SectionName) const {
     auto Sections = getSectionByName(SectionName);
     if (Sections.begin() != Sections.end() &&
         std::next(Sections.begin()) == Sections.end())
@@ -1217,10 +1236,10 @@ class BinaryContext {
     return Size;
   }
 
-  /// Verify that assembling instruction \p Inst results in the same sequence of
-  /// bytes as \p Encoding.
-  bool validateEncoding(const MCInst &Instruction,
-                        ArrayRef<uint8_t> Encoding) const;
+  /// Validate that disassembling the \p Sequence of bytes into an instruction
+  /// and assembling the instruction again, results in a byte sequence identical
+  /// to the original one.
+  bool validateInstructionEncoding(ArrayRef<uint8_t> Sequence) const;
 
   /// Return a function execution count threshold for determining whether
   /// the function is 'hot'. Consider it hot if count is above the average exec

bolt/include/bolt/Core/BinarySection.h

@@ -43,9 +43,12 @@ class BinaryData;
 class BinarySection {
   friend class BinaryContext;
 
+  /// Count the number of sections created.
+  static uint64_t Count;
+
   BinaryContext &BC;           // Owning BinaryContext
   std::string Name;            // Section name
-  const SectionRef Section;    // SectionRef (may be null)
+  const SectionRef Section;    // SectionRef for input binary sections.
   StringRef Contents;          // Input section contents
   const uint64_t Address;      // Address of section in input binary (may be 0)
   const uint64_t Size;         // Input section size
@@ -86,6 +89,7 @@ class BinarySection {
   uint64_t OutputSize{0};          // Section size in the rewritten binary.
   uint64_t OutputFileOffset{0};    // File offset in the rewritten binary file.
   StringRef OutputContents;        // Rewritten section contents.
+  const uint64_t SectionNumber;    // Order in which the section was created.
   unsigned SectionID{-1u};         // Unique ID used for address mapping.
                                    // Set by ExecutableFileMemoryManager.
   uint32_t Index{0};               // Section index in the output file.
@@ -140,20 +144,21 @@ class BinarySection {
 
 public:
   /// Copy a section.
-  explicit BinarySection(BinaryContext &BC, StringRef Name,
+  explicit BinarySection(BinaryContext &BC, const Twine &Name,
                          const BinarySection &Section)
-      : BC(BC), Name(Name), Section(Section.getSectionRef()),
+      : BC(BC), Name(Name.str()), Section(SectionRef()),
         Contents(Section.getContents()), Address(Section.getAddress()),
         Size(Section.getSize()), Alignment(Section.getAlignment()),
         ELFType(Section.getELFType()), ELFFlags(Section.getELFFlags()),
         Relocations(Section.Relocations),
-        PendingRelocations(Section.PendingRelocations), OutputName(Name) {}
+        PendingRelocations(Section.PendingRelocations), OutputName(Name.str()),
+        SectionNumber(++Count) {}
 
   BinarySection(BinaryContext &BC, SectionRef Section)
       : BC(BC), Name(getName(Section)), Section(Section),
         Contents(getContents(Section)), Address(Section.getAddress()),
         Size(Section.getSize()), Alignment(Section.getAlignment()),
-        OutputName(Name) {
+        OutputName(Name), SectionNumber(++Count) {
     if (isELF()) {
       ELFType = ELFSectionRef(Section).getType();
       ELFFlags = ELFSectionRef(Section).getFlags();
@@ -167,13 +172,14 @@ class BinarySection {
   }
 
   // TODO: pass Data as StringRef/ArrayRef? use StringRef::copy method.
-  BinarySection(BinaryContext &BC, StringRef Name, uint8_t *Data, uint64_t Size,
-                unsigned Alignment, unsigned ELFType, unsigned ELFFlags)
-      : BC(BC), Name(Name),
+  BinarySection(BinaryContext &BC, const Twine &Name, uint8_t *Data,
+                uint64_t Size, unsigned Alignment, unsigned ELFType,
+                unsigned ELFFlags)
+      : BC(BC), Name(Name.str()),
         Contents(reinterpret_cast<const char *>(Data), Data ? Size : 0),
         Address(0), Size(Size), Alignment(Alignment), ELFType(ELFType),
-        ELFFlags(ELFFlags), IsFinalized(true), OutputName(Name),
-        OutputSize(Size), OutputContents(Contents) {
+        ELFFlags(ELFFlags), IsFinalized(true), OutputName(Name.str()),
+        OutputSize(Size), OutputContents(Contents), SectionNumber(++Count) {
     assert(Alignment > 0 && "section alignment must be > 0");
   }
 
@@ -207,10 +213,34 @@ class BinarySection {
 
   // Order sections by their immutable properties.
   bool operator<(const BinarySection &Other) const {
-    return (getAddress() < Other.getAddress() ||
-            (getAddress() == Other.getAddress() &&
-             (getSize() < Other.getSize() ||
-              (getSize() == Other.getSize() && getName() < Other.getName()))));
+    // Allocatable before non-allocatable.
+    if (isAllocatable() != Other.isAllocatable())
+      return isAllocatable() > Other.isAllocatable();
+
+    // Input sections take precedence.
+    if (hasSectionRef() != Other.hasSectionRef())
+      return hasSectionRef() > Other.hasSectionRef();
+
+    // Compare allocatable input sections by their address.
+    if (hasSectionRef() && getAddress() != Other.getAddress())
+      return getAddress() < Other.getAddress();
+    if (hasSectionRef() && getAddress() && getSize() != Other.getSize())
+      return getSize() < Other.getSize();
+
+    // Code before data.
+    if (isText() != Other.isText())
+      return isText() > Other.isText();
+
+    // Read-only before writable.
+    if (isReadOnly() != Other.isReadOnly())
+      return isReadOnly() > Other.isReadOnly();
+
+    // BSS at the end.
+    if (isBSS() != Other.isBSS())
+      return isBSS() < Other.isBSS();
+
+    // Otherwise, preserve the order of creation.
+    return SectionNumber < Other.SectionNumber;
   }
 
   ///
@@ -228,13 +258,13 @@ class BinarySection {
   bool isText() const {
     if (isELF())
       return (ELFFlags & ELF::SHF_EXECINSTR);
-    return getSectionRef().isText();
+    return hasSectionRef() && getSectionRef().isText();
   }
   bool isData() const {
     if (isELF())
       return (ELFType == ELF::SHT_PROGBITS &&
               (ELFFlags & (ELF::SHF_ALLOC | ELF::SHF_WRITE)));
-    return getSectionRef().isData();
+    return hasSectionRef() && getSectionRef().isData();
   }
   bool isBSS() const {
     return (ELFType == ELF::SHT_NOBITS &&
@@ -406,6 +436,7 @@ class BinarySection {
     return SectionID;
   }
   bool hasValidSectionID() const { return SectionID != -1u; }
+  bool hasValidIndex() { return Index != 0; }
   uint32_t getIndex() const { return Index; }
 
   // mutation
@@ -416,12 +447,12 @@ class BinarySection {
     SectionID = ID;
   }
   void setIndex(uint32_t I) { Index = I; }
-  void setOutputName(StringRef Name) { OutputName = std::string(Name); }
+  void setOutputName(const Twine &Name) { OutputName = Name.str(); }
   void setAnonymous(bool Flag) { IsAnonymous = Flag; }
 
-  /// Emit the section as data, possibly with relocations. Use name \p NewName
-  //  for the section during emission if non-empty.
-  void emitAsData(MCStreamer &Streamer, StringRef NewName = StringRef()) const;
+  /// Emit the section as data, possibly with relocations.
+  /// Use name \p SectionName for the section during the emission.
+  void emitAsData(MCStreamer &Streamer, const Twine &SectionName) const;
 
   using SymbolResolverFuncTy = llvm::function_ref<uint64_t(const MCSymbol *)>;
 
@@ -430,6 +461,13 @@ class BinarySection {
   void flushPendingRelocations(raw_pwrite_stream &OS,
                                SymbolResolverFuncTy Resolver);
 
+  /// Change contents of the section.
+  void updateContents(const uint8_t *Data, size_t NewSize) {
+    OutputContents = StringRef(reinterpret_cast<const char *>(Data), NewSize);
+    OutputSize = NewSize;
+    IsFinalized = true;
+  }
+
   /// Reorder the contents of this section according to /p Order.  If
   /// /p Inplace is true, the entire contents of the section is reordered,
   /// otherwise the new contents contain only the reordered data.

bolt/include/bolt/Core/MCPlusBuilder.h

@@ -1266,8 +1266,18 @@ class MCPlusBuilder {
   /// Replace displacement in compound memory operand with given \p Label.
   bool replaceMemOperandDisp(MCInst &Inst, const MCSymbol *Label,
                              MCContext *Ctx) const {
-    return replaceMemOperandDisp(
-        Inst, MCOperand::createExpr(MCSymbolRefExpr::create(Label, *Ctx)));
+    return replaceMemOperandDisp(Inst, Label, 0, Ctx);
+  }
+
+  /// Replace displacement in compound memory operand with given \p Label
+  /// plus addend.
+  bool replaceMemOperandDisp(MCInst &Inst, const MCSymbol *Label,
+                             int64_t Addend, MCContext *Ctx) const {
+    MCInst::iterator MemOpI = getMemOperandDisp(Inst);
+    if (MemOpI == Inst.end())
+      return false;
+    return setOperandToSymbolRef(Inst, MemOpI - Inst.begin(), Label, Addend,
+                                 Ctx, 0);
   }
 
   /// Returns how many bits we have in this instruction to encode a PC-rel

bolt/include/bolt/Passes/ValidateMemRefs.h

@@ -0,0 +1,41 @@
+//===- bolt/Passes/ValidateMemRefs.h ----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef BOLT_PASSES_VALIDATEMEMREFS_H
+#define BOLT_PASSES_VALIDATEMEMREFS_H
+
+#include "bolt/Passes/BinaryPasses.h"
+
+namespace llvm::bolt {
+
+/// Post processing to check for memory references that cause a symbol
+/// in data section to be ambiguous, requiring us to avoid moving that
+/// object or disambiguating such references. This is currently
+/// limited to fixing false references to the location of jump tables.
+///
+class ValidateMemRefs : public BinaryFunctionPass {
+public:
+  explicit ValidateMemRefs(const cl::opt<bool> &PrintPass)
+      : BinaryFunctionPass(PrintPass) {}
+
+  const char *getName() const override { return "validate-mem-refs"; }
+
+  void runOnFunctions(BinaryContext &BC) override;
+
+private:
+  bool checkAndFixJTReference(BinaryFunction &BF, MCInst &Inst,
+                              uint32_t OperandNum, const MCSymbol *Sym,
+                              uint64_t Offset);
+  void runOnFunction(BinaryFunction &BF);
+
+  static std::atomic<std::uint64_t> ReplacedReferences;
+};
+
+} // namespace llvm::bolt
+
+#endif

bolt/include/bolt/Rewrite/ExecutableFileMemoryManager.h

@@ -35,6 +35,12 @@ class ExecutableFileMemoryManager : public RuntimeDyld::MemoryManager {
   };
   SmallVector<AllocInfo, 8> AllocatedSections;
 
+  // All new sections will be identified by the following prefix.
+  std::string NewSecPrefix;
+
+  // Name prefix used for sections from the input.
+  std::string OrgSecPrefix;
+
 public:
   // Our linker's main purpose is to handle a single object file, created
   // by RewriteInstance after reading the input binary and reordering it.
@@ -86,6 +92,10 @@ class ExecutableFileMemoryManager : public RuntimeDyld::MemoryManager {
   void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr,
                         size_t Size) override {}
   void deregisterEHFrames() override {}
+
+  /// Section name management.
+  void setNewSecPrefix(StringRef Prefix) { NewSecPrefix = Prefix; }
+  void setOrgSecPrefix(StringRef Prefix) { OrgSecPrefix = Prefix; }
 };
 
 } // namespace bolt

bolt/include/bolt/Rewrite/MachORewriteInstance.h

@@ -46,6 +46,7 @@ class MachORewriteInstance {
   void processProfileDataPreCFG();
   void processProfileData();
 
+  static StringRef getNewSecPrefix() { return ".bolt.new"; }
   static StringRef getOrgSecPrefix() { return ".bolt.org"; }
 
   void mapInstrumentationSection(StringRef SectionName);