LLVM and SPIRV-LLVM-Translator pulldown (WW12)

LLVM: llvm/llvm-project@86a0b39aa5721
SPIRV-LLVM-Translator: KhronosGroup/SPIRV-LLVM-Translator@950ce8f

Author: bb-sycl

bolt/include/bolt/Core/BinaryContext.h

@@ -489,7 +489,9 @@ class BinaryContext {
   void adjustCodePadding();
 
   /// Regular page size.
-  static constexpr unsigned RegularPageSize = 0x1000;
+  unsigned RegularPageSize{0x1000};
+  static constexpr unsigned RegularPageSizeX86 = 0x1000;
+  static constexpr unsigned RegularPageSizeAArch64 = 0x10000;
 
   /// Huge page size to use.
   static constexpr unsigned HugePageSize = 0x200000;

bolt/include/bolt/Core/MCPlusBuilder.h

@@ -526,11 +526,6 @@ class MCPlusBuilder {
     return false;
   }
 
-  virtual bool isMOVSX64rm32(const MCInst &Inst) const {
-    llvm_unreachable("not implemented");
-    return false;
-  }
-
   virtual bool isLeave(const MCInst &Inst) const {
     llvm_unreachable("not implemented");
     return false;
@@ -1292,6 +1287,16 @@ class MCPlusBuilder {
     return false;
   }
 
+  /// Convert a move instruction into a conditional move instruction, given a
+  /// condition code.
+  virtual bool
+  convertMoveToConditionalMove(MCInst &Inst, unsigned CC,
+                               bool AllowStackMemOp = false,
+                               bool AllowBasePtrStackMemOp = false) const {
+    llvm_unreachable("not implemented");
+    return false;
+  }
+
   /// Lower a tail call instruction \p Inst if required by target.
   virtual bool lowerTailCall(MCInst &Inst) {
     llvm_unreachable("not implemented");

bolt/include/bolt/Passes/CMOVConversion.h

@@ -0,0 +1,85 @@
+//===- bolt/Passes/CMOVConversion.h ----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass finds the following patterns:
+//         jcc
+//        /   \
+// (empty)    mov src, dst
+//        \   /
+//
+// and replaces them with:
+//
+//   cmovcc src, dst
+//
+// The advantage of performing this conversion in BOLT (compared to compiler
+// heuristic driven instruction selection) is that BOLT can use LBR
+// misprediction information and only convert poorly predictable branches.
+// Note that branch misprediction rate is different from branch bias.
+// For well-predictable branches, it might be beneficial to leave jcc+mov as is
+// from microarchitectural perspective to avoid unneeded dependencies (CMOV
+// instruction has a dataflow dependence on flags and both operands).
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef BOLT_PASSES_CMOVCONVERSION_H
+#define BOLT_PASSES_CMOVCONVERSION_H
+
+#include "bolt/Passes/BinaryPasses.h"
+
+namespace llvm {
+namespace bolt {
+
+/// Pass for folding eligible hammocks into CMOV's if profitable.
+class CMOVConversion : public BinaryFunctionPass {
+  struct Stats {
+    /// Record how many possible cases there are.
+    uint64_t StaticPossible = 0;
+    uint64_t DynamicPossible = 0;
+
+    /// Record how many cases were converted.
+    uint64_t StaticPerformed = 0;
+    uint64_t DynamicPerformed = 0;
+
+    /// Record how many mispredictions were eliminated.
+    uint64_t PossibleMP = 0;
+    uint64_t RemovedMP = 0;
+
+    Stats operator+(const Stats &O) {
+      StaticPossible += O.StaticPossible;
+      DynamicPossible += O.DynamicPossible;
+      StaticPerformed += O.StaticPerformed;
+      DynamicPerformed += O.DynamicPerformed;
+      PossibleMP += O.PossibleMP;
+      RemovedMP += O.RemovedMP;
+      return *this;
+    }
+    double getStaticRatio() { return (double)StaticPerformed / StaticPossible; }
+    double getDynamicRatio() {
+      return (double)DynamicPerformed / DynamicPossible;
+    }
+    double getMPRatio() { return (double)RemovedMP / PossibleMP; }
+
+    void dump();
+  };
+  // BinaryContext-wide stats
+  Stats Global;
+
+  void runOnFunction(BinaryFunction &Function);
+
+public:
+  explicit CMOVConversion() : BinaryFunctionPass(false) {}
+
+  const char *getName() const override { return "CMOV conversion"; }
+
+  void runOnFunctions(BinaryContext &BC) override;
+};
+
+} // namespace bolt
+} // namespace llvm
+
+#endif

bolt/include/bolt/Rewrite/RewriteInstance.h

@@ -96,7 +96,7 @@ class RewriteInstance {
 
   /// Read info from special sections. E.g. eh_frame and .gcc_except_table
   /// for exception and stack unwinding information.
-  void readSpecialSections();
+  Error readSpecialSections();
 
   /// Adjust supplied command-line options based on input data.
   void adjustCommandLineOptions();
@@ -260,9 +260,9 @@ class RewriteInstance {
   void disassemblePLTSectionX86(BinarySection &Section, uint64_t EntrySize);
 
   /// ELF-specific part. TODO: refactor into new class.
-#define ELF_FUNCTION(FUNC)                                                     \
-  template <typename ELFT> void FUNC(object::ELFObjectFile<ELFT> *Obj);        \
-  void FUNC() {                                                                \
+#define ELF_FUNCTION(TYPE, FUNC)                                               \
+  template <typename ELFT> TYPE FUNC(object::ELFObjectFile<ELFT> *Obj);        \
+  TYPE FUNC() {                                                                \
     if (auto *ELF32LE = dyn_cast<object::ELF32LEObjectFile>(InputFile))        \
       return FUNC(ELF32LE);                                                    \
     if (auto *ELF64LE = dyn_cast<object::ELF64LEObjectFile>(InputFile))        \
@@ -277,25 +277,25 @@ class RewriteInstance {
   void patchELFPHDRTable();
 
   /// Create section header table.
-  ELF_FUNCTION(patchELFSectionHeaderTable);
+  ELF_FUNCTION(void, patchELFSectionHeaderTable);
 
   /// Create the regular symbol table and patch dyn symbol tables.
-  ELF_FUNCTION(patchELFSymTabs);
+  ELF_FUNCTION(void, patchELFSymTabs);
 
   /// Read dynamic section/segment of ELF.
-  ELF_FUNCTION(readELFDynamic);
+  ELF_FUNCTION(Error, readELFDynamic);
 
   /// Patch dynamic section/segment of ELF.
-  ELF_FUNCTION(patchELFDynamic);
+  ELF_FUNCTION(void, patchELFDynamic);
 
   /// Patch .got
-  ELF_FUNCTION(patchELFGOT);
+  ELF_FUNCTION(void, patchELFGOT);
 
   /// Patch allocatable relocation sections.
-  ELF_FUNCTION(patchELFAllocatableRelaSections);
+  ELF_FUNCTION(void, patchELFAllocatableRelaSections);
 
   /// Finalize memory image of section header string table.
-  ELF_FUNCTION(finalizeSectionStringTable);
+  ELF_FUNCTION(void, finalizeSectionStringTable);
 
   /// Return a name of the input file section in the output file.
   template <typename ELFObjType, typename ELFShdrTy>
@@ -498,7 +498,8 @@ class RewriteInstance {
   };
 
   /// AArch64 PLT sections.
-  const PLTSectionInfo AArch64_PLTSections[2] = {{".plt"}, {nullptr}};
+  const PLTSectionInfo AArch64_PLTSections[3] = {
+      {".plt"}, {".iplt"}, {nullptr}};
 
   /// Return PLT information for a section with \p SectionName or nullptr
   /// if the section is not PLT.

bolt/include/bolt/Utils/CommandLineOpts.h

@@ -30,6 +30,7 @@ extern llvm::cl::OptionCategory BoltInstrCategory;
 extern llvm::cl::OptionCategory HeatmapCategory;
 
 extern llvm::cl::opt<unsigned> AlignText;
+extern llvm::cl::opt<unsigned> AlignFunctions;
 extern llvm::cl::opt<bool> AggregateOnly;
 extern llvm::cl::opt<unsigned> BucketsPerLine;
 extern llvm::cl::opt<bool> DiffOnly;

bolt/lib/Core/BinaryContext.cpp

@@ -101,6 +101,7 @@ BinaryContext::BinaryContext(std::unique_ptr<MCContext> Ctx,
       InstPrinter(std::move(InstPrinter)), MIA(std::move(MIA)),
       MIB(std::move(MIB)), MRI(std::move(MRI)), DisAsm(std::move(DisAsm)) {
   Relocation::Arch = this->TheTriple->getArch();
+  RegularPageSize = isAArch64() ? RegularPageSizeAArch64 : RegularPageSizeX86;
   PageAlign = opts::NoHugePages ? RegularPageSize : HugePageSize;
 }
 

bolt/lib/Core/BinaryEmitter.cpp

@@ -291,6 +291,12 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function, bool EmitColdPart) {
   BC.Ctx->addGenDwarfSection(Section);
 
   if (BC.HasRelocations) {
+    // Set section alignment to at least maximum possible object alignment.
+    // We need this to support LongJmp and other passes that calculates
+    // tentative layout.
+    if (Section->getAlignment() < opts::AlignFunctions)
+      Section->setAlignment(Align(opts::AlignFunctions));
+
     Streamer.emitCodeAlignment(BinaryFunction::MinAlign, &*BC.STI);
     uint16_t MaxAlignBytes = EmitColdPart ? Function.getMaxColdAlignmentBytes()
                                           : Function.getMaxAlignmentBytes();

bolt/lib/Passes/Aligner.cpp

@@ -23,6 +23,7 @@ extern cl::OptionCategory BoltOptCategory;
 
 extern cl::opt<bool> AlignBlocks;
 extern cl::opt<bool> PreserveBlocksAlignment;
+extern cl::opt<unsigned> AlignFunctions;
 
 cl::opt<unsigned>
 AlignBlocksMinSize("align-blocks-min-size",
@@ -43,13 +44,6 @@ AlignBlocksThreshold("align-blocks-threshold",
   cl::Hidden,
   cl::cat(BoltOptCategory));
 
-cl::opt<unsigned>
-AlignFunctions("align-functions",
-  cl::desc("align functions at a given value (relocation mode)"),
-  cl::init(64),
-  cl::ZeroOrMore,
-  cl::cat(BoltOptCategory));
-
 cl::opt<unsigned>
 AlignFunctionsMaxBytes("align-functions-max-bytes",
   cl::desc("maximum number of bytes to use to align functions"),