Merge remote-tracking branch 'intel/sycl' into steffen/rework_redu_res_acq

Author: steffenlarsen

.github/CODEOWNERS

@@ -6,6 +6,9 @@ clang/ @intel/dpcpp-cfe-reviewers
 # Clang driver
 clang/**/Driver/ @intel/dpcpp-clang-driver-reviewers
 
+# Clang tools
+clang-tools-extra/ @intel/dpcpp-cfe-reviewers
+
 # LLVM-SPIRV translator
 llvm-spirv/ @intel/dpcpp-spirv-reviewers
 

.github/workflows/llvm-bugs.yml

@@ -48,6 +48,7 @@ jobs:
                 to: "[email protected]",
                 subject: `[Bug ${issue.data.number}] ${issue.data.title}`,
                 template: "new-github-issue",
+                'o:tracking-clicks': 'no',
                 'h:X-Mailgun-Variables': JSON.stringify(payload)
               };
 

.github/workflows/sycl_linux_build_and_test.yml

@@ -32,7 +32,7 @@ on:
       build_configure_extra_args:
         type: string
         required: false
-        default: "--hip --cuda"
+        default: "--hip --cuda --enable-esimd-emulator"
       build_artifact_suffix:
         type: string
         required: true
@@ -200,6 +200,11 @@ jobs:
       image: ${{ matrix.image }}
       options: ${{ matrix.container_options }}
     steps:
+    - name: Reset GPU
+      if: ${{ contains(matrix.config, 'gen9') }}
+      run: |
+        sudo mount -t debugfs none /sys/kernel/debug
+        sudo bash -c 'echo 1 > /sys/kernel/debug/dri/0/i915_wedged'
     - run: cp -r /actions .
     - name: Register cleanup after job is finished
       uses: ./actions/cleanup

.github/workflows/sycl_precommit.yml

@@ -39,4 +39,4 @@ jobs:
       build_cache_size: "8G"
       build_artifact_suffix: "default"
       build_cache_suffix: "default"
-      lts_config: "hip_amdgpu;ocl_x64;ocl_gen9;l0_gen9"
+      lts_config: "hip_amdgpu;ocl_x64;ocl_gen9;l0_gen9;esimd_emu"

.mailmap

@@ -16,7 +16,7 @@
 # Combinations of both are possible too, see
 # https://git-scm.com/docs/gitmailmap for format details.
 #
-# You can commit changes for your own names and  email addresses without review.
+# You can commit changes for your own names and email addresses without review.
 # If you want to add entries for other people, please have them review the
 # addition.
 #

bolt/include/bolt/Core/BinaryContext.h

@@ -489,7 +489,9 @@ class BinaryContext {
   void adjustCodePadding();
 
   /// Regular page size.
-  static constexpr unsigned RegularPageSize = 0x1000;
+  unsigned RegularPageSize{0x1000};
+  static constexpr unsigned RegularPageSizeX86 = 0x1000;
+  static constexpr unsigned RegularPageSizeAArch64 = 0x10000;
 
   /// Huge page size to use.
   static constexpr unsigned HugePageSize = 0x200000;

bolt/include/bolt/Core/BinaryFunction.h

@@ -1299,10 +1299,11 @@ class BinaryFunction {
     case ELF::R_X86_64_32:
     case ELF::R_X86_64_32S:
     case ELF::R_X86_64_64:
+    case ELF::R_X86_64_PC8:
+    case ELF::R_X86_64_PC32:
+    case ELF::R_X86_64_PC64:
       Relocations[Offset] = Relocation{Offset, Symbol, RelType, Addend, Value};
       return;
-    case ELF::R_X86_64_PC32:
-    case ELF::R_X86_64_PC8:
     case ELF::R_X86_64_PLT32:
     case ELF::R_X86_64_GOTPCRELX:
     case ELF::R_X86_64_REX_GOTPCRELX:

bolt/include/bolt/Core/DebugData.h

@@ -490,6 +490,7 @@ class DebugInfoBinaryPatcher : public SimpleBinaryPatcher {
     PatchBaseClass,
     PatchValue32,
     PatchValue64to32,
+    PatchValue32GenericSize,
     PatchValue64,
     PatchValueVariable,
     ReferencePatchValue,
@@ -536,6 +537,22 @@ class DebugInfoBinaryPatcher : public SimpleBinaryPatcher {
     uint32_t Value;
   };
 
+  /// Patch for 4 byte entry, where original entry size is not 4 bytes or 8
+  /// bytes.
+  struct DebugPatch32GenericSize : public Patch {
+    DebugPatch32GenericSize(uint32_t O, uint32_t V, uint32_t OVS)
+        : Patch(O, DebugPatchKind::PatchValue32GenericSize) {
+      Value = V;
+      OldValueSize = OVS;
+    }
+
+    static bool classof(const Patch *Writer) {
+      return Writer->getKind() == DebugPatchKind::PatchValue32GenericSize;
+    }
+    uint32_t Value;
+    uint32_t OldValueSize;
+  };
+
   struct DebugPatch64 : public Patch {
     DebugPatch64(uint32_t O, uint64_t V)
         : Patch(O, DebugPatchKind::PatchValue64) {
@@ -693,6 +710,9 @@ class DebugInfoBinaryPatcher : public SimpleBinaryPatcher {
       case DebugPatchKind::PatchValue64to32:
         delete reinterpret_cast<DebugPatch64to32 *>(P);
         break;
+      case DebugPatchKind::PatchValue32GenericSize:
+        delete reinterpret_cast<DebugPatch32GenericSize *>(P);
+        break;
       case DebugPatchKind::PatchValue64:
         delete reinterpret_cast<DebugPatch64 *>(P);
         break;

bolt/include/bolt/Core/MCPlusBuilder.h

@@ -424,9 +424,7 @@ class MCPlusBuilder {
 
   /// Return a register number that is guaranteed to not match with
   /// any real register on the underlying architecture.
-  virtual MCPhysReg getNoRegister() const {
-    llvm_unreachable("not implemented");
-  }
+  MCPhysReg getNoRegister() const { return MCRegister::NoRegister; }
 
   /// Return a register corresponding to a function integer argument \p ArgNo
   /// if the argument is passed in a register. Or return the result of
@@ -528,11 +526,6 @@ class MCPlusBuilder {
     return false;
   }
 
-  virtual bool isMOVSX64rm32(const MCInst &Inst) const {
-    llvm_unreachable("not implemented");
-    return false;
-  }
-
   virtual bool isLeave(const MCInst &Inst) const {
     llvm_unreachable("not implemented");
     return false;
@@ -1294,6 +1287,16 @@ class MCPlusBuilder {
     return false;
   }
 
+  /// Convert a move instruction into a conditional move instruction, given a
+  /// condition code.
+  virtual bool
+  convertMoveToConditionalMove(MCInst &Inst, unsigned CC,
+                               bool AllowStackMemOp = false,
+                               bool AllowBasePtrStackMemOp = false) const {
+    llvm_unreachable("not implemented");
+    return false;
+  }
+
   /// Lower a tail call instruction \p Inst if required by target.
   virtual bool lowerTailCall(MCInst &Inst) {
     llvm_unreachable("not implemented");
@@ -1327,6 +1330,16 @@ class MCPlusBuilder {
     return IndirectBranchType::UNKNOWN;
   }
 
+  /// Analyze branch \p Instruction in PLT section and try to determine
+  /// associated got entry address.
+  virtual uint64_t analyzePLTEntry(MCInst &Instruction,
+                                   InstructionIterator Begin,
+                                   InstructionIterator End,
+                                   uint64_t BeginPC) const {
+    llvm_unreachable("not implemented");
+    return 0;
+  }
+
   virtual bool analyzeVirtualMethodCall(InstructionIterator Begin,
                                         InstructionIterator End,
                                         std::vector<MCInst *> &MethodFetchInsns,

bolt/include/bolt/Core/Relocation.h

@@ -49,7 +49,7 @@ struct Relocation {
   /// Used to validate relocation correctness.
   uint64_t Value;
 
-  /// Return size of the given relocation \p Type.
+  /// Return size in bytes of the given relocation \p Type.
   static size_t getSizeForType(uint64_t Type);
 
   /// Return size of this relocation.

bolt/include/bolt/Passes/CMOVConversion.h

@@ -0,0 +1,85 @@
+//===- bolt/Passes/CMOVConversion.h ----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass finds the following patterns:
+//         jcc
+//        /   \
+// (empty)    mov src, dst
+//        \   /
+//
+// and replaces them with:
+//
+//   cmovcc src, dst
+//
+// The advantage of performing this conversion in BOLT (compared to compiler
+// heuristic driven instruction selection) is that BOLT can use LBR
+// misprediction information and only convert poorly predictable branches.
+// Note that branch misprediction rate is different from branch bias.
+// For well-predictable branches, it might be beneficial to leave jcc+mov as is
+// from microarchitectural perspective to avoid unneeded dependencies (CMOV
+// instruction has a dataflow dependence on flags and both operands).
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef BOLT_PASSES_CMOVCONVERSION_H
+#define BOLT_PASSES_CMOVCONVERSION_H
+
+#include "bolt/Passes/BinaryPasses.h"
+
+namespace llvm {
+namespace bolt {
+
+/// Pass for folding eligible hammocks into CMOV's if profitable.
+class CMOVConversion : public BinaryFunctionPass {
+  struct Stats {
+    /// Record how many possible cases there are.
+    uint64_t StaticPossible = 0;
+    uint64_t DynamicPossible = 0;
+
+    /// Record how many cases were converted.
+    uint64_t StaticPerformed = 0;
+    uint64_t DynamicPerformed = 0;
+
+    /// Record how many mispredictions were eliminated.
+    uint64_t PossibleMP = 0;
+    uint64_t RemovedMP = 0;
+
+    Stats operator+(const Stats &O) {
+      StaticPossible += O.StaticPossible;
+      DynamicPossible += O.DynamicPossible;
+      StaticPerformed += O.StaticPerformed;
+      DynamicPerformed += O.DynamicPerformed;
+      PossibleMP += O.PossibleMP;
+      RemovedMP += O.RemovedMP;
+      return *this;
+    }
+    double getStaticRatio() { return (double)StaticPerformed / StaticPossible; }
+    double getDynamicRatio() {
+      return (double)DynamicPerformed / DynamicPossible;
+    }
+    double getMPRatio() { return (double)RemovedMP / PossibleMP; }
+
+    void dump();
+  };
+  // BinaryContext-wide stats
+  Stats Global;
+
+  void runOnFunction(BinaryFunction &Function);
+
+public:
+  explicit CMOVConversion() : BinaryFunctionPass(false) {}
+
+  const char *getName() const override { return "CMOV conversion"; }
+
+  void runOnFunctions(BinaryContext &BC) override;
+};
+
+} // namespace bolt
+} // namespace llvm
+
+#endif

bolt/include/bolt/Rewrite/RewriteInstance.h

@@ -56,7 +56,7 @@ class RewriteInstance {
   Error setProfile(StringRef Filename);
 
   /// Run all the necessary steps to read, optimize and rewrite the binary.
-  void run();
+  Error run();
 
   /// Diff this instance against another one. Non-const since we may run passes
   /// to fold identical functions.
@@ -96,7 +96,7 @@ class RewriteInstance {
 
   /// Read info from special sections. E.g. eh_frame and .gcc_except_table
   /// for exception and stack unwinding information.
-  void readSpecialSections();
+  Error readSpecialSections();
 
   /// Adjust supplied command-line options based on input data.
   void adjustCommandLineOptions();
@@ -218,7 +218,7 @@ class RewriteInstance {
 
   /// Detect addresses and offsets available in the binary for allocating
   /// new sections.
-  void discoverStorage();
+  Error discoverStorage();
 
   /// Adjust function sizes and set proper maximum size values after the whole
   /// symbol table has been processed.
@@ -246,10 +246,23 @@ class RewriteInstance {
   /// Disassemble and create function entries for PLT.
   void disassemblePLT();
 
+  /// Auxiliary function to create .plt BinaryFunction on \p EntryAddres
+  /// with the \p EntrySize size. \p TargetAddress is the .got entry
+  /// associated address.
+  void createPLTBinaryFunction(uint64_t TargetAddress, uint64_t EntryAddress,
+                               uint64_t EntrySize);
+
+  /// Disassemble aarch64-specific .plt \p Section auxiliary function
+  void disassemblePLTSectionAArch64(BinarySection &Section);
+
+  /// Disassemble X86-specific .plt \p Section auxiliary function. \p EntrySize
+  /// is the expected .plt \p Section entry function size.
+  void disassemblePLTSectionX86(BinarySection &Section, uint64_t EntrySize);
+
   /// ELF-specific part. TODO: refactor into new class.
-#define ELF_FUNCTION(FUNC)                                                     \
-  template <typename ELFT> void FUNC(object::ELFObjectFile<ELFT> *Obj);        \
-  void FUNC() {                                                                \
+#define ELF_FUNCTION(TYPE, FUNC)                                               \
+  template <typename ELFT> TYPE FUNC(object::ELFObjectFile<ELFT> *Obj);        \
+  TYPE FUNC() {                                                                \
     if (auto *ELF32LE = dyn_cast<object::ELF32LEObjectFile>(InputFile))        \
       return FUNC(ELF32LE);                                                    \
     if (auto *ELF64LE = dyn_cast<object::ELF64LEObjectFile>(InputFile))        \
@@ -264,25 +277,25 @@ class RewriteInstance {
   void patchELFPHDRTable();
 
   /// Create section header table.
-  ELF_FUNCTION(patchELFSectionHeaderTable);
+  ELF_FUNCTION(void, patchELFSectionHeaderTable);
 
   /// Create the regular symbol table and patch dyn symbol tables.
-  ELF_FUNCTION(patchELFSymTabs);
+  ELF_FUNCTION(void, patchELFSymTabs);
 
   /// Read dynamic section/segment of ELF.
-  ELF_FUNCTION(readELFDynamic);
+  ELF_FUNCTION(Error, readELFDynamic);
 
   /// Patch dynamic section/segment of ELF.
-  ELF_FUNCTION(patchELFDynamic);
+  ELF_FUNCTION(void, patchELFDynamic);
 
   /// Patch .got
-  ELF_FUNCTION(patchELFGOT);
+  ELF_FUNCTION(void, patchELFGOT);
 
   /// Patch allocatable relocation sections.
-  ELF_FUNCTION(patchELFAllocatableRelaSections);
+  ELF_FUNCTION(void, patchELFAllocatableRelaSections);
 
   /// Finalize memory image of section header string table.
-  ELF_FUNCTION(finalizeSectionStringTable);
+  ELF_FUNCTION(void, finalizeSectionStringTable);
 
   /// Return a name of the input file section in the output file.
   template <typename ELFObjType, typename ELFShdrTy>
@@ -473,7 +486,7 @@ class RewriteInstance {
   /// multiple variants generated by different linkers.
   struct PLTSectionInfo {
     const char *Name;
-    uint64_t EntrySize;
+    uint64_t EntrySize{0};
   };
 
   /// Different types of X86-64 PLT sections.
@@ -485,10 +498,8 @@ class RewriteInstance {
   };
 
   /// AArch64 PLT sections.
-  const PLTSectionInfo AArch64_PLTSections[2] = {
-      { ".plt", 16 },
-      { nullptr, 0 }
-  };
+  const PLTSectionInfo AArch64_PLTSections[3] = {
+      {".plt"}, {".iplt"}, {nullptr}};
 
   /// Return PLT information for a section with \p SectionName or nullptr
   /// if the section is not PLT.

bolt/include/bolt/Utils/CommandLineOpts.h

@@ -30,6 +30,7 @@ extern llvm::cl::OptionCategory BoltInstrCategory;
 extern llvm::cl::OptionCategory HeatmapCategory;
 
 extern llvm::cl::opt<unsigned> AlignText;
+extern llvm::cl::opt<unsigned> AlignFunctions;
 extern llvm::cl::opt<bool> AggregateOnly;
 extern llvm::cl::opt<unsigned> BucketsPerLine;
 extern llvm::cl::opt<bool> DiffOnly;

bolt/lib/Core/BinaryContext.cpp

@@ -101,6 +101,7 @@ BinaryContext::BinaryContext(std::unique_ptr<MCContext> Ctx,
       InstPrinter(std::move(InstPrinter)), MIA(std::move(MIA)),
       MIB(std::move(MIB)), MRI(std::move(MRI)), DisAsm(std::move(DisAsm)) {
   Relocation::Arch = this->TheTriple->getArch();
+  RegularPageSize = isAArch64() ? RegularPageSizeAArch64 : RegularPageSizeX86;
   PageAlign = opts::NoHugePages ? RegularPageSize : HugePageSize;
 }