LLVM and SPIRV-LLVM-Translator pulldown (WW27)

bolt/include/bolt/Core/BinaryContext.h

@@ -498,6 +498,14 @@ class BinaryContext {
   /// to function \p BF.
   std::string generateJumpTableName(const BinaryFunction &BF, uint64_t Address);
 
+  /// Free memory used by jump table offsets
+  void clearJumpTableOffsets() {
+    for (auto &JTI : JumpTables) {
+      JumpTable &JT = *JTI.second;
+      JumpTable::OffsetsType Temp;
+      Temp.swap(JT.OffsetEntries);
+    }
+  }
   /// Return true if the array of bytes represents a valid code padding.
   bool hasValidCodePadding(const BinaryFunction &BF);
 

bolt/include/bolt/Core/MCPlusBuilder.h

@@ -1046,9 +1046,14 @@ class MCPlusBuilder {
   /// Return handler and action info for invoke instruction if present.
   Optional<MCPlus::MCLandingPad> getEHInfo(const MCInst &Inst) const;
 
-  // Add handler and action info for call instruction.
+  /// Add handler and action info for call instruction.
   void addEHInfo(MCInst &Inst, const MCPlus::MCLandingPad &LP);
 
+  /// Update exception-handling info for the invoke instruction \p Inst.
+  /// Return true on success and false otherwise, e.g. if the instruction is
+  /// not an invoke.
+  bool updateEHInfo(MCInst &Inst, const MCPlus::MCLandingPad &LP);
+
   /// Return non-negative GNU_args_size associated with the instruction
   /// or -1 if there's no associated info.
   int64_t getGnuArgsSize(const MCInst &Inst) const;

bolt/include/bolt/Passes/SplitFunctions.h

@@ -31,6 +31,13 @@ class SplitFunctions : public BinaryFunctionPass {
   /// Split function body into fragments.
   void splitFunction(BinaryFunction &Function);
 
+  /// Create trampoline landing pads for exception handling code to guarantee
+  /// that every landing pad is placed in the same function fragment as the
+  /// corresponding thrower block. The trampoline landing pad, when created,
+  /// will redirect the execution to the real landing pad in a different
+  /// fragment.
+  void createEHTrampolines(BinaryFunction &Function) const;
+
   std::atomic<uint64_t> SplitBytesHot{0ull};
   std::atomic<uint64_t> SplitBytesCold{0ull};
 

bolt/lib/Core/BinaryContext.cpp

@@ -764,12 +764,31 @@ BinaryFunction *BinaryContext::createBinaryFunction(
 const MCSymbol *
 BinaryContext::getOrCreateJumpTable(BinaryFunction &Function, uint64_t Address,
                                     JumpTable::JumpTableType Type) {
+  auto isFragmentOf = [](BinaryFunction *Fragment, BinaryFunction *Parent) {
+    return (Fragment->isFragment() && Fragment->isParentFragment(Parent));
+  };
+
   if (JumpTable *JT = getJumpTableContainingAddress(Address)) {
     assert(JT->Type == Type && "jump table types have to match");
-    assert(JT->Parent == &Function &&
+    bool HasMultipleParents = isFragmentOf(JT->Parent, &Function) ||
+                              isFragmentOf(&Function, JT->Parent);
+    assert((JT->Parent == &Function || HasMultipleParents) &&
            "cannot re-use jump table of a different function");
     assert(Address == JT->getAddress() && "unexpected non-empty jump table");
 
+    // Flush OffsetEntries with INVALID_OFFSET if multiple parents
+    // Duplicate the entry for the parent function for easy access
+    if (HasMultipleParents) {
+      if (opts::Verbosity > 2) {
+        outs() << "BOLT-WARNING: Multiple fragments access same jump table: "
+               << JT->Parent->getPrintName() << "; " << Function.getPrintName()
+               << "\n";
+      }
+      constexpr uint64_t INVALID_OFFSET = std::numeric_limits<uint64_t>::max();
+      for (unsigned I = 0; I < JT->OffsetEntries.size(); ++I)
+        JT->OffsetEntries[I] = INVALID_OFFSET;
+      Function.JumpTables.emplace(Address, JT);
+    }
     return JT->getFirstLabel();
   }
 

bolt/lib/Core/BinaryEmitter.cpp

@@ -912,8 +912,8 @@ void BinaryEmitter::emitLSDA(BinaryFunction &BF, bool EmitColdPart) {
   // defined in the same section and hence cannot place the landing pad into a
   // cold fragment when the corresponding call site is in the hot fragment.
   // Because of this issue and the previously described issue of possible
-  // zero-offset landing pad we disable splitting of exception-handling
-  // code for shared objects.
+  // zero-offset landing pad we have to place landing pads in the same section
+  // as the corresponding invokes for shared objects.
   std::function<void(const MCSymbol *)> emitLandingPad;
   if (BC.HasFixedLoadAddress) {
     Streamer.emitIntValue(dwarf::DW_EH_PE_udata4, 1); // LPStart format
@@ -925,8 +925,6 @@ void BinaryEmitter::emitLSDA(BinaryFunction &BF, bool EmitColdPart) {
         Streamer.emitSymbolValue(LPSymbol, 4);
     };
   } else {
-    assert(!EmitColdPart &&
-           "cannot have exceptions in cold fragment for shared object");
     Streamer.emitIntValue(dwarf::DW_EH_PE_omit, 1); // LPStart format
     emitLandingPad = [&](const MCSymbol *LPSymbol) {
       if (!LPSymbol)

bolt/lib/Core/BinaryFunction.cpp

@@ -1654,11 +1654,13 @@ void BinaryFunction::postProcessJumpTables() {
                 "detected in function "
              << *this << '\n';
     }
-    for (unsigned I = 0; I < JT.OffsetEntries.size(); ++I) {
-      MCSymbol *Label =
-          getOrCreateLocalLabel(getAddress() + JT.OffsetEntries[I],
-                                /*CreatePastEnd*/ true);
-      JT.Entries.push_back(Label);
+    if (JT.Entries.empty()) {
+      for (unsigned I = 0; I < JT.OffsetEntries.size(); ++I) {
+        MCSymbol *Label =
+            getOrCreateLocalLabel(getAddress() + JT.OffsetEntries[I],
+                                  /*CreatePastEnd*/ true);
+        JT.Entries.push_back(Label);
+      }
     }
 
     const uint64_t BDSize =
@@ -1700,12 +1702,6 @@ void BinaryFunction::postProcessJumpTables() {
   }
   clearList(JTSites);
 
-  // Free memory used by jump table offsets.
-  for (auto &JTI : JumpTables) {
-    JumpTable &JT = *JTI.second;
-    clearList(JT.OffsetEntries);
-  }
-
   // Conservatively populate all possible destinations for unknown indirect
   // branches.
   if (opts::StrictMode && hasInternalReference()) {

bolt/lib/Core/MCPlusBuilder.cpp

@@ -159,6 +159,17 @@ void MCPlusBuilder::addEHInfo(MCInst &Inst, const MCLandingPad &LP) {
   }
 }
 
+bool MCPlusBuilder::updateEHInfo(MCInst &Inst, const MCLandingPad &LP) {
+  if (!isInvoke(Inst))
+    return false;
+
+  setAnnotationOpValue(Inst, MCAnnotation::kEHLandingPad,
+                       reinterpret_cast<int64_t>(LP.first));
+  setAnnotationOpValue(Inst, MCAnnotation::kEHAction,
+                       static_cast<int64_t>(LP.second));
+  return true;
+}
+
 int64_t MCPlusBuilder::getGnuArgsSize(const MCInst &Inst) const {
   Optional<int64_t> Value =
       getAnnotationOpValue(Inst, MCAnnotation::kGnuArgsSize);

bolt/lib/Passes/SplitFunctions.cpp

@@ -118,7 +118,7 @@ void SplitFunctions::splitFunction(BinaryFunction &BF) {
 
   bool AllCold = true;
   for (BinaryBasicBlock *BB : BF.layout()) {
-    uint64_t ExecCount = BB->getExecutionCount();
+    const uint64_t ExecCount = BB->getExecutionCount();
     if (ExecCount == BinaryBasicBlock::COUNT_NO_PROFILE)
       return;
     if (ExecCount != 0)
@@ -140,12 +140,12 @@ void SplitFunctions::splitFunction(BinaryFunction &BF) {
                       << " pre-split is <0x"
                       << Twine::utohexstr(OriginalHotSize) << ", 0x"
                       << Twine::utohexstr(ColdSize) << ">\n");
-  }
-
-  if (opts::SplitFunctions == SplitFunctions::ST_LARGE && !BC.HasRelocations) {
-    // Split only if the function wouldn't fit.
-    if (OriginalHotSize <= BF.getMaxSize())
-      return;
+    if (opts::SplitFunctions == SplitFunctions::ST_LARGE &&
+        !BC.HasRelocations) {
+      // Split only if the function wouldn't fit.
+      if (OriginalHotSize <= BF.getMaxSize())
+        return;
+    }
   }
 
   // Never outline the first basic block.
@@ -164,9 +164,9 @@ void SplitFunctions::splitFunction(BinaryFunction &BF) {
       BB->setCanOutline(false);
       continue;
     }
+
     if (BF.hasEHRanges() && !opts::SplitEH) {
-      // We cannot move landing pads (or rather entry points for landing
-      // pads).
+      // We cannot move landing pads (or rather entry points for landing pads).
       if (BB->isLandingPad()) {
         BB->setCanOutline(false);
         continue;
@@ -176,7 +176,7 @@ void SplitFunctions::splitFunction(BinaryFunction &BF) {
       // that the block never throws, it is safe to move the block to
       // decrease the size of the function.
       for (MCInst &Instr : *BB) {
-        if (BF.getBinaryContext().MIB->isInvoke(Instr)) {
+        if (BC.MIB->isInvoke(Instr)) {
           BB->setCanOutline(false);
           break;
         }
@@ -214,6 +214,12 @@ void SplitFunctions::splitFunction(BinaryFunction &BF) {
     BB->setIsCold(true);
   }
 
+  // For shared objects, place invoke instructions and corresponding landing
+  // pads in the same fragment. To reduce hot code size, create trampoline
+  // landing pads that will redirect the execution to the real LP.
+  if (!BC.HasFixedLoadAddress && BF.hasEHRanges() && BF.isSplit())
+    createEHTrampolines(BF);
+
   // Check the new size to see if it's worth splitting the function.
   if (BC.isX86() && BF.isSplit()) {
     std::tie(HotSize, ColdSize) = BC.calculateEmittedSize(BF);
@@ -237,5 +243,65 @@ void SplitFunctions::splitFunction(BinaryFunction &BF) {
   }
 }
 
+void SplitFunctions::createEHTrampolines(BinaryFunction &BF) const {
+  const auto &MIB = BF.getBinaryContext().MIB;
+
+  // Map real landing pads to the corresponding trampolines.
+  std::unordered_map<const MCSymbol *, const MCSymbol *> LPTrampolines;
+
+  // Iterate over the copy of basic blocks since we are adding new blocks to the
+  // function which will invalidate its iterators.
+  std::vector<BinaryBasicBlock *> Blocks(BF.pbegin(), BF.pend());
+  for (BinaryBasicBlock *BB : Blocks) {
+    for (MCInst &Instr : *BB) {
+      const Optional<MCPlus::MCLandingPad> EHInfo = MIB->getEHInfo(Instr);
+      if (!EHInfo || !EHInfo->first)
+        continue;
+
+      const MCSymbol *LPLabel = EHInfo->first;
+      BinaryBasicBlock *LPBlock = BF.getBasicBlockForLabel(LPLabel);
+      if (BB->isCold() == LPBlock->isCold())
+        continue;
+
+      const MCSymbol *TrampolineLabel = nullptr;
+      auto Iter = LPTrampolines.find(LPLabel);
+      if (Iter != LPTrampolines.end()) {
+        TrampolineLabel = Iter->second;
+      } else {
+        // Create a trampoline basic block in the same fragment as the thrower.
+        // Note: there's no need to insert the jump instruction, it will be
+        // added by fixBranches().
+        BinaryBasicBlock *TrampolineBB = BF.addBasicBlock();
+        TrampolineBB->setIsCold(BB->isCold());
+        TrampolineBB->setExecutionCount(LPBlock->getExecutionCount());
+        TrampolineBB->addSuccessor(LPBlock, TrampolineBB->getExecutionCount());
+        TrampolineBB->setCFIState(LPBlock->getCFIState());
+        TrampolineLabel = TrampolineBB->getLabel();
+        LPTrampolines.emplace(std::make_pair(LPLabel, TrampolineLabel));
+      }
+
+      // Substitute the landing pad with the trampoline.
+      MIB->updateEHInfo(Instr,
+                        MCPlus::MCLandingPad(TrampolineLabel, EHInfo->second));
+    }
+  }
+
+  if (LPTrampolines.empty())
+    return;
+
+  // All trampoline blocks were added to the end of the function. Place them at
+  // the end of corresponding fragments.
+  std::stable_sort(BF.layout_begin(), BF.layout_end(),
+                   [&](BinaryBasicBlock *A, BinaryBasicBlock *B) {
+                     return A->isCold() < B->isCold();
+                   });
+
+  // Conservatively introduce branch instructions.
+  BF.fixBranches();
+
+  // Update exception-handling CFG for the function.
+  BF.recomputeLandingPads();
+}
+
 } // namespace bolt
 } // namespace llvm

bolt/lib/Passes/ValidateInternalCalls.cpp

@@ -106,6 +106,7 @@ bool ValidateInternalCalls::fixCFGForPIC(BinaryFunction &Function) const {
         // block.
         std::vector<std::unique_ptr<BinaryBasicBlock>> NewBBs;
         NewBBs.emplace_back(Function.createBasicBlock());
+        NewBBs.back()->setOffset(0);
         NewBBs.back()->addInstructions(MovedInsts.begin(), MovedInsts.end());
         BB.moveAllSuccessorsTo(NewBBs.back().get());
         Function.insertBasicBlocks(&BB, std::move(NewBBs));

bolt/lib/Rewrite/RewriteInstance.cpp

@@ -1686,11 +1686,6 @@ void RewriteInstance::adjustCommandLineOptions() {
     opts::SplitEH = false;
   }
 
-  if (opts::SplitEH && !BC->HasFixedLoadAddress) {
-    errs() << "BOLT-WARNING: disabling -split-eh for shared object\n";
-    opts::SplitEH = false;
-  }
-
   if (opts::StrictMode && !BC->HasRelocations) {
     errs() << "BOLT-WARNING: disabling strict mode (-strict) in non-relocation "
               "mode\n";
@@ -2903,6 +2898,7 @@ void RewriteInstance::disassembleFunctions() {
     BC->processInterproceduralReferences(Function);
   }
 
+  BC->clearJumpTableOffsets();
   BC->populateJumpTables();
   BC->skipMarkedFragments();
 
@@ -5206,8 +5202,6 @@ uint64_t RewriteInstance::getNewFunctionAddress(uint64_t OldAddress) {
   if (!Function)
     return 0;
 
-  assert(!Function->isFragment() && "cannot get new address for a fragment");
-
   return Function->getOutputAddress();
 }
 

bolt/lib/Target/X86/X86MCPlusBuilder.cpp

@@ -3129,7 +3129,12 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     case 1: Opcode = X86::MOV8ri; break;
     case 2: Opcode = X86::MOV16ri; break;
     case 4: Opcode = X86::MOV32ri; break;
-    case 8: Opcode = X86::MOV64ri; break;
+    // Writing to a 32-bit register always zeros the upper 32 bits of the
+    // full-width register
+    case 8:
+      Opcode = X86::MOV32ri;
+      Reg = getAliasSized(Reg, 4);
+      break;
     default:
       llvm_unreachable("Unexpected size");
     }

bolt/test/X86/shared_object_entry.s

@@ -0,0 +1,38 @@
+# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o
+# RUN: ld.lld %t.o -o %t.so --shared --entry=func1.cold.1 --emit-relocs
+# RUN: llvm-bolt -relocs %t.so -o %t -reorder-functions=hfsort+ \
+# RUN:    -split-functions=3 -reorder-blocks=ext-tsp -split-all-cold \
+# RUN:    -dyno-stats -icf=1 -use-gnu-stack
+
+# Check that an entry point is a cold symbol
+# RUN: llvm-readelf -h %t.so > %t.log
+# RUN: llvm-nm %t.so >> %t.log
+# RUN: FileCheck %s --input-file %t.log
+# CHECK: Entry point address: 0x[[#%X,ENTRY:]]
+# CHECK: [[#%x,ENTRY]] {{.*}} func1.cold.1
+
+.globl func1.cold.1
+.type func1.cold.1,@function
+func1.cold.1:
+  .cfi_startproc
+.L1:
+		movq %rbx, %rdx
+		jmp .L3
+.L2:
+		# exit(0)
+		movq $60, %rax
+		xorq %rdi, %rdi
+		syscall
+  .size func1.cold.1, .-func1.cold.1
+  .cfi_endproc
+
+.globl func1
+.type func1,@function
+func1:
+  .cfi_startproc
+.L3:
+		movq %rax, %rdi
+		jmp .L2
+  call exit
+  .size func1, .-func1
+  .cfi_endproc