LLVM and SPIRV-LLVM-Translator pulldown (WW46)

bolt/include/bolt/Passes/Hugify.h

@@ -0,0 +1,29 @@
+//===- bolt/Passes/Hugify.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef BOLT_PASSES_HUGIFY_H
+#define BOLT_PASSES_HUGIFY_H
+
+#include "bolt/Passes/BinaryPasses.h"
+
+namespace llvm {
+namespace bolt {
+
+class HugePage : public BinaryFunctionPass {
+public:
+  HugePage(const cl::opt<bool> &PrintPass) : BinaryFunctionPass(PrintPass) {}
+
+  void runOnFunctions(BinaryContext &BC) override;
+
+  const char *getName() const override { return "HugePage"; }
+};
+
+} // namespace bolt
+} // namespace llvm
+
+#endif

bolt/include/bolt/RuntimeLibs/HugifyRuntimeLibrary.h

@@ -22,13 +22,11 @@ class HugifyRuntimeLibrary : public RuntimeLibrary {
 public:
   /// Add custom section names generated by the runtime libraries to \p
   /// SecNames.
-  void addRuntimeLibSections(std::vector<std::string> &SecNames) const final {
-    SecNames.push_back(".bolt.hugify.entries");
-  }
+  void addRuntimeLibSections(std::vector<std::string> &SecNames) const final {}
 
   void adjustCommandLineOptions(const BinaryContext &BC) const final;
 
-  void emitBinary(BinaryContext &BC, MCStreamer &Streamer) final;
+  void emitBinary(BinaryContext &BC, MCStreamer &Streamer) final {}
 
   void link(BinaryContext &BC, StringRef ToolPath, RuntimeDyld &RTDyld,
             std::function<void(RuntimeDyld &)> OnLoad) final;

bolt/include/bolt/Utils/CommandLineOpts.h

@@ -44,6 +44,7 @@ extern llvm::cl::opt<unsigned long long> HeatmapMinAddress;
 extern llvm::cl::opt<bool> HotData;
 extern llvm::cl::opt<bool> HotFunctionsAtEnd;
 extern llvm::cl::opt<bool> HotText;
+extern llvm::cl::opt<bool> Hugify;
 extern llvm::cl::opt<bool> Instrument;
 extern llvm::cl::opt<std::string> OutputFilename;
 extern llvm::cl::opt<std::string> PerfData;

bolt/lib/Core/BinaryEmitter.cpp

@@ -740,10 +740,12 @@ void BinaryEmitter::emitJumpTables(const BinaryFunction &BF) {
 
   for (auto &JTI : BF.jumpTables()) {
     JumpTable &JT = *JTI.second;
+    // Only emit shared jump tables once, when processing the first parent
+    if (JT.Parents.size() > 1 && JT.Parents[0] != &BF)
+      continue;
     if (opts::PrintJumpTables)
       JT.print(outs());
-    if ((opts::JumpTables == JTS_BASIC || !BF.isSimple()) &&
-        BC.HasRelocations) {
+    if (opts::JumpTables == JTS_BASIC && BC.HasRelocations) {
       JT.updateOriginal();
     } else {
       MCSection *HotSection, *ColdSection;

bolt/lib/Passes/CMakeLists.txt

@@ -15,6 +15,7 @@ add_llvm_library(LLVMBOLTPasses
   FrameOptimizer.cpp
   HFSort.cpp
   HFSortPlus.cpp
+  Hugify.cpp
   IdenticalCodeFolding.cpp
   IndirectCallPromotion.cpp
   Inliner.cpp

bolt/lib/Passes/Hugify.cpp

@@ -0,0 +1,50 @@
+//===--- bolt/Passes/Hugify.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "bolt/Passes/Hugify.h"
+#include "llvm/Support/CommandLine.h"
+
+#define DEBUG_TYPE "bolt-hugify"
+
+using namespace llvm;
+
+namespace llvm {
+namespace bolt {
+
+void HugePage::runOnFunctions(BinaryContext &BC) {
+  auto *RtLibrary = BC.getRuntimeLibrary();
+  if (!RtLibrary || !BC.isELF() || !BC.StartFunctionAddress) {
+    return;
+  }
+
+  auto createSimpleFunction =
+      [&](std::string Title, std::vector<MCInst> Instrs) -> BinaryFunction * {
+    BinaryFunction *Func = BC.createInjectedBinaryFunction(Title);
+
+    std::vector<std::unique_ptr<BinaryBasicBlock>> BBs;
+    BBs.emplace_back(Func->createBasicBlock(nullptr));
+    BBs.back()->addInstructions(Instrs.begin(), Instrs.end());
+    BBs.back()->setCFIState(0);
+    BBs.back()->setOffset(BinaryBasicBlock::INVALID_OFFSET);
+
+    Func->insertBasicBlocks(nullptr, std::move(BBs),
+                            /*UpdateLayout=*/true,
+                            /*UpdateCFIState=*/false);
+    Func->updateState(BinaryFunction::State::CFG_Finalized);
+    return Func;
+  };
+
+  const BinaryFunction *const Start =
+      BC.getBinaryFunctionAtAddress(*BC.StartFunctionAddress);
+  assert(Start && "Entry point function not found");
+  const MCSymbol *StartSym = Start->getSymbol();
+  createSimpleFunction("__bolt_hugify_start_program",
+                       BC.MIB->createSymbolTrampoline(StartSym, BC.Ctx.get()));
+}
+} // namespace bolt
+} // namespace llvm

bolt/lib/Rewrite/BinaryPassManager.cpp

@@ -13,6 +13,7 @@
 #include "bolt/Passes/AsmDump.h"
 #include "bolt/Passes/CMOVConversion.h"
 #include "bolt/Passes/FrameOptimizer.h"
+#include "bolt/Passes/Hugify.h"
 #include "bolt/Passes/IdenticalCodeFolding.h"
 #include "bolt/Passes/IndirectCallPromotion.h"
 #include "bolt/Passes/Inliner.h"
@@ -333,6 +334,8 @@ void BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) {
 
   if (opts::Instrument)
     Manager.registerPass(std::make_unique<Instrumentation>(NeverPrint));
+  else if (opts::Hugify)
+    Manager.registerPass(std::make_unique<HugePage>(NeverPrint));
 
   Manager.registerPass(std::make_unique<ShortenInstructions>(NeverPrint));
 

bolt/lib/Rewrite/RewriteInstance.cpp

@@ -479,6 +479,11 @@ Error RewriteInstance::discoverStorage() {
   NextAvailableAddress = alignTo(NextAvailableAddress, BC->PageAlign);
   NextAvailableOffset = alignTo(NextAvailableOffset, BC->PageAlign);
 
+  // Hugify: Additional huge page from left side due to
+  // weird ASLR mapping addresses (4KB aligned)
+  if (opts::Hugify && !BC->HasFixedLoadAddress)
+    NextAvailableAddress += BC->PageAlign;
+
   if (!opts::UseGnuStack) {
     // This is where the black magic happens. Creating PHDR table in a segment
     // other than that containing ELF header is tricky. Some loaders and/or
@@ -3719,6 +3724,12 @@ void RewriteInstance::mapCodeSections(RuntimeDyld &RTDyld) {
         Address = alignTo(Address, Section->getAlignment());
         Section->setOutputAddress(Address);
         Address += Section->getOutputSize();
+
+        // Hugify: Additional huge page from right side due to
+        // weird ASLR mapping addresses (4KB aligned)
+        if (opts::Hugify && !BC->HasFixedLoadAddress &&
+            Section->getName() == BC->getMainCodeSectionName())
+          Address = alignTo(Address, Section->getAlignment());
       }
 
       // Make sure we allocate enough space for huge pages.

bolt/lib/RuntimeLibs/HugifyRuntimeLibrary.cpp

@@ -60,35 +60,6 @@ void HugifyRuntimeLibrary::adjustCommandLineOptions(
   }
 }
 
-void HugifyRuntimeLibrary::emitBinary(BinaryContext &BC, MCStreamer &Streamer) {
-  const BinaryFunction *StartFunction =
-      BC.getBinaryFunctionAtAddress(*(BC.StartFunctionAddress));
-  assert(!StartFunction->isFragment() && "expected main function fragment");
-  if (!StartFunction) {
-    errs() << "BOLT-ERROR: failed to locate function at binary start address\n";
-    exit(1);
-  }
-
-  const auto Flags = BinarySection::getFlags(/*IsReadOnly=*/false,
-                                             /*IsText=*/false,
-                                             /*IsAllocatable=*/true);
-  MCSectionELF *Section =
-      BC.Ctx->getELFSection(".bolt.hugify.entries", ELF::SHT_PROGBITS, Flags);
-
-  // __bolt_hugify_init_ptr stores the poiter the hugify library needs to
-  // jump to after finishing the init code.
-  MCSymbol *InitPtr = BC.Ctx->getOrCreateSymbol("__bolt_hugify_init_ptr");
-
-  Section->setAlignment(llvm::Align(BC.RegularPageSize));
-  Streamer.switchSection(Section);
-
-  Streamer.emitLabel(InitPtr);
-  Streamer.emitSymbolAttribute(InitPtr, MCSymbolAttr::MCSA_Global);
-  Streamer.emitValue(
-      MCSymbolRefExpr::create(StartFunction->getSymbol(), *(BC.Ctx)),
-      /*Size=*/8);
-}
-
 void HugifyRuntimeLibrary::link(BinaryContext &BC, StringRef ToolPath,
                                 RuntimeDyld &RTDyld,
                                 std::function<void(RuntimeDyld &)> OnLoad) {

bolt/runtime/CMakeLists.txt

@@ -27,10 +27,11 @@ set(BOLT_RT_FLAGS
   -fno-exceptions
   -fno-rtti
   -fno-stack-protector
-  -mno-sse)
+  -mno-sse
+  -fPIE)
 
 # Don't let the compiler think it can create calls to standard libs
-target_compile_options(bolt_rt_instr PRIVATE ${BOLT_RT_FLAGS} -fPIE)
+target_compile_options(bolt_rt_instr PRIVATE ${BOLT_RT_FLAGS})
 target_include_directories(bolt_rt_instr PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 target_compile_options(bolt_rt_hugify PRIVATE ${BOLT_RT_FLAGS})
 target_include_directories(bolt_rt_hugify PRIVATE ${CMAKE_CURRENT_BINARY_DIR})

bolt/runtime/common.h

@@ -283,6 +283,22 @@ uint32_t strLen(const char *Str) {
   return Size;
 }
 
+void *strStr(const char *const Haystack, const char *const Needle) {
+  int j = 0;
+
+  for (int i = 0; i < strLen(Haystack); i++) {
+    if (Haystack[i] == Needle[0]) {
+      for (j = 1; j < strLen(Needle); j++) {
+        if (Haystack[i + j] != Needle[j])
+          break;
+      }
+      if (j == strLen(Needle))
+        return (void *)&Haystack[i];
+    }
+  }
+  return nullptr;
+}
+
 void reportNumber(const char *Msg, uint64_t Num, uint32_t Base) {
   char Buf[BufSize];
   char *Ptr = Buf;
@@ -310,6 +326,25 @@ unsigned long hexToLong(const char *Str, char Terminator = '\0') {
   return Res;
 }
 
+/// Starting from character at \p buf, find the longest consecutive sequence
+/// of digits (0-9) and convert it to uint32_t. The converted value
+/// is put into \p ret. \p end marks the end of the buffer to avoid buffer
+/// overflow. The function \returns whether a valid uint32_t value is found.
+/// \p buf will be updated to the next character right after the digits.
+static bool scanUInt32(const char *&Buf, const char *End, uint32_t &Ret) {
+  uint64_t Result = 0;
+  const char *OldBuf = Buf;
+  while (Buf < End && ((*Buf) >= '0' && (*Buf) <= '9')) {
+    Result = Result * 10 + (*Buf) - '0';
+    ++Buf;
+  }
+  if (OldBuf != Buf && Result <= 0xFFFFFFFFu) {
+    Ret = static_cast<uint32_t>(Result);
+    return true;
+  }
+  return false;
+}
+
 #if !defined(__APPLE__)
 // We use a stack-allocated buffer for string manipulation in many pieces of
 // this code, including the code that prints each line of the fdata file. This
@@ -387,6 +422,28 @@ int __madvise(void *addr, size_t length, int advice) {
   return ret;
 }
 
+#define _UTSNAME_LENGTH 65
+
+struct UtsNameTy {
+  char sysname[_UTSNAME_LENGTH];  /* Operating system name (e.g., "Linux") */
+  char nodename[_UTSNAME_LENGTH]; /* Name within "some implementation-defined
+                      network" */
+  char release[_UTSNAME_LENGTH]; /* Operating system release (e.g., "2.6.28") */
+  char version[_UTSNAME_LENGTH]; /* Operating system version */
+  char machine[_UTSNAME_LENGTH]; /* Hardware identifier */
+  char domainname[_UTSNAME_LENGTH]; /* NIS or YP domain name */
+};
+
+int __uname(struct UtsNameTy *Buf) {
+  int Ret;
+  __asm__ __volatile__("movq $63, %%rax\n"
+                       "syscall\n"
+                       : "=a"(Ret)
+                       : "D"(Buf)
+                       : "cc", "rcx", "r11", "memory");
+  return Ret;
+}
+
 struct timespec {
   uint64_t tv_sec;  /* seconds */
   uint64_t tv_nsec; /* nanoseconds */
@@ -482,6 +539,23 @@ int __fsync(int fd) {
   return ret;
 }
 
+//              %rdi      %rsi         %rdx        %r10         %r8
+// sys_prctl  int option  unsigned    unsigned    unsigned    unsigned
+//                        long arg2   long arg3   long arg4   long arg5
+int __prctl(int Option, unsigned long Arg2, unsigned long Arg3,
+            unsigned long Arg4, unsigned long Arg5) {
+  int Ret;
+  register long rdx asm("rdx") = Arg3;
+  register long r8 asm("r8") = Arg5;
+  register long r10 asm("r10") = Arg4;
+  __asm__ __volatile__("movq $157, %%rax\n"
+                       "syscall\n"
+                       : "=a"(Ret)
+                       : "D"(Option), "S"(Arg2), "d"(rdx), "r"(r10), "r"(r8)
+                       :);
+  return Ret;
+}
+
 #endif
 
 void reportError(const char *Msg, uint64_t Size) {