[ORC] Add custom object interface support to StaticLibaryDefinitionGenerator.

This adds a GetObjectFileInterface callback member to
StaticLibraryDefinitionGenerator, and adds an optional argument for initializing
that member to StaticLibraryDefinitionGenerator's named constructors. If not
supplied, it will default to getObjectFileInterface from ObjectFileInterface.h.

To enable testing a `-hidden-l<x>` option is added to the llvm-jitlink tool.
This allows archives to be loaded with all contained symbol visibilities demoted
to hidden.

The ObjectLinkingLayer::setOverrideObjectFlagsWithResponsibilityFlags method is
(belatedly) hooked up, and enabled in llvm-jitlink when `-hidden-l<x>` is used
so that the demotion is also applied at symbol resolution time (avoiding any
"mismatched symbol flags" crashes).

Author: lhames

llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h

@@ -259,26 +259,34 @@ class DynamicLibrarySearchGenerator : public DefinitionGenerator {
 /// the containing object being added to the JITDylib.
 class StaticLibraryDefinitionGenerator : public DefinitionGenerator {
 public:
+  // Interface builder function for objects loaded from this archive.
+  using GetObjectFileInterface =
+      unique_function<Expected<MaterializationUnit::Interface>(
+          ExecutionSession &ES, MemoryBufferRef ObjBuffer)>;
+
   /// Try to create a StaticLibraryDefinitionGenerator from the given path.
   ///
   /// This call will succeed if the file at the given path is a static library
   /// is a valid archive, otherwise it will return an error.
   static Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>>
-  Load(ObjectLayer &L, const char *FileName);
+  Load(ObjectLayer &L, const char *FileName,
+       GetObjectFileInterface GetObjFileInterface = GetObjectFileInterface());
 
   /// Try to create a StaticLibraryDefinitionGenerator from the given path.
   ///
   /// This call will succeed if the file at the given path is a static library
   /// or a MachO universal binary containing a static library that is compatible
   /// with the given triple. Otherwise it will return an error.
   static Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>>
-  Load(ObjectLayer &L, const char *FileName, const Triple &TT);
+  Load(ObjectLayer &L, const char *FileName, const Triple &TT,
+       GetObjectFileInterface GetObjFileInterface = GetObjectFileInterface());
 
   /// Try to create a StaticLibrarySearchGenerator from the given memory buffer.
   /// This call will succeed if the buffer contains a valid archive, otherwise
   /// it will return an error.
   static Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>>
-  Create(ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer);
+  Create(ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer,
+         GetObjectFileInterface GetObjFileInterface = GetObjectFileInterface());
 
   Error tryToGenerate(LookupState &LS, LookupKind K, JITDylib &JD,
                       JITDylibLookupFlags JDLookupFlags,
@@ -287,9 +295,11 @@ class StaticLibraryDefinitionGenerator : public DefinitionGenerator {
 private:
   StaticLibraryDefinitionGenerator(ObjectLayer &L,
                                    std::unique_ptr<MemoryBuffer> ArchiveBuffer,
+                                   GetObjectFileInterface GetObjFileInterface,
                                    Error &Err);
 
   ObjectLayer &L;
+  GetObjectFileInterface GetObjFileInterface;
   std::unique_ptr<MemoryBuffer> ArchiveBuffer;
   std::unique_ptr<object::Archive> Archive;
 };

llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp

@@ -8,6 +8,7 @@
 
 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
 #include "llvm/ExecutionEngine/Orc/Layer.h"
+#include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalVariable.h"
@@ -269,25 +270,30 @@ Error DynamicLibrarySearchGenerator::tryToGenerate(
 }
 
 Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>>
-StaticLibraryDefinitionGenerator::Load(ObjectLayer &L, const char *FileName) {
+StaticLibraryDefinitionGenerator::Load(
+    ObjectLayer &L, const char *FileName,
+    GetObjectFileInterface GetObjFileInterface) {
   auto ArchiveBuffer = errorOrToExpected(MemoryBuffer::getFile(FileName));
 
   if (!ArchiveBuffer)
     return ArchiveBuffer.takeError();
 
-  return Create(L, std::move(*ArchiveBuffer));
+  return Create(L, std::move(*ArchiveBuffer), std::move(GetObjFileInterface));
 }
 
 Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>>
-StaticLibraryDefinitionGenerator::Load(ObjectLayer &L, const char *FileName,
-                                       const Triple &TT) {
+StaticLibraryDefinitionGenerator::Load(
+    ObjectLayer &L, const char *FileName, const Triple &TT,
+    GetObjectFileInterface GetObjFileInterface) {
+
   auto B = object::createBinary(FileName);
   if (!B)
     return B.takeError();
 
   // If this is a regular archive then create an instance from it.
   if (isa<object::Archive>(B->getBinary()))
-    return Create(L, std::move(B->takeBinary().second));
+    return Create(L, std::move(B->takeBinary().second),
+                  std::move(GetObjFileInterface));
 
   // If this is a universal binary then search for a slice matching the given
   // Triple.
@@ -309,7 +315,8 @@ StaticLibraryDefinitionGenerator::Load(ObjectLayer &L, const char *FileName,
                   " .. " + formatv("{0:x}", Obj.getOffset() + Obj.getSize()) +
                   ": " + SliceBuffer.getError().message(),
               SliceBuffer.getError());
-        return Create(L, std::move(*SliceBuffer));
+        return Create(L, std::move(*SliceBuffer),
+                      std::move(GetObjFileInterface));
       }
     }
 
@@ -326,11 +333,13 @@ StaticLibraryDefinitionGenerator::Load(ObjectLayer &L, const char *FileName,
 
 Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>>
 StaticLibraryDefinitionGenerator::Create(
-    ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer) {
+    ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer,
+    GetObjectFileInterface GetObjFileInterface) {
   Error Err = Error::success();
 
   std::unique_ptr<StaticLibraryDefinitionGenerator> ADG(
-      new StaticLibraryDefinitionGenerator(L, std::move(ArchiveBuffer), Err));
+      new StaticLibraryDefinitionGenerator(
+          L, std::move(ArchiveBuffer), std::move(GetObjFileInterface), Err));
 
   if (Err)
     return std::move(Err);
@@ -371,17 +380,28 @@ Error StaticLibraryDefinitionGenerator::tryToGenerate(
     MemoryBufferRef ChildBufferRef(ChildBufferInfo.first,
                                    ChildBufferInfo.second);
 
-    if (auto Err = L.add(JD, MemoryBuffer::getMemBuffer(ChildBufferRef, false)))
+    auto I = GetObjFileInterface(L.getExecutionSession(), ChildBufferRef);
+    if (!I)
+      return I.takeError();
+
+    if (auto Err = L.add(JD, MemoryBuffer::getMemBuffer(ChildBufferRef, false),
+                         std::move(*I)))
       return Err;
   }
 
   return Error::success();
 }
 
 StaticLibraryDefinitionGenerator::StaticLibraryDefinitionGenerator(
-    ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer, Error &Err)
-    : L(L), ArchiveBuffer(std::move(ArchiveBuffer)),
-      Archive(std::make_unique<object::Archive>(*this->ArchiveBuffer, Err)) {}
+    ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer,
+    GetObjectFileInterface GetObjFileInterface, Error &Err)
+    : L(L), GetObjFileInterface(std::move(GetObjFileInterface)),
+      ArchiveBuffer(std::move(ArchiveBuffer)),
+      Archive(std::make_unique<object::Archive>(*this->ArchiveBuffer, Err)) {
+
+  if (!this->GetObjFileInterface)
+    this->GetObjFileInterface = getObjectFileInterface;
+}
 
 } // End namespace orc.
 } // End namespace llvm.

llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp

@@ -249,7 +249,8 @@ class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
 
     {
 
-      // Check that InternedResult matches up with MR->getSymbols().
+      // Check that InternedResult matches up with MR->getSymbols(), overriding
+      // flags if requested.
       // This guards against faulty transformations / compilers / object caches.
 
       // First check that there aren't any missing symbols.
@@ -258,16 +259,20 @@ class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
       SymbolNameVector MissingSymbols;
       for (auto &KV : MR->getSymbols()) {
 
+        auto I = InternedResult.find(KV.first);
+
         // If this is a materialization-side-effects only symbol then bump
         // the counter and make sure it's *not* defined, otherwise make
         // sure that it is defined.
         if (KV.second.hasMaterializationSideEffectsOnly()) {
           ++NumMaterializationSideEffectsOnlySymbols;
-          if (InternedResult.count(KV.first))
+          if (I != InternedResult.end())
             ExtraSymbols.push_back(KV.first);
           continue;
-        } else if (!InternedResult.count(KV.first))
+        } else if (I == InternedResult.end())
           MissingSymbols.push_back(KV.first);
+        else if (Layer.OverrideObjectFlags)
+          I->second.setFlags(KV.second);
       }
 
       // If there were missing symbols then report the error.

llvm/test/ExecutionEngine/JITLink/X86/MachO_archive_load_hidden_expect_failure.s

@@ -0,0 +1,25 @@
+# RUN: rm -rf %t && mkdir -p %t
+# RUN: llvm-mc -triple x86_64-apple-macosx10.9 -filetype=obj \
+# RUN:   -o %t/MachO_extra_def_strong.o %S/Inputs/MachO_extra_def_strong.s
+# RUN: llvm-ar crs %t/libExtraDef.a %t/MachO_extra_def_strong.o
+# RUN: llvm-mc -triple x86_64-apple-macosx10.9 -filetype=obj \
+# RUN:   -o %t/MachO_archive_load_hidden_support.o %s
+# RUN: not llvm-jitlink -noexec %t/MachO_archive_load_hidden_support.o -lFoo \
+# RUN:   -jd Foo -L%t -hidden-lExtraDef
+#
+# Expect this test to fail when we try to reference ExtraDef, which should have
+# be hidden in JITDylib Foo. This tests that we're correctly overriding the
+# object interface when we load the object.
+
+        .section  __TEXT,__text,regular,pure_instructions
+
+        .globl  _main
+        .p2align  4, 0x90
+_main:
+        retq
+
+	.section	__DATA,__data
+	.globl	ExtraDefRef
+	.p2align	3
+ExtraDefRef:
+	.quad	ExtraDef

llvm/test/ExecutionEngine/JITLink/X86/MachO_archive_load_hidden_expect_success.s

@@ -0,0 +1,25 @@
+# RUN: rm -rf %t && mkdir -p %t
+# RUN: llvm-mc -triple x86_64-apple-macosx10.9 -filetype=obj \
+# RUN:   -o %t/MachO_extra_def_strong.o %S/Inputs/MachO_extra_def_strong.s
+# RUN: llvm-ar crs %t/libExtraDef.a %t/MachO_extra_def_strong.o
+# RUN: llvm-mc -triple x86_64-apple-macosx10.9 -filetype=obj \
+# RUN:   -o %t/MachO_archive_load_hidden_support.o %s
+# RUN: llvm-jitlink -noexec %t/MachO_archive_load_hidden_support.o \
+# RUN:   -L%t -hidden-lExtraDef
+#
+# Expect this test to succeed -- ExtraDef should be hidden, but visible to
+# ExtraDefRef as they're linked in the same JITDylib. This tests that we're
+# correctly handling the change in object interface when linking ExtraDef.
+
+        .section  __TEXT,__text,regular,pure_instructions
+
+        .globl  _main
+        .p2align  4, 0x90
+_main:
+        retq
+
+	.section	__DATA,__data
+	.globl	ExtraDefRef
+	.p2align	3
+ExtraDefRef:
+	.quad	ExtraDef

llvm/tools/llvm-jitlink/llvm-jitlink.cpp

@@ -82,6 +82,10 @@ static cl::list<std::string>
               cl::desc("Link against library X in the library search paths"),
               cl::Prefix, cl::cat(JITLinkCategory));
 
+static cl::list<std::string> LibrariesHidden(
+    "hidden-l", cl::desc("Link against library X in the library search paths"),
+    cl::Prefix, cl::cat(JITLinkCategory));
+
 static cl::opt<bool> NoExec("noexec", cl::desc("Do not execute loaded code"),
                             cl::init(false), cl::cat(JITLinkCategory));
 
@@ -1413,9 +1417,20 @@ static Error addObjects(Session &S,
   return Error::success();
 }
 
+static Expected<MaterializationUnit::Interface>
+getObjectFileInterfaceHidden(ExecutionSession &ES, MemoryBufferRef ObjBuffer) {
+  auto I = getObjectFileInterface(ES, ObjBuffer);
+  if (I) {
+    for (auto &KV : I->SymbolFlags)
+      KV.second &= ~JITSymbolFlags::Exported;
+  }
+  return I;
+}
+
 static Error addLibraries(Session &S,
                           const std::map<unsigned, JITDylib *> &IdxToJD) {
 
+  // 1. Collect search paths for each JITDylib.
   DenseMap<const JITDylib *, SmallVector<StringRef, 2>> JDSearchPaths;
 
   for (auto LSPItr = LibrarySearchPaths.begin(),
@@ -1447,16 +1462,58 @@ static Error addLibraries(Session &S,
     }
   });
 
+  // 2. Collect library loads from -lx, -hidden-lx.
+  struct LibraryLoad {
+    StringRef LibName;
+    unsigned Position;
+    StringRef *CandidateExtensions;
+    enum { Standard, Hidden } Modifier;
+  };
+  std::vector<LibraryLoad> LibraryLoads;
+  StringRef StandardExtensions[] = {".so", ".dylib", ".a"};
+  StringRef ArchiveExtensionsOnly[] = {".a"};
+
+  // Add -lx arguments to LibraryLoads.
   for (auto LibItr = Libraries.begin(), LibEnd = Libraries.end();
        LibItr != LibEnd; ++LibItr) {
+    LibraryLoad LL;
+    LL.LibName = *LibItr;
+    LL.Position = Libraries.getPosition(LibItr - Libraries.begin());
+    LL.CandidateExtensions = StandardExtensions;
+    LL.Modifier = LibraryLoad::Standard;
+    LibraryLoads.push_back(std::move(LL));
+  }
+
+  // Add -hidden-lx arguments to LibraryLoads.
+  for (auto LibHiddenItr = LibrariesHidden.begin(),
+            LibHiddenEnd = LibrariesHidden.end();
+       LibHiddenItr != LibHiddenEnd; ++LibHiddenItr) {
+    LibraryLoad LL;
+    LL.LibName = *LibHiddenItr;
+    LL.Position =
+        LibrariesHidden.getPosition(LibHiddenItr - LibrariesHidden.begin());
+    LL.CandidateExtensions = ArchiveExtensionsOnly;
+    LL.Modifier = LibraryLoad::Hidden;
+    LibraryLoads.push_back(std::move(LL));
+  }
+
+  // If there are any load-<modified> options then turn on flag overrides
+  // to avoid flag mismatch errors.
+  if (!LibrariesHidden.empty())
+    S.ObjLayer.setOverrideObjectFlagsWithResponsibilityFlags(true);
+
+  // Sort library loads by position in the argument list.
+  llvm::sort(LibraryLoads, [](const LibraryLoad &LHS, const LibraryLoad &RHS) {
+    return LHS.Position < RHS.Position;
+  });
 
+  // 3. Process library loads.
+  for (auto &LL : LibraryLoads) {
     bool LibFound = false;
-    StringRef LibName(*LibItr);
-    unsigned LibIdx = Libraries.getPosition(LibItr - Libraries.begin());
-    auto &JD = *std::prev(IdxToJD.lower_bound(LibIdx))->second;
+    auto &JD = *std::prev(IdxToJD.lower_bound(LL.Position))->second;
 
     // If this is the name of a JITDylib then link against that.
-    if (auto *LJD = S.ES.getJITDylibByName(LibName)) {
+    if (auto *LJD = S.ES.getJITDylibByName(LL.LibName)) {
       JD.addToLinkOrder(*LJD);
       continue;
     }
@@ -1467,10 +1524,11 @@ static Error addLibraries(Session &S,
       for (StringRef SearchPath : JDSearchPathsItr->second) {
         for (const char *LibExt : {".dylib", ".so", ".a"}) {
           SmallVector<char, 256> LibPath;
-          LibPath.reserve(SearchPath.size() + strlen("lib") + LibName.size() +
-                          strlen(LibExt) + 2); // +2 for pathsep, null term.
+          LibPath.reserve(SearchPath.size() + strlen("lib") +
+                          LL.LibName.size() + strlen(LibExt) +
+                          2); // +2 for pathsep, null term.
           llvm::copy(SearchPath, std::back_inserter(LibPath));
-          sys::path::append(LibPath, "lib" + LibName + LibExt);
+          sys::path::append(LibPath, "lib" + LL.LibName + LibExt);
           LibPath.push_back('\0');
 
           // Skip missing or non-regular paths.
@@ -1513,9 +1571,21 @@ static Error addLibraries(Session &S,
           }
           case file_magic::archive:
           case file_magic::macho_universal_binary: {
+            unique_function<Expected<MaterializationUnit::Interface>(
+                ExecutionSession & ES, MemoryBufferRef ObjBuffer)>
+                GetObjFileInterface;
+            switch (LL.Modifier) {
+            case LibraryLoad::Standard:
+              GetObjFileInterface = getObjectFileInterface;
+              break;
+            case LibraryLoad::Hidden:
+              GetObjFileInterface = getObjectFileInterfaceHidden;
+              break;
+            }
             auto G = StaticLibraryDefinitionGenerator::Load(
                 S.ObjLayer, LibPath.data(),
-                S.ES.getExecutorProcessControl().getTargetTriple());
+                S.ES.getExecutorProcessControl().getTargetTriple(),
+                std::move(GetObjFileInterface));
             if (!G)
               return G.takeError();
             JD.addGenerator(std::move(*G));
@@ -1545,7 +1615,7 @@ static Error addLibraries(Session &S,
     if (!LibFound)
       return make_error<StringError>("While linking " + JD.getName() +
                                          ", could not find library for -l" +
-                                         LibName,
+                                         LL.LibName,
                                      inconvertibleErrorCode());
   }
 

llvm/tools/llvm-jitlink/llvm-jitlink.h

@@ -32,6 +32,7 @@ namespace llvm {
 struct Session;
 
 struct Session {
+
   orc::ExecutionSession ES;
   orc::JITDylib *MainJD = nullptr;
   orc::ObjectLinkingLayer ObjLayer;