[lld][WebAssembly] Report undefined symbols by default -shared/-pie builds

Previously we would ignore/import all undefined symbols when using
`-shared` or `-pie`.  With this change we now track symbol in shared
libraries and report undefined symbols by default.

This can be a breaking change for any users of dyanmic linking.
However, the old behaviour is still available using
`--unresolved-symbols=import-dynamic`.  This rationale for allowing this
type of breaking change is that `-pie` and `-shared` are both still
experimental will warn as such, unless `--experimental-pic` is passed.

See emscripten-core/emscripten#18198

Author: sbc100

lld/test/wasm/Inputs/ret32.s

@@ -1,4 +1,3 @@
-  .hidden ret32
   .globl  ret32
 ret32:
   .functype ret32 (f32) -> (i32)

lld/test/wasm/dylink.s

@@ -0,0 +1,31 @@
+# RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-emscripten -o %t.o %s
+# RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-emscripten %p/Inputs/ret32.s -o %t.ret32.o
+# RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-emscripten %p/Inputs/libsearch-dyn.s -o %t.dyn.o
+# RUN: wasm-ld --experimental-pic -shared %t.ret32.o %t.dyn.o -o %t.lib.so
+# RUN: not wasm-ld --experimental-pic -pie -o %t.wasm %t.o 2>&1 | FileCheck --check-prefix=ERROR %s
+# RUN: wasm-ld --experimental-pic -pie -o %t.wasm %t.o %t.lib.so
+# RUN: obj2yaml %t.wasm | FileCheck %s
+
+# ERROR: error: {{.*}}: undefined symbol: ret32
+# ERROR: error: {{.*}}: undefined symbol: _dynamic
+.functype ret32 (f32) -> (i32)
+
+.globl _start
+_start:
+  .functype _start () -> ()
+  f32.const 0.0
+  call ret32
+  drop
+  i32.const _dynamic@GOT
+  drop
+  end_function
+
+# CHECK:      Sections:
+# CHECK-NEXT:   - Type:            CUSTOM
+# CHECK-NEXT:     Name:            dylink.0
+# CHECK-NEXT:     MemorySize:      0
+# CHECK-NEXT:     MemoryAlignment: 0
+# CHECK-NEXT:     TableSize:       0
+# CHECK-NEXT:     TableAlignment:  0
+# CHECK-NEXT:     Needed:
+# CHECK-NEXT:       - {{.*}}.lib.so

lld/test/wasm/emit-relocs.s

@@ -54,7 +54,7 @@ foo:
 # CHECK-NEXT:       - Index:           1
 # CHECK-NEXT:         Kind:            FUNCTION
 # CHECK-NEXT:         Name:            ret32
-# CHECK-NEXT:         Flags:           [ VISIBILITY_HIDDEN ]
+# CHECK-NEXT:         Flags:           [  ]
 # CHECK-NEXT:         Function:        1
 # CHECK-NEXT:       - Index:           2
 # CHECK-NEXT:         Kind:            DATA

lld/test/wasm/pie.s

@@ -1,6 +1,6 @@
 # RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-emscripten -o %t.o %s
 # RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-emscripten %S/Inputs/internal_func.s -o %t.internal_func.o
-# RUN: wasm-ld --no-gc-sections --experimental-pic -pie -o %t.wasm %t.o %t.internal_func.o
+# RUN: wasm-ld --no-gc-sections --experimental-pic -pie --unresolved-symbols=import-dynamic -o %t.wasm %t.o %t.internal_func.o
 # RUN: obj2yaml %t.wasm | FileCheck %s
 # RUN: llvm-objdump --disassemble-symbols=__wasm_call_ctors,__wasm_apply_data_relocs --no-show-raw-insn --no-leading-addr %t.wasm | FileCheck %s --check-prefixes DISASSEM
 
@@ -150,7 +150,7 @@ _start:
 # instruction in the InitExpr.  We also, therefore, do not need these globals
 # to be mutable.
 
-# RUN: wasm-ld --no-gc-sections --experimental-pic -pie --extra-features=extended-const -o %t.extended.wasm %t.o %t.internal_func.o
+# RUN: wasm-ld --no-gc-sections --experimental-pic -pie --unresolved-symbols=import-dynamic --extra-features=extended-const -o %t.extended.wasm %t.o %t.internal_func.o
 # RUN: obj2yaml %t.extended.wasm | FileCheck %s --check-prefix=EXTENDED-CONST
 
 # EXTENDED-CONST-NOT: __wasm_apply_global_relocs
@@ -207,7 +207,7 @@ _start:
 # to be generated along with __wasm_start as the start
 # function.
 
-# RUN: wasm-ld --no-gc-sections --shared-memory --experimental-pic -pie -o %t.shmem.wasm %t.o %t.internal_func.o
+# RUN: wasm-ld --no-gc-sections --shared-memory --experimental-pic -pie --unresolved-symbols=import-dynamic -o %t.shmem.wasm %t.o %t.internal_func.o
 # RUN: obj2yaml %t.shmem.wasm | FileCheck %s --check-prefix=SHMEM
 # RUN: llvm-objdump --disassemble-symbols=__wasm_start --no-show-raw-insn --no-leading-addr %t.shmem.wasm | FileCheck %s --check-prefix DISASSEM-SHMEM
 

lld/test/wasm/shared-needed.s

@@ -1,17 +1,28 @@
 # RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown -o %t.o %s
 # RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown %p/Inputs/ret32.s -o %t.ret32.o
 
-# RUN: wasm-ld -shared --experimental-pic -o %t1.so %t.o
-# RUN: obj2yaml %t1.so | FileCheck %s -check-prefix=SO1
+# RUN: wasm-ld -shared --experimental-pic -o %t.ret32.so %t.ret32.o
+# RUN: obj2yaml %t.ret32.so | FileCheck %s -check-prefix=SO1
+
+# Without linking against the ret32.so shared object we expect and undefined
+# symbol error
+
+# RUN: not wasm-ld -shared --experimental-pic -o %t.so %t.o 2>&1 | FileCheck %s --check-prefix=ERROR
+# ERROR: undefined symbol: ret32
+
+# RUN: wasm-ld -shared --experimental-pic -o %t.so %t.o %t.ret32.so
+# RUN: obj2yaml %t.so | FileCheck %s -check-prefix=SO2
 
-# RUN: wasm-ld -shared --experimental-pic -o %t2.so %t1.so %t.ret32.o
-# RUN: obj2yaml %t2.so | FileCheck %s -check-prefix=SO2
 
 .globl foo
 .globl data
 
+.functype ret32 (f32) -> (i32)
+
 foo:
-  .functype foo () -> ()
+  .functype foo (f32) -> (i32)
+  local.get 0
+  call ret32
   end_function
 
 .section .data,"",@
@@ -24,8 +35,8 @@ data:
 # SO1:      Sections:
 # SO1-NEXT:   - Type:            CUSTOM
 # SO1-NEXT:     Name:            dylink.0
-# SO1-NEXT:     MemorySize:      4
-# SO1-NEXT:     MemoryAlignment: 2
+# SO1-NEXT:     MemorySize:      0
+# SO1-NEXT:     MemoryAlignment: 0
 # SO1-NEXT:     TableSize:       0
 # SO1-NEXT:     TableAlignment:  0
 # SO1-NEXT:     Needed:          []
@@ -34,10 +45,10 @@ data:
 # SO2:      Sections:
 # SO2-NEXT:   - Type:            CUSTOM
 # SO2-NEXT:     Name:            dylink.0
-# SO2-NEXT:     MemorySize:      0
-# SO2-NEXT:     MemoryAlignment: 0
+# SO2-NEXT:     MemorySize:      4
+# SO2-NEXT:     MemoryAlignment: 2
 # SO2-NEXT:     TableSize:       0
 # SO2-NEXT:     TableAlignment:  0
 # SO2-NEXT:     Needed:
-# SO2-NEXT:       - shared-needed.s.tmp1.so
+# SO2-NEXT:       - shared-needed.s.tmp.ret32.so
 # SO2-NEXT:   - Type:            TYPE

lld/test/wasm/shared.s

@@ -1,14 +1,14 @@
 # RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown -o %t.o %s
-# RUN: wasm-ld --experimental-pic -shared -o %t.wasm %t.o
+# RUN: wasm-ld --experimental-pic --unresolved-symbols=import-dynamic -shared -o %t.wasm %t.o
 # RUN: obj2yaml %t.wasm | FileCheck %s
 # RUN: llvm-objdump --disassemble-symbols=__wasm_call_ctors,__wasm_apply_data_relocs --no-show-raw-insn --no-leading-addr %t.wasm | FileCheck %s --check-prefixes DIS
 
 .functype func_external () -> ()
 
 # Linker-synthesized globals
 .globaltype __stack_pointer, i32
-.globaltype	__table_base, i32, immutable
-.globaltype	__memory_base, i32, immutable
+.globaltype __table_base, i32, immutable
+.globaltype __memory_base, i32, immutable
 
 .section .data.data,"",@
 data:

lld/test/wasm/shared64.s

@@ -1,5 +1,5 @@
 # RUN: llvm-mc -filetype=obj -triple=wasm64-unknown-unknown -o %t.o %s
-# RUN: wasm-ld -mwasm64 --experimental-pic -shared -o %t.wasm %t.o
+# RUN: wasm-ld -mwasm64 --experimental-pic --unresolved-symbols=import-dynamic -shared -o %t.wasm %t.o
 # RUN: obj2yaml %t.wasm | FileCheck %s
 # RUN: llvm-objdump --disassemble-symbols=__wasm_call_ctors,__wasm_apply_data_relocs --no-show-raw-insn --no-leading-addr %t.wasm | FileCheck %s --check-prefixes DIS
 

lld/test/wasm/signature-mismatch.s

@@ -93,7 +93,7 @@ ret32_address_main:
 # RELOC-NEXT:       - Index:           1
 # RELOC-NEXT:         Kind:            FUNCTION
 # RELOC-NEXT:         Name:            ret32
-# RELOC-NEXT:         Flags:           [ VISIBILITY_HIDDEN ]
+# RELOC-NEXT:         Flags:           [ ]
 # RELOC-NEXT:         Function:        2
 # RELOC-NEXT:       - Index:           2
 # RELOC-NEXT:         Kind:            DATA

lld/test/wasm/tag-section.ll

@@ -11,7 +11,7 @@
 ; RUN: llc -filetype=obj -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling -relocation-model=pic %p/Inputs/tag-section1.ll -o %t1.o
 ; RUN: llc -filetype=obj -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling -relocation-model=pic %p/Inputs/tag-section2.ll -o %t2.o
 ; RUN: llc -filetype=obj -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling -relocation-model=pic %s -o %t.o
-; RUN: wasm-ld --import-undefined --experimental-pic -pie -o %t.wasm %t.o %t1.o %t2.o
+; RUN: wasm-ld --import-undefined --experimental-pic --unresolved-symbols=import-dynamic -pie -o %t.wasm %t.o %t1.o %t2.o
 ; RUN: obj2yaml %t.wasm | FileCheck %s --check-prefix=PIC
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"

lld/test/wasm/undef-shared.s

@@ -0,0 +1,12 @@
+# RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown %s -o %t.o
+# RUN: not wasm-ld --experimental-pic %t.o -o /dev/null -shared 2>&1 | FileCheck %s
+
+# CHECK: error: {{.*}}: undefined symbol: hidden
+.global hidden
+.hidden hidden
+
+.global foo
+.section .data,"",@
+foo:
+ .int32 hidden
+ .size foo,4

lld/test/wasm/undefined-data.s

@@ -1,7 +1,7 @@
 # RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown -o %t.o %s
 # RUN: not wasm-ld -o %t.wasm %t.o 2>&1 | FileCheck %s -check-prefix=UNDEF
 # RUN: wasm-ld --allow-undefined -o %t.wasm %t.o
-# RUN: not wasm-ld --shared -o %t.wasm %t.o 2>&1 | FileCheck %s -check-prefix=SHARED
+# RUN: not wasm-ld --experimental-pic -shared --unresolved-symbols=import-dynamic -o %t.wasm %t.o 2>&1 | FileCheck %s -check-prefix=SHARED
 
 .globl  _start
 _start:

lld/test/wasm/unresolved-symbols.s

@@ -85,7 +85,7 @@
 # RUN: llvm-readobj %t4.wasm > /dev/null 2>&1
 
 ## import-dynamic should fail due to incompatible relocations.
-# RUN: not wasm-ld %t/main.o -o %t5.wasm --unresolved-symbols=import-dynamic 2>&1 | FileCheck -check-prefix=ERRNOPIC %s
+# RUN: not wasm-ld %t/main.o -o %t5.wasm --experimental-pic --unresolved-symbols=import-dynamic 2>&1 | FileCheck -check-prefix=ERRNOPIC %s
 # ERRNOPIC: relocation R_WASM_MEMORY_ADDR_SLEB cannot be used against symbol `undef_data`; recompile with -fPIC
 # ERRNOPIC: relocation R_WASM_TABLE_INDEX_SLEB cannot be used against symbol `undef_func`; recompile with -fPIC
 

lld/wasm/InputFiles.cpp

@@ -175,7 +175,7 @@ uint64_t ObjFile::calcNewValue(const WasmRelocation &reloc, uint64_t tombstone,
   case R_WASM_MEMORY_ADDR_TLS_SLEB:
   case R_WASM_MEMORY_ADDR_TLS_SLEB64:
   case R_WASM_MEMORY_ADDR_LOCREL_I32: {
-    if (isa<UndefinedData>(sym) || sym->isUndefWeak())
+    if (isa<UndefinedData>(sym) || sym->isShared() || sym->isUndefWeak())
       return 0;
     auto D = cast<DefinedData>(sym);
     uint64_t value = D->getVA() + reloc.Addend;
@@ -388,7 +388,7 @@ static bool shouldMerge(const WasmSegment &seg) {
 }
 
 void ObjFile::parseLazy() {
-  LLVM_DEBUG(dbgs() << "ObjFile::parseLazy: " << toString(this) << "\n");
+  LLVM_DEBUG(dbgs() << "ObjFile::parseLazy: " << toString(this) << " " << wasmObj.get() << "\n");
   for (const SymbolRef &sym : wasmObj->symbols()) {
     const WasmSymbol &wasmSym = wasmObj->getWasmSymbol(sym.getRawDataRefImpl());
     if (!wasmSym.isDefined())
@@ -403,21 +403,51 @@ void ObjFile::parseLazy() {
 }
 
 ObjFile::ObjFile(MemoryBufferRef m, StringRef archiveName, bool lazy)
-    : InputFile(ObjectKind, m) {
+    : WasmFileBase(ObjectKind, m) {
   this->lazy = lazy;
   this->archiveName = std::string(archiveName);
 
   // If this isn't part of an archive, it's eagerly linked, so mark it live.
   if (archiveName.empty())
     markLive();
+}
+
+void SharedFile::parse() {
+  assert(wasmObj->isSharedObject());
 
+  for (const SymbolRef &sym : wasmObj->symbols()) {
+    const WasmSymbol &wasmSym = wasmObj->getWasmSymbol(sym.getRawDataRefImpl());
+    if (wasmSym.isDefined()) {
+      StringRef name = wasmSym.Info.Name;
+      uint32_t flags = wasmSym.Info.Flags;
+      Symbol *s;
+      LLVM_DEBUG(dbgs() << "shared symbol: " << name << "\n");
+      switch (wasmSym.Info.Kind) {
+      case WASM_SYMBOL_TYPE_FUNCTION:
+        if (name == "__wasm_apply_data_relocs" || name == "__wasm_call_ctors") {
+          continue;
+        }
+        s = symtab->addSharedFunction(name, flags, this, wasmSym.Signature);
+        break;
+      case WASM_SYMBOL_TYPE_DATA:
+        s = symtab->addSharedData(name, flags, this);
+        break;
+      default:
+        continue;
+      }
+      symbols.push_back(s);
+    }
+  }
+}
+
+WasmFileBase::WasmFileBase(Kind k, MemoryBufferRef m) : InputFile(k, m) {
+  // Parse a memory buffer as a wasm file.
+  LLVM_DEBUG(dbgs() << "Reading object: " << toString(this) << "\n");
   std::unique_ptr<Binary> bin = CHECK(createBinary(mb), toString(this));
 
   auto *obj = dyn_cast<WasmObjectFile>(bin.get());
   if (!obj)
     fatal(toString(this) + ": not a wasm file");
-  if (!obj->isRelocatableObject())
-    fatal(toString(this) + ": not a relocatable wasm file");
 
   bin.release();
   wasmObj.reset(obj);
@@ -429,6 +459,9 @@ void ObjFile::parse(bool ignoreComdats) {
   // Parse a memory buffer as a wasm file.
   LLVM_DEBUG(dbgs() << "ObjFile::parse: " << toString(this) << "\n");
 
+  if (!wasmObj->isRelocatableObject())
+    fatal(toString(this) + ": not a relocatable wasm file");
+
   // Build up a map of function indices to table indices for use when
   // verifying the existing table index relocations
   uint32_t totalFunctions =

lld/wasm/InputFiles.h

@@ -87,18 +87,26 @@ class InputFile {
   bool live;
 };
 
+class WasmFileBase : public InputFile {
+public:
+  explicit WasmFileBase(Kind k, MemoryBufferRef m);
+
+  // Returns the underlying wasm file.
+  const WasmObjectFile *getWasmObj() const { return wasmObj.get(); }
+
+protected:
+  std::unique_ptr<WasmObjectFile> wasmObj;
+};
+
 // .o file (wasm object file)
-class ObjFile : public InputFile {
+class ObjFile : public WasmFileBase {
 public:
   ObjFile(MemoryBufferRef m, StringRef archiveName, bool lazy = false);
   static bool classof(const InputFile *f) { return f->kind() == ObjectKind; }
 
   void parse(bool ignoreComdats = false);
   void parseLazy();
 
-  // Returns the underlying wasm file.
-  const WasmObjectFile *getWasmObj() const { return wasmObj.get(); }
-
   uint32_t calcNewIndex(const WasmRelocation &reloc) const;
   uint64_t calcNewValue(const WasmRelocation &reloc, uint64_t tombstone,
                         const InputChunk *chunk) const;
@@ -139,14 +147,15 @@ class ObjFile : public InputFile {
 
   bool isExcludedByComdat(const InputChunk *chunk) const;
   void addLegacyIndirectFunctionTableIfNeeded(uint32_t tableSymbolCount);
-
-  std::unique_ptr<WasmObjectFile> wasmObj;
 };
 
 // .so file.
-class SharedFile : public InputFile {
+class SharedFile : public WasmFileBase {
 public:
-  explicit SharedFile(MemoryBufferRef m) : InputFile(SharedKind, m) {}
+  explicit SharedFile(MemoryBufferRef m) : WasmFileBase(SharedKind, m) {}
+
+  void parse();
+
   static bool classof(const InputFile *f) { return f->kind() == SharedKind; }
 };
 

lld/wasm/MarkLive.cpp

@@ -63,13 +63,15 @@ void MarkLive::enqueue(Symbol *sym) {
   sym->markLive();
 
   if (markImplicitDeps) {
-    // Mark ctor functions in the object that defines this symbol live.
-    // The ctor functions are all referenced by the synthetic callCtors
-    // function. However, this function does not contain relocations so we
-    // have to manually mark the ctors as live.
-    enqueueInitFunctions(cast<ObjFile>(file));
-    // Mark retained segments in the object that defines this symbol live.
-    enqueueRetainedSegments(cast<ObjFile>(file));
+    if (auto obj = dyn_cast<ObjFile>(file)) {
+      // Mark as live the ctor functions in the object that defines this symbol.
+      // The ctor functions are all referenced by the synthetic callCtors
+      // function. However, this function does not contain relocations so we
+      // have to manually mark the ctors as live.
+      enqueueInitFunctions(obj);
+      // Mark retained segments in the object that defines this symbol live.
+      enqueueRetainedSegments(obj);
+    }
   }
 
   if (InputChunk *chunk = sym->getChunk())

lld/wasm/Relocations.cpp

@@ -19,6 +19,8 @@ using namespace llvm::wasm;
 namespace lld::wasm {
 
 static bool requiresGOTAccess(const Symbol *sym) {
+  if (sym->isShared())
+    return true;
   if (!ctx.isPic &&
       config->unresolvedSymbols != UnresolvedPolicy::ImportDynamic)
     return false;
@@ -142,9 +144,12 @@ void scanRelocations(InputChunk *chunk) {
       break;
     }
 
-    if (ctx.isPic ||
+    bool shouldImport =
+        sym->isShared() ||
         (sym->isUndefined() &&
-         config->unresolvedSymbols == UnresolvedPolicy::ImportDynamic)) {
+         config->unresolvedSymbols == UnresolvedPolicy::ImportDynamic);
+
+    if (shouldImport) {
       switch (reloc.Type) {
       case R_WASM_TABLE_INDEX_SLEB:
       case R_WASM_TABLE_INDEX_SLEB64:
@@ -163,8 +168,8 @@ void scanRelocations(InputChunk *chunk) {
       case R_WASM_MEMORY_ADDR_I32:
       case R_WASM_MEMORY_ADDR_I64:
         // These relocation types are only present in the data section and
-        // will be converted into code by `generateRelocationCode`.  This code
-        // requires the symbols to have GOT entries.
+        // will be converted into code by `generateRelocationCode`.  This
+        // code requires the symbols to have GOT entries.
         if (requiresGOTAccess(sym))
           addGOTEntry(sym);
         break;