[asan] Re-exec without ASLR if needed on 32-bit Linux

High-entropy ASLR allows up to 16-bits of entropy (256MB), which is a
significant chunk of the 32-bit address space (4GB, less if running with
a 32-bit kernel). This, combined with ASan's shadow (512MB) and ASan's
fixed shadow offset (512MB), makes it possible for large binaries to
fail to map the shadow.

This patch will re-exec without ASLR if it cannot map the shadow, thus
reclaiming the 256MB of address space.

Alternatives considered:
1) We don't attempt to lower ASan's fixed shadow offset, because that
   would limit non-PIE binaries.
2) We don't switch to a dynamic shadow offset, because ASan on 32-bit
   Linux relies on the constant offset to optimize its instrumentation
   and compiler-rt.

This is loosely inspired by
llvm#78351,
llvm#85142, and llvm#85674, though those were required because there were no static mappings that could fully shadow the range of user mappings; this is not the case for ASan.

Author: thurstond

compiler-rt/lib/asan/asan_internal.h

@@ -82,6 +82,7 @@ void ReplaceSystemMalloc();
 uptr FindDynamicShadowStart();
 void AsanCheckDynamicRTPrereqs();
 void AsanCheckIncompatibleRT();
+void TryReExecWithoutASLR();
 
 // Unpoisons platform-specific stacks.
 // Returns true if all stacks have been unpoisoned.

compiler-rt/lib/asan/asan_linux.cpp

@@ -21,6 +21,7 @@
 #  include <pthread.h>
 #  include <stdio.h>
 #  include <sys/mman.h>
+#  include <sys/personality.h>
 #  include <sys/resource.h>
 #  include <sys/syscall.h>
 #  include <sys/time.h>
@@ -107,6 +108,33 @@ void FlushUnneededASanShadowMemory(uptr p, uptr size) {
   ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
 }
 
+void ReExecWithoutASLR() {
+  // ASLR personality check.
+  // Caution: 'personality' is sometimes forbidden by sandboxes, so only call
+  // this function as a last resort (when the memory mapping is incompatible
+  // and ASan would fail anyway).
+  int old_personality = personality(0xffffffff);
+  bool aslr_on =
+      (old_personality != -1) && ((old_personality & ADDR_NO_RANDOMIZE) == 0);
+
+  if (aslr_on) {
+    // Disable ASLR if the memory layout was incompatible.
+    // Alternatively, we could just keep re-execing until we get lucky
+    // with a compatible randomized layout, but the risk is that if it's
+    // not an ASLR-related issue, we will be stuck in an infinite loop of
+    // re-execing (unless we change ReExec to pass a parameter of the
+    // number of retries allowed.)
+    VReport(1,
+            "WARNING: AddressSanitizer: memory layout is incompatible, "
+            "possibly due to high-entropy ASLR.\n"
+            "Re-execing with fixed virtual address space.\n"
+            "N.B. reducing ASLR entropy is preferable.\n");
+    CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1);
+
+    ReExec();
+  }
+}
+
 #  if SANITIZER_ANDROID
 // FIXME: should we do anything for Android?
 void AsanCheckDynamicRTPrereqs() {}

compiler-rt/lib/asan/asan_mac.cpp

@@ -55,6 +55,9 @@ uptr FindDynamicShadowStart() {
                           GetMmapGranularity());
 }
 
+// Not used.
+void TryReExecWithoutASLR() {}
+
 // No-op. Mac does not support static linkage anyway.
 void AsanCheckDynamicRTPrereqs() {}
 

compiler-rt/lib/asan/asan_shadow_setup.cpp

@@ -109,6 +109,15 @@ void InitializeShadowMemory() {
     ProtectGap(kShadowGap2Beg, kShadowGap2End - kShadowGap2Beg + 1);
     ProtectGap(kShadowGap3Beg, kShadowGap3End - kShadowGap3Beg + 1);
   } else {
+#  if SANITIZER_LINUX
+    // The shadow mappings can shadow the entire user address space. However,
+    // on 32-bit systems, the maximum ASLR entropy (currently up to 16-bits
+    // == 256MB) is a significant chunk of the address space; reclaiming it by
+    // disabling ASLR might allow chonky binaries to run.
+    if (sizeof(uptr) == 32)
+      TryReExecWithoutASLR();
+#  endif
+
     Report(
         "Shadow memory range interleaves with an existing memory mapping. "
         "ASan cannot proceed correctly. ABORTING.\n");

compiler-rt/lib/asan/asan_win.cpp

@@ -43,6 +43,7 @@ uptr __asan_get_shadow_memory_dynamic_address() {
   __asan_init();
   return __asan_shadow_memory_dynamic_address;
 }
+
 }  // extern "C"
 
 // ---------------------- Windows-specific interceptors ---------------- {{{
@@ -279,6 +280,9 @@ uptr FindDynamicShadowStart() {
                           GetMmapGranularity());
 }
 
+// Not used
+void TryReExecWithoutASLR() {}
+
 void AsanCheckDynamicRTPrereqs() {}
 
 void AsanCheckIncompatibleRT() {}