[compiler-rt] Avoid memintrinsic calls inserted by the compiler

D135716 introduced -ftrivial-auto-var-init=pattern where supported.
Unfortunately this introduces unwanted memset() for large stack arrays,
as shown by the new tests added for asan and msan (tsan already had this
test).

In general, the problem of compiler-inserted memintrinsic calls
(memset/memcpy/memmove) is not new to compiler-rt, and has been a
problem before.

To avoid introducing unwanted memintrinsic calls, we redefine
memintrinsics as __sanitizer_internal_mem* at the assembly level for
most source files automatically (where sanitizer_common_internal_defs.h
is included).

In few cases, redefining a symbol in this way causes issues for
interceptors, namely the memintrinsic interceptor themselves. For such
source files we have to selectively disable the redefinition.

Other alternatives have been considered, but simply do not work well in
the context of compiler-rt:

	1. Linker --wrap:  this does not work because --wrap only
	   applies to the final link, and would not apply when building
	   sanitizer static libraries.

	2. Changing references to memset() via objcopy:  this may work,
	   but due to the complexities of the build system, introducing
	   such a post-processing step for the right object files (in
	   particular object files defining memset cannot be touched)
	   seems infeasible.

The chosen solution works well (as shown by the tests). Other libraries
have chosen the same solution where nothing else works (see e.g. glibc's
"symbol-hacks.h").

Reviewed By: vitalybuka, dvyukov

Differential Revision: https://reviews.llvm.org/D151152

Author: melver

compiler-rt/lib/asan/asan_interceptors_memintrinsics.cpp

@@ -11,6 +11,8 @@
 // ASan versions of memcpy, memmove, and memset.
 //===---------------------------------------------------------------------===//
 
+#define SANITIZER_COMMON_NO_REDEFINE_BUILTINS
+
 #include "asan_interceptors_memintrinsics.h"
 
 #include "asan_interceptors.h"

compiler-rt/lib/hwasan/hwasan_interceptors.cpp

@@ -14,6 +14,8 @@
 // sanitizer_common/sanitizer_common_interceptors.h
 //===----------------------------------------------------------------------===//
 
+#define SANITIZER_COMMON_NO_REDEFINE_BUILTINS
+
 #include "hwasan.h"
 #include "hwasan_allocator.h"
 #include "hwasan_checks.h"

compiler-rt/lib/interception/tests/CMakeLists.txt

@@ -17,6 +17,7 @@ set(INTERCEPTION_TEST_CFLAGS_COMMON
   -I${COMPILER_RT_SOURCE_DIR}/include
   -I${COMPILER_RT_SOURCE_DIR}/lib
   -I${COMPILER_RT_SOURCE_DIR}/lib/interception
+  -DSANITIZER_COMMON_NO_REDEFINE_BUILTINS
   -fno-rtti
   -O2
   -Werror=sign-compare)

compiler-rt/lib/memprof/memprof_interceptors_memintrinsics.cpp

@@ -11,6 +11,8 @@
 // MemProf versions of memcpy, memmove, and memset.
 //===---------------------------------------------------------------------===//
 
+#define SANITIZER_COMMON_NO_REDEFINE_BUILTINS
+
 #include "memprof_interceptors_memintrinsics.h"
 
 #include "memprof_interceptors.h"

compiler-rt/lib/msan/msan_interceptors.cpp

@@ -14,6 +14,8 @@
 // sanitizer_common/sanitizer_common_interceptors.h
 //===----------------------------------------------------------------------===//
 
+#define SANITIZER_COMMON_NO_REDEFINE_BUILTINS
+
 #include "interception/interception.h"
 #include "msan.h"
 #include "msan_chained_origin_depot.h"

compiler-rt/lib/sanitizer_common/CMakeLists.txt

@@ -172,6 +172,7 @@ set(SANITIZER_IMPL_HEADERS
   sanitizer_procmaps.h
   sanitizer_ptrauth.h
   sanitizer_quarantine.h
+  sanitizer_redefine_builtins.h
   sanitizer_report_decorator.h
   sanitizer_ring_buffer.h
   sanitizer_signal_interceptors.inc

compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc

@@ -9,6 +9,10 @@
 // Memintrinsic function interceptors for tools like AddressSanitizer,
 // ThreadSanitizer, MemorySanitizer, etc.
 //
+// These interceptors are part of the common interceptors, but separated out so
+// that implementations may add them, if necessary, to a separate source file
+// that should define SANITIZER_COMMON_NO_REDEFINE_BUILTINS at the top.
+//
 // This file should be included into the tool's memintrinsic interceptor file,
 // which has to define its own macros:
 //   COMMON_INTERCEPTOR_ENTER
@@ -20,6 +24,10 @@
 //   COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED
 //===----------------------------------------------------------------------===//
 
+#ifdef SANITIZER_REDEFINE_BUILTINS_H
+#error "Define SANITIZER_COMMON_NO_REDEFINE_BUILTINS in .cpp file"
+#endif
+
 #include "interception/interception.h"
 #include "sanitizer_platform_interceptors.h"
 

compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc

@@ -46,3 +46,7 @@ INTERFACE_FUNCTION(__sanitizer_purge_allocator)
 INTERFACE_FUNCTION(__sanitizer_print_memory_profile)
 INTERFACE_WEAK_FUNCTION(__sanitizer_free_hook)
 INTERFACE_WEAK_FUNCTION(__sanitizer_malloc_hook)
+// Memintrinsic functions.
+INTERFACE_FUNCTION(__sanitizer_internal_memcpy)
+INTERFACE_FUNCTION(__sanitizer_internal_memmove)
+INTERFACE_FUNCTION(__sanitizer_internal_memset)

compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h

@@ -13,6 +13,7 @@
 #define SANITIZER_DEFS_H
 
 #include "sanitizer_platform.h"
+#include "sanitizer_redefine_builtins.h"
 
 #ifndef SANITIZER_DEBUG
 # define SANITIZER_DEBUG 0

compiler-rt/lib/sanitizer_common/sanitizer_libc.cpp

@@ -10,6 +10,9 @@
 // run-time libraries. See sanitizer_libc.h for details.
 //===----------------------------------------------------------------------===//
 
+// Do not redefine builtins; this file is defining the builtin replacements.
+#define SANITIZER_COMMON_NO_REDEFINE_BUILTINS
+
 #include "sanitizer_allocator_internal.h"
 #include "sanitizer_common.h"
 #include "sanitizer_libc.h"
@@ -46,15 +49,23 @@ int internal_memcmp(const void* s1, const void* s2, uptr n) {
   return 0;
 }
 
-void *internal_memcpy(void *dest, const void *src, uptr n) {
+void *internal_memcpy(void *dest, const void *src, uptr n)
+    ALIAS(__sanitizer_internal_memcpy);
+SANITIZER_INTERFACE_ATTRIBUTE
+extern "C" void *__sanitizer_internal_memcpy(void *dest, const void *src,
+                                             uptr n) {
   char *d = (char*)dest;
   const char *s = (const char *)src;
   for (uptr i = 0; i < n; ++i)
     d[i] = s[i];
   return dest;
 }
 
-void *internal_memmove(void *dest, const void *src, uptr n) {
+void *internal_memmove(void *dest, const void *src, uptr n)
+    ALIAS(__sanitizer_internal_memmove);
+SANITIZER_INTERFACE_ATTRIBUTE
+extern "C" void *__sanitizer_internal_memmove(void *dest, const void *src,
+                                              uptr n) {
   char *d = (char*)dest;
   const char *s = (const char *)src;
   sptr i, signed_n = (sptr)n;
@@ -72,7 +83,10 @@ void *internal_memmove(void *dest, const void *src, uptr n) {
   return dest;
 }
 
-void *internal_memset(void* s, int c, uptr n) {
+void *internal_memset(void *s, int c, uptr n)
+    ALIAS(__sanitizer_internal_memset);
+SANITIZER_INTERFACE_ATTRIBUTE
+extern "C" void *__sanitizer_internal_memset(void *s, int c, uptr n) {
   // Optimize for the most performance-critical case:
   if ((reinterpret_cast<uptr>(s) % 16) == 0 && (n % 16) == 0) {
     u64 *p = reinterpret_cast<u64*>(s);

compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h

@@ -0,0 +1,27 @@
+//===-- sanitizer_redefine_builtins.h ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Redefine builtin functions to use internal versions. This is needed where
+// compiler optimizations end up producing unwanted libcalls!
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_COMMON_NO_REDEFINE_BUILTINS
+#ifndef SANITIZER_REDEFINE_BUILTINS_H
+#define SANITIZER_REDEFINE_BUILTINS_H
+
+// The asm hack only works with GCC and Clang.
+#if !defined(_MSC_VER) || defined(__clang__)
+
+asm("memcpy = __sanitizer_internal_memcpy");
+asm("memmove = __sanitizer_internal_memmove");
+asm("memset = __sanitizer_internal_memset");
+
+#endif  // !_MSC_VER || __clang__
+
+#endif  // SANITIZER_REDEFINE_BUILTINS_H
+#endif  // SANITIZER_COMMON_NO_REDEFINE_BUILTINS

compiler-rt/lib/tsan/rtl/tsan_interceptors_memintrinsics.cpp

@@ -10,6 +10,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+#define SANITIZER_COMMON_NO_REDEFINE_BUILTINS
+
 #include "tsan_interceptors.h"
 #include "tsan_interface.h"
 

compiler-rt/test/asan/TestCases/Linux/check_memcpy.c

@@ -0,0 +1,8 @@
+// Verify runtime doesn't contain compiler-emitted memcpy/memmove calls.
+//
+// REQUIRES: shared_unwind, x86_64-target-arch
+
+// RUN: %clang_asan -O1 %s -o %t
+// RUN: llvm-objdump -d -l %t | FileCheck --implicit-check-not="{{(callq|jmpq) .*<(__interceptor_.*)?mem(cpy|set|move)>}}" %s
+
+int main() { return 0; }

compiler-rt/test/msan/Linux/check_memcpy.c

@@ -0,0 +1,8 @@
+// Verify runtime doesn't contain compiler-emitted memcpy/memmove calls.
+//
+// REQUIRES: shared_unwind, x86_64-target-arch
+
+// RUN: %clang_msan -O1 %s -o %t
+// RUN: llvm-objdump -d -l %t | FileCheck --implicit-check-not="{{(callq|jmpq) .*<(__interceptor_.*)?mem(cpy|set|move)>}}" %s
+
+int main() { return 0; }

compiler-rt/test/tsan/Linux/check_memcpy.c

@@ -5,16 +5,9 @@
 // This could fail if using a static libunwind because that static libunwind
 // could be uninstrumented and contain memcpy/memmove calls not intercepted by
 // tsan.
-// REQUIRES: shared_unwind
+// REQUIRES: shared_unwind, x86_64-target-arch
 
 // RUN: %clang_tsan -O1 %s -o %t
-// RUN: llvm-objdump -d -l %t | FileCheck %s
-
-int main() {
-  return 0;
-}
-
-// CHECK-NOT: callq {{.*<(__interceptor_)?mem(cpy|set)>}}
-// tail calls:
-// CHECK-NOT: jmpq {{.*<(__interceptor_)?mem(cpy|set)>}}
+// RUN: llvm-objdump -d -l %t | FileCheck --implicit-check-not="{{(callq|jmpq) .*<(__interceptor_.*)?mem(cpy|set|move)>}}" %s
 
+int main() { return 0; }