[hwasan, asan] Intercept vfork.

Summary:
Intercept vfork on arm, aarch64, i386 and x86_64.

Reviewers: pcc, vitalybuka

Subscribers: kubamracek, mgorny, javed.absar, krytarowski, kristof.beyls, #sanitizers, llvm-commits

Tags: #sanitizers, #llvm

Differential Revision: https://reviews.llvm.org/D58533

llvm-svn: 355030

compiler-rt/include/sanitizer/hwasan_interface.h

@@ -50,6 +50,10 @@ extern "C" {
   // does would cause false reports.
   void __hwasan_handle_longjmp(const void *sp_dst);
 
+  // Set memory tag for the part of the current thread stack below sp_dst to
+  // zero. Call this in vfork() before returning in the parent process.
+  void __hwasan_handle_vfork(const void *sp_dst);
+
   // Libc hook for thread creation. Should be called in the child thread before
   // any instrumented code.
   void __hwasan_thread_enter();
@@ -65,6 +69,10 @@ extern "C" {
   // Print one-line report about the memory usage of the current process.
   void __hwasan_print_memory_usage();
 
+  /* Returns the offset of the first byte in the memory range that can not be
+   * accessed through the pointer in x, or -1 if the whole range is good. */
+  intptr_t __hwasan_test_shadow(const volatile void *x, size_t size);
+
   int __sanitizer_posix_memalign(void **memptr, size_t alignment, size_t size);
   void * __sanitizer_memalign(size_t alignment, size_t size);
   void * __sanitizer_aligned_alloc(size_t alignment, size_t size);

compiler-rt/lib/asan/CMakeLists.txt

@@ -32,6 +32,10 @@ set(ASAN_SOURCES
   asan_thread.cc
   asan_win.cc)
 
+if (NOT WIN32)
+  list(APPEND ASAN_SOURCES asan_interceptors_vfork.S)
+endif()
+
 set(ASAN_CXX_SOURCES
   asan_new_delete.cc)
 

compiler-rt/lib/asan/asan_interceptors.cc

@@ -579,6 +579,11 @@ INTERCEPTOR(int, __cxa_atexit, void (*func)(void *), void *arg,
 }
 #endif  // ASAN_INTERCEPT___CXA_ATEXIT
 
+#if defined(__linux__)
+DEFINE_REAL(int, vfork);
+DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork);
+#endif
+
 // ---------------------- InitializeAsanInterceptors ---------------- {{{1
 namespace __asan {
 void InitializeAsanInterceptors() {
@@ -656,6 +661,10 @@ void InitializeAsanInterceptors() {
   ASAN_INTERCEPT_FUNC(__cxa_atexit);
 #endif
 
+#if defined(__linux__)
+  ASAN_INTERCEPT_FUNC(vfork);
+#endif
+
   InitializePlatformInterceptors();
 
   VReport(1, "AddressSanitizer: libc interceptors initialized\n");

compiler-rt/lib/asan/asan_interceptors_vfork.S

@@ -0,0 +1,12 @@
+#include "sanitizer_common/sanitizer_asm.h"
+
+#if defined(__linux__)
+#define COMMON_INTERCEPTOR_SPILL_AREA __asan_extra_spill_area
+#define COMMON_INTERCEPTOR_HANDLE_VFORK __asan_handle_vfork
+#include "sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S"
+#include "sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S"
+#include "sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S"
+#include "sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S"
+#endif
+
+NO_EXEC_STACK_DIRECTIVE

compiler-rt/lib/asan/asan_interface.inc

@@ -38,6 +38,7 @@ INTERFACE_FUNCTION(__asan_get_report_pc)
 INTERFACE_FUNCTION(__asan_get_report_sp)
 INTERFACE_FUNCTION(__asan_get_shadow_mapping)
 INTERFACE_FUNCTION(__asan_handle_no_return)
+INTERFACE_FUNCTION(__asan_handle_vfork)
 INTERFACE_FUNCTION(__asan_init)
 INTERFACE_FUNCTION(__asan_load_cxx_array_cookie)
 INTERFACE_FUNCTION(__asan_load1)

compiler-rt/lib/asan/asan_interface_internal.h

@@ -249,6 +249,8 @@ extern "C" {
 
   SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
   const char* __asan_default_suppressions();
+
+  SANITIZER_INTERFACE_ATTRIBUTE void __asan_handle_vfork(void *sp);
 }  // extern "C"
 
 #endif  // ASAN_INTERFACE_INTERNAL_H

compiler-rt/lib/asan/asan_rtl.cc

@@ -597,6 +597,19 @@ void NOINLINE __asan_handle_no_return() {
     curr_thread->fake_stack()->HandleNoReturn();
 }
 
+extern "C" void *__asan_extra_spill_area() {
+  AsanThread *t = GetCurrentThread();
+  CHECK(t);
+  return t->extra_spill_area();
+}
+
+void __asan_handle_vfork(void *sp) {
+  AsanThread *t = GetCurrentThread();
+  CHECK(t);
+  uptr bottom = t->stack_bottom();
+  PoisonShadow(bottom, (uptr)sp - bottom, 0);
+}
+
 void NOINLINE __asan_set_death_callback(void (*callback)(void)) {
   SetUserDieCallback(callback);
 }

compiler-rt/lib/asan/asan_thread.h

@@ -130,6 +130,8 @@ class AsanThread {
   AsanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; }
   AsanStats &stats() { return stats_; }
 
+  void *extra_spill_area() { return &extra_spill_area_; }
+
  private:
   // NOTE: There is no AsanThread constructor. It is allocated
   // via mmap() and *must* be valid in zero-initialized state.
@@ -165,6 +167,7 @@ class AsanThread {
   AsanThreadLocalMallocStorage malloc_storage_;
   AsanStats stats_;
   bool unwinding_;
+  uptr extra_spill_area_;
 };
 
 // ScopedUnwinding is a scope for stacktracing member of a context

compiler-rt/lib/hwasan/CMakeLists.txt

@@ -6,6 +6,7 @@ set(HWASAN_RTL_SOURCES
   hwasan_allocator.cpp
   hwasan_dynamic_shadow.cpp
   hwasan_interceptors.cpp
+  hwasan_interceptors_vfork.S
   hwasan_linux.cpp
   hwasan_memintrinsics.cpp
   hwasan_poisoning.cpp

compiler-rt/lib/hwasan/hwasan.cpp

@@ -480,6 +480,30 @@ void __hwasan_handle_longjmp(const void *sp_dst) {
   TagMemory(sp, dst - sp, 0);
 }
 
+void __hwasan_handle_vfork(const void *sp_dst) {
+  uptr sp = (uptr)sp_dst;
+  Thread *t = GetCurrentThread();
+  CHECK(t);
+  uptr top = t->stack_top();
+  uptr bottom = t->stack_bottom();
+  static const uptr kMaxExpectedCleanupSize = 64 << 20;  // 64M
+  if (top == 0 || bottom == 0 || sp < bottom || sp >= top ||
+      sp - bottom > kMaxExpectedCleanupSize) {
+    Report(
+        "WARNING: HWASan is ignoring requested __hwasan_handle_vfork: "
+        "stack top: %zx; current %zx; bottom: %zx \n"
+        "False positive error reports may follow\n",
+        top, sp, bottom);
+    return;
+  }
+  TagMemory(bottom, sp - bottom, 0);
+}
+
+extern "C" void *__hwasan_extra_spill_area() {
+  Thread *t = GetCurrentThread();
+  return &t->vfork_spill();
+}
+
 void __hwasan_print_memory_usage() {
   InternalScopedString s(kMemoryUsageBufferSize);
   HwasanFormatMemoryUsage(s);

compiler-rt/lib/hwasan/hwasan_interceptors.cpp

@@ -227,6 +227,11 @@ INTERCEPTOR(int, pthread_create, void *th, void *attr,
 }
 #endif
 
+#if HWASAN_WITH_INTERCEPTORS
+DEFINE_REAL(int, vfork);
+DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork);
+#endif
+
 static void BeforeFork() {
   StackDepotLockAll();
 }
@@ -266,9 +271,12 @@ void InitializeInterceptors() {
   INTERCEPT_FUNCTION(fork);
 
 #if HWASAN_WITH_INTERCEPTORS
+#if defined(__linux__)
+  INTERCEPT_FUNCTION(vfork);
+#endif  // __linux__
 #if !defined(__aarch64__)
   INTERCEPT_FUNCTION(pthread_create);
-#endif
+#endif  // __aarch64__
   INTERCEPT_FUNCTION(realloc);
   INTERCEPT_FUNCTION(free);
 #endif

compiler-rt/lib/hwasan/hwasan_interceptors_vfork.S

@@ -0,0 +1,10 @@
+#include "sanitizer_common/sanitizer_asm.h"
+
+#if defined(__linux__)
+#define COMMON_INTERCEPTOR_SPILL_AREA __hwasan_extra_spill_area
+#define COMMON_INTERCEPTOR_HANDLE_VFORK __hwasan_handle_vfork
+#include "sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S"
+#include "sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S"
+#endif
+
+NO_EXEC_STACK_DIRECTIVE

compiler-rt/lib/hwasan/hwasan_interface_internal.h

@@ -116,6 +116,9 @@ void __hwasan_print_shadow(const void *x, uptr size);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_handle_longjmp(const void *sp_dst);
 
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_handle_vfork(const void *sp_dst);
+
 SANITIZER_INTERFACE_ATTRIBUTE
 u16 __sanitizer_unaligned_load16(const uu16 *p);
 

compiler-rt/lib/hwasan/hwasan_thread.h

@@ -67,11 +67,14 @@ class Thread {
     Print("Thread: ");
   }
 
+  uptr &vfork_spill() { return vfork_spill_; }
+
  private:
   // NOTE: There is no Thread constructor. It is allocated
   // via mmap() and *must* be valid in zero-initialized state.
   void ClearShadowForThreadStackAndTLS();
   void Print(const char *prefix);
+  uptr vfork_spill_;
   uptr stack_top_;
   uptr stack_bottom_;
   uptr tls_begin_;

compiler-rt/lib/interception/interception.h

@@ -185,11 +185,17 @@ const interpose_substitution substitution_##func_name[] \
 #endif  // SANITIZER_MAC
 
 #if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
-#define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \
+# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \
   DECLARE_REAL(ret_type, func, __VA_ARGS__) \
   extern "C" ret_type WRAP(func)(__VA_ARGS__);
+// Declare an interceptor and its wrapper defined in a different translation
+// unit (ex. asm).
+# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...)    \
+  extern "C" ret_type WRAP(func)(__VA_ARGS__); \
+  extern "C" ret_type func(__VA_ARGS__);
 #else
-#define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...)
+# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...)
+# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...)
 #endif
 
 // Generally, you don't need to use DEFINE_REAL by itself, as INTERCEPTOR

compiler-rt/lib/sanitizer_common/CMakeLists.txt

@@ -126,6 +126,7 @@ set(SANITIZER_IMPL_HEADERS
   sanitizer_common_interceptors.inc
   sanitizer_common_interceptors_format.inc
   sanitizer_common_interceptors_ioctl.inc
+  sanitizer_common_interceptors_vfork_aarch64.inc.S
   sanitizer_common_interface.inc
   sanitizer_common_interface_posix.inc
   sanitizer_common_syscalls.inc

compiler-rt/lib/sanitizer_common/sanitizer_asm.h

@@ -44,16 +44,18 @@
 
 #if !defined(__APPLE__)
 # define ASM_HIDDEN(symbol) .hidden symbol
-# define ASM_TYPE_FUNCTION(symbol) .type symbol, @function
+# define ASM_TYPE_FUNCTION(symbol) .type symbol, %function
 # define ASM_SIZE(symbol) .size symbol, .-symbol
 # define ASM_SYMBOL(symbol) symbol
 # define ASM_SYMBOL_INTERCEPTOR(symbol) symbol
+# define ASM_WRAPPER_NAME(symbol) __interceptor_##symbol
 #else
 # define ASM_HIDDEN(symbol)
 # define ASM_TYPE_FUNCTION(symbol)
 # define ASM_SIZE(symbol)
 # define ASM_SYMBOL(symbol) _##symbol
 # define ASM_SYMBOL_INTERCEPTOR(symbol) _wrap_##symbol
+# define ASM_WRAPPER_NAME(symbol) __interceptor_##symbol
 #endif
 
 #if defined(__ELF__) && (defined(__GNU__) || defined(__FreeBSD__) || \

compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S

@@ -0,0 +1,43 @@
+#if defined(__aarch64__) && defined(__linux__)
+
+#include "sanitizer_common/sanitizer_asm.h"
+
+ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA)
+
+.comm _ZN14__interception10real_vforkE,8,8
+.globl ASM_WRAPPER_NAME(vfork)
+ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
+ASM_WRAPPER_NAME(vfork):
+        // Save x30 in the off-stack spill area.
+        stp     xzr, x30, [sp, #-16]!
+        bl      COMMON_INTERCEPTOR_SPILL_AREA
+        ldp     xzr, x30, [sp], 16
+        str     x30, [x0]
+
+        // Call real vfork. This may return twice. User code that runs between the first and the second return
+        // may clobber the stack frame of the interceptor; that's why it does not have a frame.
+        adrp    x0, _ZN14__interception10real_vforkE
+        ldr     x0, [x0, :lo12:_ZN14__interception10real_vforkE]
+        blr     x0
+
+        stp     x0, xzr, [sp, #-16]!
+        cmp     x0, #0
+        b.eq   .L_exit
+
+        // x0 != 0 => parent process. Clear stack shadow.
+        add    x0, sp, #16
+        bl     COMMON_INTERCEPTOR_HANDLE_VFORK
+
+.L_exit:
+        // Restore x30.
+        bl     COMMON_INTERCEPTOR_SPILL_AREA
+        ldr    x30, [x0]
+        ldp    x0, xzr, [sp], 16
+
+        ret
+ASM_SIZE(vfork)
+
+.weak vfork
+.set vfork, ASM_WRAPPER_NAME(vfork)
+
+#endif

compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S

@@ -0,0 +1,49 @@
+#if defined(__arm__) && defined(__linux__)
+
+#include "sanitizer_common/sanitizer_asm.h"
+
+ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA)
+
+.comm _ZN14__interception10real_vforkE,4,4
+.globl ASM_WRAPPER_NAME(vfork)
+ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
+ASM_WRAPPER_NAME(vfork):
+        // Save LR in the off-stack spill area.
+        push    {r4, lr}
+        bl      COMMON_INTERCEPTOR_SPILL_AREA
+        pop     {r4, lr}
+        str     lr, [r0]
+
+        // Call real vfork. This may return twice. User code that runs between the first and the second return
+        // may clobber the stack frame of the interceptor; that's why it does not have a frame.
+        ldr     r0, .LCPI0_0
+.LPC0_0:
+        ldr     r0, [pc, r0]
+        mov     lr, pc
+        bx      r0
+
+        push    {r0, r4}
+        cmp     r0, #0
+        beq     .L_exit
+
+        // r0 != 0 => parent process. Clear stack shadow.
+        add     r0, sp, #8
+        bl      COMMON_INTERCEPTOR_HANDLE_VFORK
+
+.L_exit:
+        // Restore LR.
+        bl      COMMON_INTERCEPTOR_SPILL_AREA
+        ldr     lr, [r0]
+        pop     {r0, r4}
+
+        mov     pc, lr
+
+.LCPI0_0:
+        .long   _ZN14__interception10real_vforkE - (.LPC0_0+8)
+
+ASM_SIZE(vfork)
+
+.weak vfork
+.set vfork, ASM_WRAPPER_NAME(vfork)
+
+#endif

compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S

@@ -0,0 +1,45 @@
+#if defined(__i386__) && defined(__linux__)
+
+#include "sanitizer_common/sanitizer_asm.h"
+
+.comm _ZN14__interception10real_vforkE,4,4
+.globl ASM_WRAPPER_NAME(vfork)
+ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
+ASM_WRAPPER_NAME(vfork):
+        // Store return address in the spill area and tear down the stack frame.
+        sub     $12, %esp
+        call    COMMON_INTERCEPTOR_SPILL_AREA
+        mov     12(%esp), %ecx
+        mov     %ecx, (%eax)
+        add     $16, %esp
+
+
+        call    *_ZN14__interception10real_vforkE
+
+        // Restore return address from the spill area.
+        sub     $16, %esp
+        mov     %eax, 4(%esp)
+        call    COMMON_INTERCEPTOR_SPILL_AREA
+        mov     (%eax), %ecx
+        mov     %ecx, 12(%esp)
+        mov     4(%esp), %eax
+
+
+        // Call handle_vfork in the parent process (%rax != 0).
+        test    %eax, %eax
+        je      .L_exit
+
+        lea     16(%esp), %ecx
+        mov     %ecx, (%esp)
+        call    COMMON_INTERCEPTOR_HANDLE_VFORK
+
+.L_exit:
+        mov     4(%esp), %eax
+        add     $12, %esp
+        ret
+ASM_SIZE(vfork)
+
+.weak vfork
+.set vfork, ASM_WRAPPER_NAME(vfork)
+
+#endif

compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S

@@ -0,0 +1,41 @@
+#if defined(__x86_64__) && defined(__linux__)
+
+#include "sanitizer_common/sanitizer_asm.h"
+
+.comm _ZN14__interception10real_vforkE,8,8
+.globl ASM_WRAPPER_NAME(vfork)
+ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
+ASM_WRAPPER_NAME(vfork):
+        // Store return address in the spill area and tear down the stack frame.
+        push    %rcx
+        call    COMMON_INTERCEPTOR_SPILL_AREA
+        pop     %rcx
+        pop     %rdi
+        mov     %rdi, (%rax)
+
+        call    *_ZN14__interception10real_vforkE(%rip)
+
+        // Restore return address from the spill area.
+        push    %rcx
+        push    %rax
+        call    COMMON_INTERCEPTOR_SPILL_AREA
+        mov     (%rax), %rdx
+        mov     %rdx, 8(%rsp)
+        mov     (%rsp), %rax
+
+        // Call handle_vfork in the parent process (%rax != 0).
+        test    %rax, %rax
+        je      .L_exit
+
+        lea     16(%rsp), %rdi
+        call    COMMON_INTERCEPTOR_HANDLE_VFORK
+
+.L_exit:
+        pop     %rax
+        ret
+ASM_SIZE(vfork)
+
+.weak vfork
+.set vfork, ASM_WRAPPER_NAME(vfork)
+
+#endif

compiler-rt/test/asan/TestCases/Linux/vfork.cc

@@ -0,0 +1,31 @@
+// https://github.com/google/sanitizers/issues/925
+// RUN: %clang_asan -O0 %s -o %t && %run %t 2>&1
+
+// REQUIRES: aarch64-target-arch || x86_64-target-arch || i386-target-arch || arm-target-arch
+
+#include <assert.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <sanitizer/asan_interface.h>
+
+__attribute__((noinline, no_sanitize("address"))) void child() {
+  alignas(8) char x[100000];
+  __asan_poison_memory_region(x, sizeof(x));
+  _exit(0);
+}
+
+__attribute__((noinline, no_sanitize("address"))) void parent() {
+  alignas(8) char x[100000];
+  assert(__asan_address_is_poisoned(x + 5000) == 0);
+}
+
+int main(int argc, char **argv) {
+  if (vfork())
+    parent();
+  else
+    child();
+
+  return 0;
+}

compiler-rt/test/hwasan/TestCases/Linux/vfork.c

@@ -0,0 +1,32 @@
+// https://github.com/google/sanitizers/issues/925
+// RUN: %clang_hwasan -O0 %s -o %t && %run %t 2>&1
+
+// REQUIRES: aarch64-target-arch || x86_64-target-arch
+
+#include <assert.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <sanitizer/hwasan_interface.h>
+
+__attribute__((noinline, no_sanitize("hwaddress"))) void child() {
+  char x[10000];
+  __hwasan_tag_memory(x, 0xAA, sizeof(x));
+  _exit(0);
+}
+
+__attribute__((noinline, no_sanitize("hwaddress"))) void parent() {
+  char x[10000];
+  __hwasan_print_shadow(&x, sizeof(x));
+  assert(__hwasan_test_shadow(x, sizeof(x)) == -1);
+}
+
+int main(int argc, char **argv) {
+  if (vfork())
+    parent();
+  else
+    child();
+
+  return 0;
+}

llvm/utils/gn/secondary/compiler-rt/lib/hwasan/BUILD.gn

@@ -44,6 +44,7 @@ source_set("sources") {
     "hwasan_dynamic_shadow.h",
     "hwasan_flags.h",
     "hwasan_interceptors.cpp",
+    "hwasan_interceptors_vfork.S",
     "hwasan_interface_internal.h",
     "hwasan_linux.cpp",
     "hwasan_malloc_bisect.h",