tsan: switch to new allocator

llvm-svn: 161953

compiler-rt/lib/tsan/rtl/tsan_defs.h

@@ -133,6 +133,7 @@ struct ReportStack;
 class ReportDesc;
 class RegionAlloc;
 class StackTrace;
+struct MBlock;
 
 }  // namespace __tsan
 

compiler-rt/lib/tsan/rtl/tsan_interceptors.cc

@@ -624,23 +624,23 @@ TSAN_INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *p) {
 
 TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) {
   SCOPED_TSAN_INTERCEPTOR(memalign, align, sz);
-  return user_alloc_aligned(thr, pc, sz, align);
+  return user_alloc(thr, pc, sz, align);
 }
 
 TSAN_INTERCEPTOR(void*, valloc, uptr sz) {
   SCOPED_TSAN_INTERCEPTOR(valloc, sz);
-  return user_alloc_aligned(thr, pc, sz, kPageSize);
+  return user_alloc(thr, pc, sz, kPageSize);
 }
 
 TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) {
   SCOPED_TSAN_INTERCEPTOR(pvalloc, sz);
   sz = RoundUp(sz, kPageSize);
-  return user_alloc_aligned(thr, pc, sz, kPageSize);
+  return user_alloc(thr, pc, sz, kPageSize);
 }
 
 TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
   SCOPED_TSAN_INTERCEPTOR(posix_memalign, memptr, align, sz);
-  *memptr = user_alloc_aligned(thr, pc, sz, align);
+  *memptr = user_alloc(thr, pc, sz, align);
   return 0;
 }
 

compiler-rt/lib/tsan/rtl/tsan_mman.cc

@@ -11,6 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 #include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
 #include "tsan_mman.h"
 #include "tsan_rtl.h"
 #include "tsan_report.h"
@@ -18,6 +19,19 @@
 
 namespace __tsan {
 
+extern char allocator_placeholder[];
+INLINE Allocator *allocator() {
+  return reinterpret_cast<Allocator*>(&allocator_placeholder);
+}
+
+void InitializeAllocator() {
+  allocator()->Init();
+}
+
+void AlloctorThreadFinish(ThreadState *thr) {
+  allocator()->SwallowCache(&thr->alloc_cache);
+}
+
 static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
   if (!thr->in_signal_handler || !flags()->report_signal_unsafe)
     return;
@@ -28,15 +42,13 @@ static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
   OutputReport(rep, rep.GetReport()->stacks[0]);
 }
 
-void *user_alloc(ThreadState *thr, uptr pc, uptr sz) {
+void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align) {
   CHECK_GT(thr->in_rtl, 0);
-  if (sz + sizeof(MBlock) < sz)
-    return 0;
-  MBlock *b = (MBlock*)InternalAlloc(sz + sizeof(MBlock));
-  if (b == 0)
+  void *p = allocator()->Allocate(&thr->alloc_cache, sz, align);
+  if (p == 0)
     return 0;
+  MBlock *b = (MBlock*)allocator()->GetMetaData(p);
   b->size = sz;
-  void *p = b + 1;
   if (CTX() && CTX()->initialized) {
     MemoryResetRange(thr, pc, (uptr)p, sz);
   }
@@ -49,12 +61,11 @@ void user_free(ThreadState *thr, uptr pc, void *p) {
   CHECK_GT(thr->in_rtl, 0);
   CHECK_NE(p, (void*)0);
   DPrintf("#%d: free(%p)\n", thr->tid, p);
-  MBlock *b = user_mblock(thr, p);
-  p = b + 1;
+  MBlock *b = (MBlock*)allocator()->GetMetaData(p);
   if (CTX() && CTX()->initialized && thr->in_rtl == 1) {
     MemoryRangeFreed(thr, pc, (uptr)p, b->size);
   }
-  InternalFree(b);
+  allocator()->Deallocate(&thr->alloc_cache, p);
   SignalUnsafeCall(thr, pc);
 }
 
@@ -78,26 +89,10 @@ void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz) {
   return p2;
 }
 
-void *user_alloc_aligned(ThreadState *thr, uptr pc, uptr sz, uptr align) {
-  CHECK_GT(thr->in_rtl, 0);
-  void *p = user_alloc(thr, pc, sz + align);
-  void *pa = RoundUp(p, align);
-  DCHECK_LE((uptr)pa + sz, (uptr)p + sz + align);
-  return pa;
-}
-
 MBlock *user_mblock(ThreadState *thr, void *p) {
   CHECK_GT(thr->in_rtl, 0);
   CHECK_NE(p, (void*)0);
-  MBlock *b = (MBlock*)InternalAllocBlock(p);
-  // FIXME: Output a warning, it's a user error.
-  if (p < (char*)(b + 1) || p > (char*)(b + 1) + b->size) {
-    TsanPrintf("user_mblock p=%p b=%p size=%zu beg=%p end=%p\n",
-        p, b, b->size, (char*)(b + 1), (char*)(b + 1) + b->size);
-    CHECK_GE(p, (char*)(b + 1));
-    CHECK_LE(p, (char*)(b + 1) + b->size);
-  }
-  return b;
+  return (MBlock*)allocator()->GetMetaData(p);
 }
 
 void *internal_alloc(MBlockType typ, uptr sz) {

compiler-rt/lib/tsan/rtl/tsan_mman.h

@@ -17,13 +17,14 @@
 
 namespace __tsan {
 
-// Descriptor of user's memory block.
-struct MBlock {
-  uptr size;
-};
+const uptr kDefaultAlignment = 16;
+
+void InitializeAllocator();
+void AlloctorThreadFinish(ThreadState *thr);
 
 // For user allocations.
-void *user_alloc(ThreadState *thr, uptr pc, uptr sz);
+void *user_alloc(ThreadState *thr, uptr pc, uptr sz,
+                 uptr align = kDefaultAlignment);
 // Does not accept NULL.
 void user_free(ThreadState *thr, uptr pc, void *p);
 void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz);

compiler-rt/lib/tsan/rtl/tsan_platform.h

@@ -31,7 +31,7 @@ static const uptr kLinuxShadowMsk = 0x100000000000ULL;
 static const uptr kLinuxAppMemBeg = 0x2a0000000000ULL;
 static const uptr kLinuxAppMemEnd = 0x7fffffffffffULL;
 #else
-static const uptr kLinuxAppMemBeg = 0x7ef000000000ULL;
+static const uptr kLinuxAppMemBeg = 0x7cf000000000ULL;
 static const uptr kLinuxAppMemEnd = 0x7fffffffffffULL;
 #endif
 

compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc

@@ -104,7 +104,7 @@ void InitializeShadowMemory() {
   if (shadow != kLinuxShadowBeg) {
     TsanPrintf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
     TsanPrintf("FATAL: Make sure to compile with -fPIE and "
-               "to link with -pie.\n");
+               "to link with -pie (%p, %p).\n", shadow, kLinuxShadowBeg);
     Die();
   }
 #ifndef TSAN_GO

compiler-rt/lib/tsan/rtl/tsan_rtl.cc

@@ -32,6 +32,7 @@ namespace __tsan {
 
 #ifndef TSAN_GO
 THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(64);
+char allocator_placeholder[sizeof(Allocator)] ALIGNED(64);
 #endif
 static char ctx_placeholder[sizeof(Context)] ALIGNED(64);
 
@@ -163,6 +164,7 @@ void Initialize(ThreadState *thr) {
     return;
   is_initialized = true;
   ScopedInRtl in_rtl;
+  InitializeAllocator();
   InitializeInterceptors();
   const char *env = InitializePlatform();
   InitializeMutex();

compiler-rt/lib/tsan/rtl/tsan_rtl.h

@@ -27,6 +27,7 @@
 #define TSAN_RTL_H
 
 #include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_allocator64.h"
 #include "tsan_clock.h"
 #include "tsan_defs.h"
 #include "tsan_flags.h"
@@ -37,6 +38,28 @@
 
 namespace __tsan {
 
+// Descriptor of user's memory block.
+struct MBlock {
+  uptr size;
+};
+
+#ifndef TSAN_GO
+#if defined(TSAN_COMPAT_SHADOW) && TSAN_COMPAT_SHADOW
+const uptr kAllocatorSpace = 0x7e0000000000ULL;
+#else
+const uptr kAllocatorSpace = 0x7d0000000000ULL;
+#endif
+const uptr kAllocatorSize  =  0x10000000000ULL;  // 1T.
+
+typedef SizeClassAllocator64<kAllocatorSpace, kAllocatorSize, sizeof(MBlock),
+    DefaultSizeClassMap> PrimaryAllocator;
+typedef SizeClassAllocatorLocalCache<PrimaryAllocator::kNumClasses,
+    PrimaryAllocator> AllocatorCache;
+typedef LargeMmapAllocator SecondaryAllocator;
+typedef CombinedAllocator<PrimaryAllocator, AllocatorCache,
+    SecondaryAllocator> Allocator;
+#endif
+
 void TsanPrintf(const char *format, ...);
 
 // FastState (from most significant bit):
@@ -237,6 +260,9 @@ struct ThreadState {
   uptr *shadow_stack_end;
 #endif
   ThreadClock clock;
+#ifndef TSAN_GO
+  AllocatorCache alloc_cache;
+#endif
   u64 stat[StatCnt];
   const int tid;
   int in_rtl;

compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc

@@ -238,6 +238,7 @@ void ThreadFinish(ThreadState *thr) {
   }
   tctx->epoch1 = thr->fast_state.epoch();
 
+  AlloctorThreadFinish(thr);
   thr->~ThreadState();
   StatAggregate(ctx->stat, thr->stat);
   tctx->thr = 0;