tsan/asan: unify atomics (move atomics from tsan to sanitizer_common)

llvm-svn: 159437

compiler-rt/lib/asan/asan_allocator.cc

@@ -34,6 +34,7 @@
 #include "asan_stats.h"
 #include "asan_thread.h"
 #include "asan_thread_registry.h"
+#include "sanitizer_common/sanitizer_atomic.h"
 
 #if defined(_WIN32) && !defined(__clang__)
 #include <intrin.h>
@@ -159,8 +160,8 @@ enum {
 struct ChunkBase {
   // First 8 bytes.
   uptr  chunk_state : 8;
-  uptr  size_class  : 8;
   uptr  alloc_tid   : 24;
+  uptr  size_class  : 8;
   uptr  free_tid    : 24;
 
   // Second 8 bytes.
@@ -420,7 +421,7 @@ class MallocInfo {
 
  private:
   PageGroup *FindPageGroupUnlocked(uptr addr) {
-    int n = n_page_groups_;
+    int n = atomic_load(&n_page_groups_, memory_order_relaxed);
     // If the page groups are not sorted yet, sort them.
     if (n_sorted_page_groups_ < n) {
       SortArray((uptr*)page_groups_, n);
@@ -562,9 +563,9 @@ class MallocInfo {
     pg->end = pg->beg + mmap_size;
     pg->size_of_chunk = size;
     pg->last_chunk = (uptr)(mem + size * (n_chunks - 1));
-    int page_group_idx = AtomicInc(&n_page_groups_) - 1;
-    CHECK(page_group_idx < (int)ASAN_ARRAY_SIZE(page_groups_));
-    page_groups_[page_group_idx] = pg;
+    int idx = atomic_fetch_add(&n_page_groups_, 1, memory_order_relaxed);
+    CHECK(idx < (int)ASAN_ARRAY_SIZE(page_groups_));
+    page_groups_[idx] = pg;
     return res;
   }
 
@@ -573,7 +574,7 @@ class MallocInfo {
   AsanLock mu_;
 
   PageGroup *page_groups_[kMaxAvailableRam / kMinMmapSize];
-  int n_page_groups_;  // atomic
+  atomic_uint32_t n_page_groups_;
   int n_sorted_page_groups_;
 };
 
@@ -721,7 +722,8 @@ static void Deallocate(u8 *ptr, AsanStackTrace *stack) {
   AsanChunk *m = PtrToChunk((uptr)ptr);
 
   // Flip the chunk_state atomically to avoid race on double-free.
-  u8 old_chunk_state = AtomicExchange((u8*)m, CHUNK_QUARANTINE);
+  u8 old_chunk_state = atomic_exchange((atomic_uint8_t*)m, CHUNK_QUARANTINE,
+                                       memory_order_acq_rel);
 
   if (old_chunk_state == CHUNK_QUARANTINE) {
     AsanReport("ERROR: AddressSanitizer attempting double-free on %p:\n", ptr);

compiler-rt/lib/asan/asan_rtl.cc

@@ -21,14 +21,15 @@
 #include "asan_stats.h"
 #include "asan_thread.h"
 #include "asan_thread_registry.h"
+#include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_libc.h"
 
 namespace __sanitizer {
 using namespace __asan;
 
 void Die() {
-  static int num_calls = 0;
-  if (AtomicInc(&num_calls) > 1) {
+  static atomic_uint32_t num_calls;
+  if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) {
     // Don't die twice - run a busy loop.
     while (1) { }
   }
@@ -343,8 +344,8 @@ void NOINLINE __asan_set_error_report_callback(void (*callback)(const char*)) {
 void __asan_report_error(uptr pc, uptr bp, uptr sp,
                          uptr addr, bool is_write, uptr access_size) {
   // Do not print more than one report, otherwise they will mix up.
-  static int num_calls = 0;
-  if (AtomicInc(&num_calls) > 1) return;
+  static atomic_uint32_t num_calls;
+  if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) return;
 
   AsanPrintf("===================================================="
              "=============\n");

compiler-rt/lib/sanitizer_common/sanitizer_atomic.h

@@ -1,4 +1,4 @@
-//===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===//
+//===-- sanitizer_atomic.h --------------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -6,20 +6,13 @@
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
-//
-// This file is a part of ThreadSanitizer (TSan), a race detector.
-//
-// Atomic operations. For now implies IA-32/Intel64.
-//===----------------------------------------------------------------------===//
 
-#ifndef TSAN_ATOMIC_H
-#define TSAN_ATOMIC_H
+#ifndef SANITIZER_ATOMIC_H
+#define SANITIZER_ATOMIC_H
 
-#include "tsan_defs.h"
+#include "sanitizer_internal_defs.h"
 
-namespace __tsan {
-
-const int kCacheLineSize = 64;
+namespace __sanitizer {
 
 enum memory_order {
   memory_order_relaxed = 1 << 0,
@@ -30,6 +23,16 @@ enum memory_order {
   memory_order_seq_cst = 1 << 5,
 };
 
+struct atomic_uint8_t {
+  typedef u8 Type;
+  volatile Type val_dont_use;
+};
+
+struct atomic_uint16_t {
+  typedef u16 Type;
+  volatile Type val_dont_use;
+};
+
 struct atomic_uint32_t {
   typedef u32 Type;
   volatile Type val_dont_use;
@@ -50,13 +53,15 @@ INLINE void atomic_signal_fence(memory_order) {
 }
 
 INLINE void atomic_thread_fence(memory_order) {
-  __asm__ __volatile__("mfence" ::: "memory");
+  __sync_synchronize();
 }
 
 INLINE void proc_yield(int cnt) {
   __asm__ __volatile__("" ::: "memory");
+#if defined(__i386__) || defined(__x86_64__)
   for (int i = 0; i < cnt; i++)
     __asm__ __volatile__("pause");
+#endif
   __asm__ __volatile__("" ::: "memory");
 }
 
@@ -109,9 +114,15 @@ INLINE typename T::Type atomic_fetch_sub(volatile T *a,
   return __sync_fetch_and_add(&a->val_dont_use, -v);
 }
 
-INLINE uptr atomic_exchange(volatile atomic_uintptr_t *a, uptr v,
-                            memory_order mo) {
-  __asm__ __volatile__("xchg %1, %0" : "+r"(v), "+m"(*a) : : "memory", "cc");
+template<typename T>
+INLINE typename T::Type atomic_exchange(volatile T *a,
+    typename T::Type v, memory_order mo) {
+  DCHECK(!((uptr)a % sizeof(*a)));
+  if (mo & (memory_order_release | memory_order_acq_rel | memory_order_seq_cst))
+    __sync_synchronize();
+  v = __sync_lock_test_and_set(&a->val_dont_use, v);
+  if (mo == memory_order_seq_cst)
+    __sync_synchronize();
   return v;
 }
 
@@ -129,12 +140,14 @@ INLINE bool atomic_compare_exchange_strong(volatile T *a,
   return false;
 }
 
-INLINE bool atomic_compare_exchange_weak(volatile atomic_uintptr_t *a,
-                                         uptr *cmp, uptr xchg,
-                                         memory_order mo) {
+template<typename T>
+INLINE bool atomic_compare_exchange_weak(volatile T *a,
+                                           typename T::Type *cmp,
+                                           typename T::Type xchg,
+                                           memory_order mo) {
   return atomic_compare_exchange_strong(a, cmp, xchg, mo);
 }
 
-}  // namespace __tsan
+}  // namespace __sanitizer
 
-#endif  // TSAN_ATOMIC_H
+#endif  // SANITIZER_ATOMIC_H

compiler-rt/lib/sanitizer_common/sanitizer_common.h

@@ -77,11 +77,6 @@ void NORETURN Abort();
 int Atexit(void (*function)(void));
 void SortArray(uptr *array, uptr size);
 
-// Atomics
-int AtomicInc(int *a);
-u16 AtomicExchange(u16 *a, u16 new_val);
-u8 AtomicExchange(u8 *a, u8 new_val);
-
 // Math
 inline bool IsPowerOfTwo(uptr x) {
   return (x & (x - 1)) == 0;

compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h

@@ -112,6 +112,24 @@ void NORETURN CheckFailed(const char *file, int line, const char *cond,
 #define CHECK_GT(a, b) CHECK_IMPL((a), >,  (b))
 #define CHECK_GE(a, b) CHECK_IMPL((a), >=, (b))
 
+#if TSAN_DEBUG
+#define DCHECK(a)       CHECK(a)
+#define DCHECK_EQ(a, b) CHECK_EQ(a, b)
+#define DCHECK_NE(a, b) CHECK_NE(a, b)
+#define DCHECK_LT(a, b) CHECK_LT(a, b)
+#define DCHECK_LE(a, b) CHECK_LE(a, b)
+#define DCHECK_GT(a, b) CHECK_GT(a, b)
+#define DCHECK_GE(a, b) CHECK_GE(a, b)
+#else
+#define DCHECK(a)
+#define DCHECK_EQ(a, b)
+#define DCHECK_NE(a, b)
+#define DCHECK_LT(a, b)
+#define DCHECK_LE(a, b)
+#define DCHECK_GT(a, b)
+#define DCHECK_GE(a, b)
+#endif
+
 #define UNIMPLEMENTED() CHECK("unimplemented" && 0)
 
 #define COMPILER_CHECK(pred) IMPL_COMPILER_ASSERT(pred, __LINE__)

compiler-rt/lib/tsan/rtl/tsan_defs.h

@@ -54,24 +54,6 @@ const bool kCollectStats = true;
 const bool kCollectStats = false;
 #endif
 
-#if TSAN_DEBUG
-#define DCHECK(a)       CHECK(a)
-#define DCHECK_EQ(a, b) CHECK_EQ(a, b)
-#define DCHECK_NE(a, b) CHECK_NE(a, b)
-#define DCHECK_LT(a, b) CHECK_LT(a, b)
-#define DCHECK_LE(a, b) CHECK_LE(a, b)
-#define DCHECK_GT(a, b) CHECK_GT(a, b)
-#define DCHECK_GE(a, b) CHECK_GE(a, b)
-#else
-#define DCHECK(a)
-#define DCHECK_EQ(a, b)
-#define DCHECK_NE(a, b)
-#define DCHECK_LT(a, b)
-#define DCHECK_LE(a, b)
-#define DCHECK_GT(a, b)
-#define DCHECK_GE(a, b)
-#endif
-
 // The following "build consistency" machinery ensures that all source files
 // are built in the same configuration. Inconsistent builds lead to
 // hard to debug crashes.

compiler-rt/lib/tsan/rtl/tsan_interceptors.cc

@@ -12,11 +12,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "interception/interception.h"
+#include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_libc.h"
 #include "sanitizer_common/sanitizer_placement_new.h"
 #include "tsan_rtl.h"
 #include "tsan_interface.h"
-#include "tsan_atomic.h"
 #include "tsan_platform.h"
 #include "tsan_mman.h"
 

compiler-rt/lib/tsan/rtl/tsan_mutex.h

@@ -13,7 +13,7 @@
 #ifndef TSAN_MUTEX_H
 #define TSAN_MUTEX_H
 
-#include "tsan_atomic.h"
+#include "sanitizer_common/sanitizer_atomic.h"
 #include "tsan_defs.h"
 
 namespace __tsan {

compiler-rt/lib/tsan/rtl/tsan_rtl.cc

@@ -12,6 +12,7 @@
 // Main file (entry points) for the TSan run-time.
 //===----------------------------------------------------------------------===//
 
+#include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_libc.h"
 #include "sanitizer_common/sanitizer_placement_new.h"
@@ -19,7 +20,6 @@
 #include "tsan_platform.h"
 #include "tsan_rtl.h"
 #include "tsan_interface.h"
-#include "tsan_atomic.h"
 #include "tsan_mman.h"
 #include "tsan_suppressions.h"
 

compiler-rt/lib/tsan/rtl/tsan_sync.h

@@ -13,7 +13,8 @@
 #ifndef TSAN_SYNC_H
 #define TSAN_SYNC_H
 
-#include "tsan_atomic.h"
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
 #include "tsan_clock.h"
 #include "tsan_defs.h"
 #include "tsan_mutex.h"

compiler-rt/lib/tsan/rtl_tests/tsan_mutex.cc

@@ -10,7 +10,7 @@
 // This file is a part of ThreadSanitizer (TSan), a race detector.
 //
 //===----------------------------------------------------------------------===//
-#include "tsan_atomic.h"
+#include "sanitizer_common/sanitizer_atomic.h"
 #include "tsan_interface.h"
 #include "tsan_interface_ann.h"
 #include "tsan_test_util.h"

compiler-rt/lib/tsan/rtl_tests/tsan_test_util_linux.cc

@@ -1,3 +1,4 @@
+
 //===-- tsan_test_util_linux.cc -------------------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
@@ -12,9 +13,9 @@
 // Test utils, linux implementation.
 //===----------------------------------------------------------------------===//
 
+#include "sanitizer_common/sanitizer_atomic.h"
 #include "tsan_interface.h"
 #include "tsan_test_util.h"
-#include "tsan_atomic.h"
 #include "tsan_report.h"
 
 #include "gtest/gtest.h"

compiler-rt/lib/tsan/unit_tests/tsan_mutex_test.cc

@@ -11,7 +11,8 @@
 //
 //===----------------------------------------------------------------------===//
 #include "sanitizer_common/sanitizer_internal_defs.h"
-#include "tsan_atomic.h"
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
 #include "tsan_mutex.h"
 #include "gtest/gtest.h"