diff --git a/compiler-rt/lib/asan/asan_allocator.cc b/compiler-rt/lib/asan/asan_allocator.cc index 25d98bbfee87..acd466ec95db 100644 --- a/compiler-rt/lib/asan/asan_allocator.cc +++ b/compiler-rt/lib/asan/asan_allocator.cc @@ -34,6 +34,7 @@ #include "asan_stats.h" #include "asan_thread.h" #include "asan_thread_registry.h" +#include "sanitizer_common/sanitizer_atomic.h" #if defined(_WIN32) && !defined(__clang__) #include @@ -159,8 +160,8 @@ enum { struct ChunkBase { // First 8 bytes. uptr chunk_state : 8; - uptr size_class : 8; uptr alloc_tid : 24; + uptr size_class : 8; uptr free_tid : 24; // Second 8 bytes. @@ -420,7 +421,7 @@ class MallocInfo { private: PageGroup *FindPageGroupUnlocked(uptr addr) { - int n = n_page_groups_; + int n = atomic_load(&n_page_groups_, memory_order_relaxed); // If the page groups are not sorted yet, sort them. if (n_sorted_page_groups_ < n) { SortArray((uptr*)page_groups_, n); @@ -562,9 +563,9 @@ class MallocInfo { pg->end = pg->beg + mmap_size; pg->size_of_chunk = size; pg->last_chunk = (uptr)(mem + size * (n_chunks - 1)); - int page_group_idx = AtomicInc(&n_page_groups_) - 1; - CHECK(page_group_idx < (int)ASAN_ARRAY_SIZE(page_groups_)); - page_groups_[page_group_idx] = pg; + int idx = atomic_fetch_add(&n_page_groups_, 1, memory_order_relaxed); + CHECK(idx < (int)ASAN_ARRAY_SIZE(page_groups_)); + page_groups_[idx] = pg; return res; } @@ -573,7 +574,7 @@ class MallocInfo { AsanLock mu_; PageGroup *page_groups_[kMaxAvailableRam / kMinMmapSize]; - int n_page_groups_; // atomic + atomic_uint32_t n_page_groups_; int n_sorted_page_groups_; }; @@ -721,7 +722,8 @@ static void Deallocate(u8 *ptr, AsanStackTrace *stack) { AsanChunk *m = PtrToChunk((uptr)ptr); // Flip the chunk_state atomically to avoid race on double-free. - u8 old_chunk_state = AtomicExchange((u8*)m, CHUNK_QUARANTINE); + u8 old_chunk_state = atomic_exchange((atomic_uint8_t*)m, CHUNK_QUARANTINE, + memory_order_acq_rel); if (old_chunk_state == CHUNK_QUARANTINE) { AsanReport("ERROR: AddressSanitizer attempting double-free on %p:\n", ptr); diff --git a/compiler-rt/lib/asan/asan_rtl.cc b/compiler-rt/lib/asan/asan_rtl.cc index 71e05a7d18aa..616266cf68c7 100644 --- a/compiler-rt/lib/asan/asan_rtl.cc +++ b/compiler-rt/lib/asan/asan_rtl.cc @@ -21,14 +21,15 @@ #include "asan_stats.h" #include "asan_thread.h" #include "asan_thread_registry.h" +#include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_libc.h" namespace __sanitizer { using namespace __asan; void Die() { - static int num_calls = 0; - if (AtomicInc(&num_calls) > 1) { + static atomic_uint32_t num_calls; + if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) { // Don't die twice - run a busy loop. while (1) { } } @@ -343,8 +344,8 @@ void NOINLINE __asan_set_error_report_callback(void (*callback)(const char*)) { void __asan_report_error(uptr pc, uptr bp, uptr sp, uptr addr, bool is_write, uptr access_size) { // Do not print more than one report, otherwise they will mix up. - static int num_calls = 0; - if (AtomicInc(&num_calls) > 1) return; + static atomic_uint32_t num_calls; + if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) return; AsanPrintf("====================================================" "=============\n"); diff --git a/compiler-rt/lib/tsan/rtl/tsan_atomic.h b/compiler-rt/lib/sanitizer_common/sanitizer_atomic.h similarity index 73% rename from compiler-rt/lib/tsan/rtl/tsan_atomic.h rename to compiler-rt/lib/sanitizer_common/sanitizer_atomic.h index 6fcd9f9245b0..df3344ec8367 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_atomic.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_atomic.h @@ -1,4 +1,4 @@ -//===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===// +//===-- sanitizer_atomic.h --------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -6,20 +6,13 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// -// -// This file is a part of ThreadSanitizer (TSan), a race detector. -// -// Atomic operations. For now implies IA-32/Intel64. -//===----------------------------------------------------------------------===// -#ifndef TSAN_ATOMIC_H -#define TSAN_ATOMIC_H +#ifndef SANITIZER_ATOMIC_H +#define SANITIZER_ATOMIC_H -#include "tsan_defs.h" +#include "sanitizer_internal_defs.h" -namespace __tsan { - -const int kCacheLineSize = 64; +namespace __sanitizer { enum memory_order { memory_order_relaxed = 1 << 0, @@ -30,6 +23,16 @@ enum memory_order { memory_order_seq_cst = 1 << 5, }; +struct atomic_uint8_t { + typedef u8 Type; + volatile Type val_dont_use; +}; + +struct atomic_uint16_t { + typedef u16 Type; + volatile Type val_dont_use; +}; + struct atomic_uint32_t { typedef u32 Type; volatile Type val_dont_use; @@ -50,13 +53,15 @@ INLINE void atomic_signal_fence(memory_order) { } INLINE void atomic_thread_fence(memory_order) { - __asm__ __volatile__("mfence" ::: "memory"); + __sync_synchronize(); } INLINE void proc_yield(int cnt) { __asm__ __volatile__("" ::: "memory"); +#if defined(__i386__) || defined(__x86_64__) for (int i = 0; i < cnt; i++) __asm__ __volatile__("pause"); +#endif __asm__ __volatile__("" ::: "memory"); } @@ -109,9 +114,15 @@ INLINE typename T::Type atomic_fetch_sub(volatile T *a, return __sync_fetch_and_add(&a->val_dont_use, -v); } -INLINE uptr atomic_exchange(volatile atomic_uintptr_t *a, uptr v, - memory_order mo) { - __asm__ __volatile__("xchg %1, %0" : "+r"(v), "+m"(*a) : : "memory", "cc"); +template +INLINE typename T::Type atomic_exchange(volatile T *a, + typename T::Type v, memory_order mo) { + DCHECK(!((uptr)a % sizeof(*a))); + if (mo & (memory_order_release | memory_order_acq_rel | memory_order_seq_cst)) + __sync_synchronize(); + v = __sync_lock_test_and_set(&a->val_dont_use, v); + if (mo == memory_order_seq_cst) + __sync_synchronize(); return v; } @@ -129,12 +140,14 @@ INLINE bool atomic_compare_exchange_strong(volatile T *a, return false; } -INLINE bool atomic_compare_exchange_weak(volatile atomic_uintptr_t *a, - uptr *cmp, uptr xchg, - memory_order mo) { +template +INLINE bool atomic_compare_exchange_weak(volatile T *a, + typename T::Type *cmp, + typename T::Type xchg, + memory_order mo) { return atomic_compare_exchange_strong(a, cmp, xchg, mo); } -} // namespace __tsan +} // namespace __sanitizer -#endif // TSAN_ATOMIC_H +#endif // SANITIZER_ATOMIC_H diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/compiler-rt/lib/sanitizer_common/sanitizer_common.h index f24539b69b01..e691f48a6f38 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_common.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_common.h @@ -77,11 +77,6 @@ void NORETURN Abort(); int Atexit(void (*function)(void)); void SortArray(uptr *array, uptr size); -// Atomics -int AtomicInc(int *a); -u16 AtomicExchange(u16 *a, u16 new_val); -u8 AtomicExchange(u8 *a, u8 new_val); - // Math inline bool IsPowerOfTwo(uptr x) { return (x & (x - 1)) == 0; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h b/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h index ea914abc799e..b8cf61fad84a 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h @@ -112,6 +112,24 @@ void NORETURN CheckFailed(const char *file, int line, const char *cond, #define CHECK_GT(a, b) CHECK_IMPL((a), >, (b)) #define CHECK_GE(a, b) CHECK_IMPL((a), >=, (b)) +#if TSAN_DEBUG +#define DCHECK(a) CHECK(a) +#define DCHECK_EQ(a, b) CHECK_EQ(a, b) +#define DCHECK_NE(a, b) CHECK_NE(a, b) +#define DCHECK_LT(a, b) CHECK_LT(a, b) +#define DCHECK_LE(a, b) CHECK_LE(a, b) +#define DCHECK_GT(a, b) CHECK_GT(a, b) +#define DCHECK_GE(a, b) CHECK_GE(a, b) +#else +#define DCHECK(a) +#define DCHECK_EQ(a, b) +#define DCHECK_NE(a, b) +#define DCHECK_LT(a, b) +#define DCHECK_LE(a, b) +#define DCHECK_GT(a, b) +#define DCHECK_GE(a, b) +#endif + #define UNIMPLEMENTED() CHECK("unimplemented" && 0) #define COMPILER_CHECK(pred) IMPL_COMPILER_ASSERT(pred, __LINE__) diff --git a/compiler-rt/lib/tsan/rtl/tsan_defs.h b/compiler-rt/lib/tsan/rtl/tsan_defs.h index c2450312fbf7..3d9cd54b5ead 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_defs.h +++ b/compiler-rt/lib/tsan/rtl/tsan_defs.h @@ -54,24 +54,6 @@ const bool kCollectStats = true; const bool kCollectStats = false; #endif -#if TSAN_DEBUG -#define DCHECK(a) CHECK(a) -#define DCHECK_EQ(a, b) CHECK_EQ(a, b) -#define DCHECK_NE(a, b) CHECK_NE(a, b) -#define DCHECK_LT(a, b) CHECK_LT(a, b) -#define DCHECK_LE(a, b) CHECK_LE(a, b) -#define DCHECK_GT(a, b) CHECK_GT(a, b) -#define DCHECK_GE(a, b) CHECK_GE(a, b) -#else -#define DCHECK(a) -#define DCHECK_EQ(a, b) -#define DCHECK_NE(a, b) -#define DCHECK_LT(a, b) -#define DCHECK_LE(a, b) -#define DCHECK_GT(a, b) -#define DCHECK_GE(a, b) -#endif - // The following "build consistency" machinery ensures that all source files // are built in the same configuration. Inconsistent builds lead to // hard to debug crashes. diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc b/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc index 2e9426bc16fe..5dbaf866beec 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc +++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc @@ -12,11 +12,11 @@ //===----------------------------------------------------------------------===// #include "interception/interception.h" +#include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_placement_new.h" #include "tsan_rtl.h" #include "tsan_interface.h" -#include "tsan_atomic.h" #include "tsan_platform.h" #include "tsan_mman.h" diff --git a/compiler-rt/lib/tsan/rtl/tsan_mutex.h b/compiler-rt/lib/tsan/rtl/tsan_mutex.h index 2180978f92f4..6fbf8220eab8 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_mutex.h +++ b/compiler-rt/lib/tsan/rtl/tsan_mutex.h @@ -13,7 +13,7 @@ #ifndef TSAN_MUTEX_H #define TSAN_MUTEX_H -#include "tsan_atomic.h" +#include "sanitizer_common/sanitizer_atomic.h" #include "tsan_defs.h" namespace __tsan { diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl.cc b/compiler-rt/lib/tsan/rtl/tsan_rtl.cc index a66f6726848e..822dfa164dfe 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_rtl.cc +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl.cc @@ -12,6 +12,7 @@ // Main file (entry points) for the TSan run-time. //===----------------------------------------------------------------------===// +#include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_placement_new.h" @@ -19,7 +20,6 @@ #include "tsan_platform.h" #include "tsan_rtl.h" #include "tsan_interface.h" -#include "tsan_atomic.h" #include "tsan_mman.h" #include "tsan_suppressions.h" diff --git a/compiler-rt/lib/tsan/rtl/tsan_sync.h b/compiler-rt/lib/tsan/rtl/tsan_sync.h index 51989d223aad..34d3e0b2132f 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_sync.h +++ b/compiler-rt/lib/tsan/rtl/tsan_sync.h @@ -13,7 +13,8 @@ #ifndef TSAN_SYNC_H #define TSAN_SYNC_H -#include "tsan_atomic.h" +#include "sanitizer_common/sanitizer_atomic.h" +#include "sanitizer_common/sanitizer_common.h" #include "tsan_clock.h" #include "tsan_defs.h" #include "tsan_mutex.h" diff --git a/compiler-rt/lib/tsan/rtl_tests/tsan_mutex.cc b/compiler-rt/lib/tsan/rtl_tests/tsan_mutex.cc index 8fa46706ce8e..4d9c77961818 100644 --- a/compiler-rt/lib/tsan/rtl_tests/tsan_mutex.cc +++ b/compiler-rt/lib/tsan/rtl_tests/tsan_mutex.cc @@ -10,7 +10,7 @@ // This file is a part of ThreadSanitizer (TSan), a race detector. // //===----------------------------------------------------------------------===// -#include "tsan_atomic.h" +#include "sanitizer_common/sanitizer_atomic.h" #include "tsan_interface.h" #include "tsan_interface_ann.h" #include "tsan_test_util.h" diff --git a/compiler-rt/lib/tsan/rtl_tests/tsan_test_util_linux.cc b/compiler-rt/lib/tsan/rtl_tests/tsan_test_util_linux.cc index 35a7c4815244..5bc393bf6c2e 100644 --- a/compiler-rt/lib/tsan/rtl_tests/tsan_test_util_linux.cc +++ b/compiler-rt/lib/tsan/rtl_tests/tsan_test_util_linux.cc @@ -1,3 +1,4 @@ + //===-- tsan_test_util_linux.cc -------------------------------------------===// // // The LLVM Compiler Infrastructure @@ -12,9 +13,9 @@ // Test utils, linux implementation. //===----------------------------------------------------------------------===// +#include "sanitizer_common/sanitizer_atomic.h" #include "tsan_interface.h" #include "tsan_test_util.h" -#include "tsan_atomic.h" #include "tsan_report.h" #include "gtest/gtest.h" diff --git a/compiler-rt/lib/tsan/unit_tests/tsan_mutex_test.cc b/compiler-rt/lib/tsan/unit_tests/tsan_mutex_test.cc index 1e11ee8b0d87..e26dd486f9f9 100644 --- a/compiler-rt/lib/tsan/unit_tests/tsan_mutex_test.cc +++ b/compiler-rt/lib/tsan/unit_tests/tsan_mutex_test.cc @@ -11,7 +11,8 @@ // //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_internal_defs.h" -#include "tsan_atomic.h" +#include "sanitizer_common/sanitizer_atomic.h" +#include "sanitizer_common/sanitizer_common.h" #include "tsan_mutex.h" #include "gtest/gtest.h"