2012-06-30 01:10:08 +08:00
|
|
|
//===-- sanitizer_mutex.h ---------------------------------------*- C++ -*-===//
|
|
|
|
//
|
2019-01-19 16:50:56 +08:00
|
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
2012-06-30 01:10:08 +08:00
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
2012-06-30 01:32:18 +08:00
|
|
|
//
|
|
|
|
// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
2012-06-30 01:10:08 +08:00
|
|
|
|
|
|
|
#ifndef SANITIZER_MUTEX_H
|
|
|
|
#define SANITIZER_MUTEX_H
|
|
|
|
|
|
|
|
#include "sanitizer_atomic.h"
|
2012-07-02 15:09:21 +08:00
|
|
|
#include "sanitizer_internal_defs.h"
|
|
|
|
#include "sanitizer_libc.h"
|
2021-07-10 01:29:41 +08:00
|
|
|
#include "sanitizer_thread_safety.h"
|
2012-06-30 01:10:08 +08:00
|
|
|
|
|
|
|
namespace __sanitizer {
|
|
|
|
|
2022-01-05 20:35:13 +08:00
|
|
|
class SANITIZER_MUTEX StaticSpinMutex {
|
2012-06-30 01:32:18 +08:00
|
|
|
public:
|
2012-08-30 18:02:48 +08:00
|
|
|
void Init() {
|
2012-06-30 01:32:18 +08:00
|
|
|
atomic_store(&state_, 0, memory_order_relaxed);
|
|
|
|
}
|
|
|
|
|
2022-01-05 20:35:13 +08:00
|
|
|
void Lock() SANITIZER_ACQUIRE() {
|
2021-07-20 16:36:41 +08:00
|
|
|
if (LIKELY(TryLock()))
|
2012-07-02 15:09:21 +08:00
|
|
|
return;
|
|
|
|
LockSlow();
|
2012-06-30 01:32:18 +08:00
|
|
|
}
|
|
|
|
|
2022-01-05 20:35:13 +08:00
|
|
|
bool TryLock() SANITIZER_TRY_ACQUIRE(true) {
|
2013-01-11 19:03:35 +08:00
|
|
|
return atomic_exchange(&state_, 1, memory_order_acquire) == 0;
|
|
|
|
}
|
|
|
|
|
2022-01-05 20:35:13 +08:00
|
|
|
void Unlock() SANITIZER_RELEASE() {
|
|
|
|
atomic_store(&state_, 0, memory_order_release);
|
|
|
|
}
|
2012-06-30 01:32:18 +08:00
|
|
|
|
2022-01-05 20:35:13 +08:00
|
|
|
void CheckLocked() const SANITIZER_CHECK_LOCKED() {
|
2013-10-17 19:18:11 +08:00
|
|
|
CHECK_EQ(atomic_load(&state_, memory_order_relaxed), 1);
|
|
|
|
}
|
|
|
|
|
2012-06-30 01:32:18 +08:00
|
|
|
private:
|
|
|
|
atomic_uint8_t state_;
|
|
|
|
|
2021-07-20 16:36:41 +08:00
|
|
|
void LockSlow();
|
2012-08-30 18:02:48 +08:00
|
|
|
};
|
2012-07-02 15:09:21 +08:00
|
|
|
|
2022-01-05 20:35:13 +08:00
|
|
|
class SANITIZER_MUTEX SpinMutex : public StaticSpinMutex {
|
2012-08-30 18:02:48 +08:00
|
|
|
public:
|
|
|
|
SpinMutex() {
|
|
|
|
Init();
|
|
|
|
}
|
2021-07-21 15:10:00 +08:00
|
|
|
|
|
|
|
SpinMutex(const SpinMutex &) = delete;
|
|
|
|
void operator=(const SpinMutex &) = delete;
|
2012-06-30 01:32:18 +08:00
|
|
|
};
|
|
|
|
|
2021-07-15 23:15:48 +08:00
|
|
|
// Semaphore provides an OS-dependent way to park/unpark threads.
|
|
|
|
// The last thread returned from Wait can destroy the object
|
|
|
|
// (destruction-safety).
|
|
|
|
class Semaphore {
|
|
|
|
public:
|
|
|
|
constexpr Semaphore() {}
|
|
|
|
Semaphore(const Semaphore &) = delete;
|
|
|
|
void operator=(const Semaphore &) = delete;
|
|
|
|
|
|
|
|
void Wait();
|
|
|
|
void Post(u32 count = 1);
|
|
|
|
|
|
|
|
private:
|
|
|
|
atomic_uint32_t state_ = {0};
|
|
|
|
};
|
|
|
|
|
2021-07-20 17:17:00 +08:00
|
|
|
typedef int MutexType;
|
|
|
|
|
|
|
|
enum {
|
|
|
|
// Used as sentinel and to catch unassigned types
|
|
|
|
// (should not be used as real Mutex type).
|
|
|
|
MutexInvalid = 0,
|
|
|
|
MutexThreadRegistry,
|
|
|
|
// Each tool own mutexes must start at this number.
|
|
|
|
MutexLastCommon,
|
|
|
|
// Type for legacy mutexes that are not checked for deadlocks.
|
|
|
|
MutexUnchecked = -1,
|
|
|
|
// Special marks that can be used in MutexMeta::can_lock table.
|
|
|
|
// The leaf mutexes can be locked under any other non-leaf mutex,
|
|
|
|
// but no other mutex can be locked while under a leaf mutex.
|
|
|
|
MutexLeaf = -1,
|
|
|
|
// Multiple mutexes of this type can be locked at the same time.
|
|
|
|
MutexMulti = -3,
|
|
|
|
};
|
|
|
|
|
|
|
|
// Go linker does not support THREADLOCAL variables,
|
|
|
|
// so we can't use per-thread state.
|
Disable SANITIZER_CHECK_DEADLOCKS on Darwin platforms.
Although THREADLOCAL variables are supported on Darwin they cannot be
used very early on during process init (before dyld has set it up).
Unfortunately the checked lock is used before dyld has setup TLS leading
to an abort call (`_tlv_boostrap()` is never supposed to be called at
runtime).
To avoid this problem `SANITIZER_CHECK_DEADLOCKS` is now disabled on
Darwin platforms. This fixes running TSan tests (an possibly other
Sanitizers) when `COMPILER_RT_DEBUG=ON`.
For reference the crashing backtrace looks like this:
```
* thread #1, stop reason = signal SIGABRT
* frame #0: 0x00000002044da0ae dyld`__abort_with_payload + 10
frame #1: 0x00000002044f01af dyld`abort_with_payload_wrapper_internal + 80
frame #2: 0x00000002044f01e1 dyld`abort_with_payload + 9
frame #3: 0x000000010c989060 dyld_sim`abort_with_payload + 26
frame #4: 0x000000010c94908b dyld_sim`dyld4::halt(char const*) + 375
frame #5: 0x000000010c988f5c dyld_sim`abort + 16
frame #6: 0x000000010c96104f dyld_sim`dyld4::APIs::_tlv_bootstrap() + 9
frame #7: 0x000000010cd8d6d2 libclang_rt.tsan_iossim_dynamic.dylib`__sanitizer::CheckedMutex::LockImpl(this=<unavailable>, pc=<unavailable>) at sanitizer_mutex.cpp:218:58 [opt]
frame #8: 0x000000010cd8a0f7 libclang_rt.tsan_iossim_dynamic.dylib`__sanitizer::Mutex::Lock() [inlined] __sanitizer::CheckedMutex::Lock(this=0x000000010d733c90) at sanitizer_mutex.h:124:5 [opt]
frame #9: 0x000000010cd8a0ee libclang_rt.tsan_iossim_dynamic.dylib`__sanitizer::Mutex::Lock(this=0x000000010d733c90) at sanitizer_mutex.h:162:19 [opt]
frame #10: 0x000000010cd8a0bf libclang_rt.tsan_iossim_dynamic.dylib`__sanitizer::GenericScopedLock<__sanitizer::Mutex>::GenericScopedLock(this=0x000000030c7479a8, mu=<unavailable>) at sanitizer_mutex.h:364:10 [opt]
frame #11: 0x000000010cd89819 libclang_rt.tsan_iossim_dynamic.dylib`__sanitizer::GenericScopedLock<__sanitizer::Mutex>::GenericScopedLock(this=0x000000030c7479a8, mu=<unavailable>) at sanitizer_mutex.h:363:67 [opt]
frame #12: 0x000000010cd8985b libclang_rt.tsan_iossim_dynamic.dylib`__sanitizer::LibIgnore::OnLibraryLoaded(this=0x000000010d72f480, name=0x0000000000000000) at sanitizer_libignore.cpp:39:8 [opt]
frame #13: 0x000000010cda7aaa libclang_rt.tsan_iossim_dynamic.dylib`__tsan::InitializeLibIgnore() at tsan_interceptors_posix.cpp:219:16 [opt]
frame #14: 0x000000010cdce0bb libclang_rt.tsan_iossim_dynamic.dylib`__tsan::Initialize(thr=0x0000000110141400) at tsan_rtl.cpp:403:3 [opt]
frame #15: 0x000000010cda7b8e libclang_rt.tsan_iossim_dynamic.dylib`__tsan::ScopedInterceptor::ScopedInterceptor(__tsan::ThreadState*, char const*, unsigned long) [inlined] __tsan::LazyInitialize(thr=0x0000000110141400) at tsan_rtl.h:665:5 [opt]
frame #16: 0x000000010cda7b86 libclang_rt.tsan_iossim_dynamic.dylib`__tsan::ScopedInterceptor::ScopedInterceptor(this=0x000000030c747af8, thr=0x0000000110141400, fname=<unavailable>, pc=4568918787) at tsan_interceptors_posix.cpp:247:3 [opt]
frame #17: 0x000000010cda7bb9 libclang_rt.tsan_iossim_dynamic.dylib`__tsan::ScopedInterceptor::ScopedInterceptor(this=0x000000030c747af8, thr=<unavailable>, fname=<unavailable>, pc=<unavailable>) at tsan_interceptors_posix.cpp:246:59 [opt]
frame #18: 0x000000010cdb72b7 libclang_rt.tsan_iossim_dynamic.dylib`::wrap_strlcpy(dst="\xd2", src="0xd1d398d1bb0a007b", size=20) at sanitizer_common_interceptors.inc:7386:3 [opt]
frame #19: 0x0000000110542b03 libsystem_c.dylib`__guard_setup + 140
frame #20: 0x00000001104f8ab4 libsystem_c.dylib`_libc_initializer + 65
...
```
rdar://83723445
Differential Revision: https://reviews.llvm.org/D111243
2021-10-07 00:41:51 +08:00
|
|
|
// Disable checked locks on Darwin. Although Darwin platforms support
|
|
|
|
// THREADLOCAL variables they are not usable early on during process init when
|
|
|
|
// `__sanitizer::Mutex` is used.
|
2021-08-05 10:24:56 +08:00
|
|
|
#define SANITIZER_CHECK_DEADLOCKS \
|
2022-05-24 05:35:42 +08:00
|
|
|
(SANITIZER_DEBUG && !SANITIZER_GO && SANITIZER_SUPPORTS_THREADLOCAL && !SANITIZER_APPLE)
|
2021-07-20 17:17:00 +08:00
|
|
|
|
|
|
|
#if SANITIZER_CHECK_DEADLOCKS
|
|
|
|
struct MutexMeta {
|
|
|
|
MutexType type;
|
|
|
|
const char *name;
|
|
|
|
// The table fixes what mutexes can be locked under what mutexes.
|
|
|
|
// If the entry for MutexTypeFoo contains MutexTypeBar,
|
|
|
|
// then Bar mutex can be locked while under Foo mutex.
|
|
|
|
// Can also contain the special MutexLeaf/MutexMulti marks.
|
|
|
|
MutexType can_lock[10];
|
|
|
|
};
|
|
|
|
#endif
|
|
|
|
|
|
|
|
class CheckedMutex {
|
|
|
|
public:
|
2021-07-28 19:08:49 +08:00
|
|
|
explicit constexpr CheckedMutex(MutexType type)
|
2021-07-20 17:17:00 +08:00
|
|
|
#if SANITIZER_CHECK_DEADLOCKS
|
|
|
|
: type_(type)
|
|
|
|
#endif
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
ALWAYS_INLINE void Lock() {
|
|
|
|
#if SANITIZER_CHECK_DEADLOCKS
|
|
|
|
LockImpl(GET_CALLER_PC());
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
ALWAYS_INLINE void Unlock() {
|
|
|
|
#if SANITIZER_CHECK_DEADLOCKS
|
|
|
|
UnlockImpl();
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
// Checks that the current thread does not hold any mutexes
|
|
|
|
// (e.g. when returning from a runtime function to user code).
|
|
|
|
static void CheckNoLocks() {
|
|
|
|
#if SANITIZER_CHECK_DEADLOCKS
|
|
|
|
CheckNoLocksImpl();
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
#if SANITIZER_CHECK_DEADLOCKS
|
|
|
|
const MutexType type_;
|
|
|
|
|
|
|
|
void LockImpl(uptr pc);
|
|
|
|
void UnlockImpl();
|
|
|
|
static void CheckNoLocksImpl();
|
|
|
|
#endif
|
|
|
|
};
|
|
|
|
|
2021-07-18 16:19:37 +08:00
|
|
|
// Reader-writer mutex.
|
2021-07-23 16:41:40 +08:00
|
|
|
// Derive from CheckedMutex for the purposes of EBO.
|
|
|
|
// We could make it a field marked with [[no_unique_address]],
|
|
|
|
// but this attribute is not supported by some older compilers.
|
2022-01-05 20:35:13 +08:00
|
|
|
class SANITIZER_MUTEX Mutex : CheckedMutex {
|
2021-07-18 16:19:37 +08:00
|
|
|
public:
|
2021-07-28 19:08:49 +08:00
|
|
|
explicit constexpr Mutex(MutexType type = MutexUnchecked)
|
|
|
|
: CheckedMutex(type) {}
|
2021-07-18 16:19:37 +08:00
|
|
|
|
2022-01-05 20:35:13 +08:00
|
|
|
void Lock() SANITIZER_ACQUIRE() {
|
2021-07-23 16:41:40 +08:00
|
|
|
CheckedMutex::Lock();
|
2021-07-18 16:19:37 +08:00
|
|
|
u64 reset_mask = ~0ull;
|
|
|
|
u64 state = atomic_load_relaxed(&state_);
|
|
|
|
for (uptr spin_iters = 0;; spin_iters++) {
|
|
|
|
u64 new_state;
|
|
|
|
bool locked = (state & (kWriterLock | kReaderLockMask)) != 0;
|
|
|
|
if (LIKELY(!locked)) {
|
|
|
|
// The mutex is not read-/write-locked, try to lock.
|
|
|
|
new_state = (state | kWriterLock) & reset_mask;
|
|
|
|
} else if (spin_iters > kMaxSpinIters) {
|
|
|
|
// We've spun enough, increment waiting writers count and block.
|
|
|
|
// The counter will be decremented by whoever wakes us.
|
|
|
|
new_state = (state + kWaitingWriterInc) & reset_mask;
|
|
|
|
} else if ((state & kWriterSpinWait) == 0) {
|
|
|
|
// Active spinning, but denote our presence so that unlocking
|
|
|
|
// thread does not wake up other threads.
|
|
|
|
new_state = state | kWriterSpinWait;
|
|
|
|
} else {
|
|
|
|
// Active spinning.
|
|
|
|
state = atomic_load(&state_, memory_order_relaxed);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
|
|
|
|
memory_order_acquire)))
|
|
|
|
continue;
|
|
|
|
if (LIKELY(!locked))
|
|
|
|
return; // We've locked the mutex.
|
|
|
|
if (spin_iters > kMaxSpinIters) {
|
|
|
|
// We've incremented waiting writers, so now block.
|
|
|
|
writers_.Wait();
|
|
|
|
spin_iters = 0;
|
|
|
|
} else {
|
|
|
|
// We've set kWriterSpinWait, but we are still in active spinning.
|
|
|
|
}
|
|
|
|
// We either blocked and were unblocked,
|
|
|
|
// or we just spun but set kWriterSpinWait.
|
|
|
|
// Either way we need to reset kWriterSpinWait
|
|
|
|
// next time we take the lock or block again.
|
|
|
|
reset_mask = ~kWriterSpinWait;
|
2021-08-10 21:48:12 +08:00
|
|
|
state = atomic_load(&state_, memory_order_relaxed);
|
|
|
|
DCHECK_NE(state & kWriterSpinWait, 0);
|
2021-07-18 16:19:37 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-04-01 22:30:55 +08:00
|
|
|
bool TryLock() SANITIZER_TRY_ACQUIRE(true) {
|
|
|
|
u64 state = atomic_load_relaxed(&state_);
|
|
|
|
for (;;) {
|
|
|
|
if (UNLIKELY(state & (kWriterLock | kReaderLockMask)))
|
|
|
|
return false;
|
|
|
|
// The mutex is not read-/write-locked, try to lock.
|
|
|
|
if (LIKELY(atomic_compare_exchange_weak(
|
|
|
|
&state_, &state, state | kWriterLock, memory_order_acquire))) {
|
|
|
|
CheckedMutex::Lock();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-01-05 20:35:13 +08:00
|
|
|
void Unlock() SANITIZER_RELEASE() {
|
2021-07-23 16:41:40 +08:00
|
|
|
CheckedMutex::Unlock();
|
2021-07-18 16:19:37 +08:00
|
|
|
bool wake_writer;
|
|
|
|
u64 wake_readers;
|
|
|
|
u64 new_state;
|
|
|
|
u64 state = atomic_load_relaxed(&state_);
|
|
|
|
do {
|
|
|
|
DCHECK_NE(state & kWriterLock, 0);
|
|
|
|
DCHECK_EQ(state & kReaderLockMask, 0);
|
|
|
|
new_state = state & ~kWriterLock;
|
2021-08-10 21:48:12 +08:00
|
|
|
wake_writer = (state & (kWriterSpinWait | kReaderSpinWait)) == 0 &&
|
|
|
|
(state & kWaitingWriterMask) != 0;
|
2021-07-18 16:19:37 +08:00
|
|
|
if (wake_writer)
|
|
|
|
new_state = (new_state - kWaitingWriterInc) | kWriterSpinWait;
|
|
|
|
wake_readers =
|
2021-08-10 21:48:12 +08:00
|
|
|
wake_writer || (state & kWriterSpinWait) != 0
|
2021-07-18 16:19:37 +08:00
|
|
|
? 0
|
|
|
|
: ((state & kWaitingReaderMask) >> kWaitingReaderShift);
|
|
|
|
if (wake_readers)
|
2021-08-10 21:48:12 +08:00
|
|
|
new_state = (new_state & ~kWaitingReaderMask) | kReaderSpinWait;
|
2021-07-18 16:19:37 +08:00
|
|
|
} while (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
|
|
|
|
memory_order_release)));
|
|
|
|
if (UNLIKELY(wake_writer))
|
|
|
|
writers_.Post();
|
|
|
|
else if (UNLIKELY(wake_readers))
|
|
|
|
readers_.Post(wake_readers);
|
|
|
|
}
|
|
|
|
|
2022-01-05 20:35:13 +08:00
|
|
|
void ReadLock() SANITIZER_ACQUIRE_SHARED() {
|
2021-07-23 16:41:40 +08:00
|
|
|
CheckedMutex::Lock();
|
2021-08-10 21:48:12 +08:00
|
|
|
u64 reset_mask = ~0ull;
|
2021-07-18 16:19:37 +08:00
|
|
|
u64 state = atomic_load_relaxed(&state_);
|
2021-08-10 21:48:12 +08:00
|
|
|
for (uptr spin_iters = 0;; spin_iters++) {
|
|
|
|
bool locked = (state & kWriterLock) != 0;
|
|
|
|
u64 new_state;
|
|
|
|
if (LIKELY(!locked)) {
|
|
|
|
new_state = (state + kReaderLockInc) & reset_mask;
|
|
|
|
} else if (spin_iters > kMaxSpinIters) {
|
|
|
|
new_state = (state + kWaitingReaderInc) & reset_mask;
|
|
|
|
} else if ((state & kReaderSpinWait) == 0) {
|
|
|
|
// Active spinning, but denote our presence so that unlocking
|
|
|
|
// thread does not wake up other threads.
|
|
|
|
new_state = state | kReaderSpinWait;
|
|
|
|
} else {
|
|
|
|
// Active spinning.
|
|
|
|
state = atomic_load(&state_, memory_order_relaxed);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
|
|
|
|
memory_order_acquire)))
|
|
|
|
continue;
|
2021-07-18 16:19:37 +08:00
|
|
|
if (LIKELY(!locked))
|
2021-08-10 21:48:12 +08:00
|
|
|
return; // We've locked the mutex.
|
|
|
|
if (spin_iters > kMaxSpinIters) {
|
|
|
|
// We've incremented waiting readers, so now block.
|
|
|
|
readers_.Wait();
|
|
|
|
spin_iters = 0;
|
|
|
|
} else {
|
|
|
|
// We've set kReaderSpinWait, but we are still in active spinning.
|
|
|
|
}
|
|
|
|
reset_mask = ~kReaderSpinWait;
|
|
|
|
state = atomic_load(&state_, memory_order_relaxed);
|
|
|
|
}
|
2021-07-18 16:19:37 +08:00
|
|
|
}
|
|
|
|
|
2022-01-05 20:35:13 +08:00
|
|
|
void ReadUnlock() SANITIZER_RELEASE_SHARED() {
|
2021-07-23 16:41:40 +08:00
|
|
|
CheckedMutex::Unlock();
|
2021-07-18 16:19:37 +08:00
|
|
|
bool wake;
|
|
|
|
u64 new_state;
|
|
|
|
u64 state = atomic_load_relaxed(&state_);
|
|
|
|
do {
|
|
|
|
DCHECK_NE(state & kReaderLockMask, 0);
|
2021-08-10 21:48:12 +08:00
|
|
|
DCHECK_EQ(state & kWriterLock, 0);
|
2021-07-18 16:19:37 +08:00
|
|
|
new_state = state - kReaderLockInc;
|
2021-08-10 21:48:12 +08:00
|
|
|
wake = (new_state &
|
|
|
|
(kReaderLockMask | kWriterSpinWait | kReaderSpinWait)) == 0 &&
|
2021-07-18 16:19:37 +08:00
|
|
|
(new_state & kWaitingWriterMask) != 0;
|
|
|
|
if (wake)
|
|
|
|
new_state = (new_state - kWaitingWriterInc) | kWriterSpinWait;
|
|
|
|
} while (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
|
|
|
|
memory_order_release)));
|
|
|
|
if (UNLIKELY(wake))
|
|
|
|
writers_.Post();
|
|
|
|
}
|
|
|
|
|
|
|
|
// This function does not guarantee an explicit check that the calling thread
|
|
|
|
// is the thread which owns the mutex. This behavior, while more strictly
|
|
|
|
// correct, causes problems in cases like StopTheWorld, where a parent thread
|
|
|
|
// owns the mutex but a child checks that it is locked. Rather than
|
|
|
|
// maintaining complex state to work around those situations, the check only
|
|
|
|
// checks that the mutex is owned.
|
2022-01-05 20:35:13 +08:00
|
|
|
void CheckWriteLocked() const SANITIZER_CHECK_LOCKED() {
|
2021-07-18 16:19:37 +08:00
|
|
|
CHECK(atomic_load(&state_, memory_order_relaxed) & kWriterLock);
|
|
|
|
}
|
|
|
|
|
2022-01-05 20:35:13 +08:00
|
|
|
void CheckLocked() const SANITIZER_CHECK_LOCKED() { CheckWriteLocked(); }
|
2021-07-18 16:19:37 +08:00
|
|
|
|
2022-01-05 20:35:13 +08:00
|
|
|
void CheckReadLocked() const SANITIZER_CHECK_LOCKED() {
|
2021-07-18 16:19:37 +08:00
|
|
|
CHECK(atomic_load(&state_, memory_order_relaxed) & kReaderLockMask);
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
atomic_uint64_t state_ = {0};
|
|
|
|
Semaphore writers_;
|
|
|
|
Semaphore readers_;
|
|
|
|
|
|
|
|
// The state has 3 counters:
|
|
|
|
// - number of readers holding the lock,
|
|
|
|
// if non zero, the mutex is read-locked
|
|
|
|
// - number of waiting readers,
|
|
|
|
// if not zero, the mutex is write-locked
|
|
|
|
// - number of waiting writers,
|
|
|
|
// if non zero, the mutex is read- or write-locked
|
|
|
|
// And 2 flags:
|
|
|
|
// - writer lock
|
|
|
|
// if set, the mutex is write-locked
|
|
|
|
// - a writer is awake and spin-waiting
|
|
|
|
// the flag is used to prevent thundering herd problem
|
|
|
|
// (new writers are not woken if this flag is set)
|
2021-08-10 21:48:12 +08:00
|
|
|
// - a reader is awake and spin-waiting
|
2021-07-18 16:19:37 +08:00
|
|
|
//
|
2021-08-10 21:48:12 +08:00
|
|
|
// Both writers and readers use active spinning before blocking.
|
2021-07-18 16:19:37 +08:00
|
|
|
// But readers are more aggressive and always take the mutex
|
|
|
|
// if there are any other readers.
|
2021-08-10 21:48:12 +08:00
|
|
|
// After wake up both writers and readers compete to lock the
|
|
|
|
// mutex again. This is needed to allow repeated locks even in presence
|
|
|
|
// of other blocked threads.
|
2021-07-18 16:19:37 +08:00
|
|
|
static constexpr u64 kCounterWidth = 20;
|
|
|
|
static constexpr u64 kReaderLockShift = 0;
|
|
|
|
static constexpr u64 kReaderLockInc = 1ull << kReaderLockShift;
|
|
|
|
static constexpr u64 kReaderLockMask = ((1ull << kCounterWidth) - 1)
|
|
|
|
<< kReaderLockShift;
|
|
|
|
static constexpr u64 kWaitingReaderShift = kCounterWidth;
|
|
|
|
static constexpr u64 kWaitingReaderInc = 1ull << kWaitingReaderShift;
|
|
|
|
static constexpr u64 kWaitingReaderMask = ((1ull << kCounterWidth) - 1)
|
|
|
|
<< kWaitingReaderShift;
|
|
|
|
static constexpr u64 kWaitingWriterShift = 2 * kCounterWidth;
|
|
|
|
static constexpr u64 kWaitingWriterInc = 1ull << kWaitingWriterShift;
|
|
|
|
static constexpr u64 kWaitingWriterMask = ((1ull << kCounterWidth) - 1)
|
|
|
|
<< kWaitingWriterShift;
|
|
|
|
static constexpr u64 kWriterLock = 1ull << (3 * kCounterWidth);
|
|
|
|
static constexpr u64 kWriterSpinWait = 1ull << (3 * kCounterWidth + 1);
|
2021-08-10 21:48:12 +08:00
|
|
|
static constexpr u64 kReaderSpinWait = 1ull << (3 * kCounterWidth + 2);
|
|
|
|
|
|
|
|
static constexpr uptr kMaxSpinIters = 1500;
|
2021-07-18 16:19:37 +08:00
|
|
|
|
2021-07-28 19:08:49 +08:00
|
|
|
Mutex(LinkerInitialized) = delete;
|
2021-07-20 19:31:10 +08:00
|
|
|
Mutex(const Mutex &) = delete;
|
|
|
|
void operator=(const Mutex &) = delete;
|
2021-07-18 16:19:37 +08:00
|
|
|
};
|
|
|
|
|
2021-07-15 23:15:48 +08:00
|
|
|
void FutexWait(atomic_uint32_t *p, u32 cmp);
|
|
|
|
void FutexWake(atomic_uint32_t *p, u32 count);
|
|
|
|
|
2021-07-10 01:29:41 +08:00
|
|
|
template <typename MutexType>
|
2022-01-05 20:35:13 +08:00
|
|
|
class SANITIZER_SCOPED_LOCK GenericScopedLock {
|
2012-06-30 01:10:08 +08:00
|
|
|
public:
|
2022-01-05 20:35:13 +08:00
|
|
|
explicit GenericScopedLock(MutexType *mu) SANITIZER_ACQUIRE(mu) : mu_(mu) {
|
2012-06-30 01:10:08 +08:00
|
|
|
mu_->Lock();
|
|
|
|
}
|
|
|
|
|
2022-01-05 20:35:13 +08:00
|
|
|
~GenericScopedLock() SANITIZER_RELEASE() { mu_->Unlock(); }
|
2012-06-30 01:10:08 +08:00
|
|
|
|
|
|
|
private:
|
|
|
|
MutexType *mu_;
|
|
|
|
|
2021-07-01 11:48:23 +08:00
|
|
|
GenericScopedLock(const GenericScopedLock &) = delete;
|
|
|
|
void operator=(const GenericScopedLock &) = delete;
|
2012-06-30 01:10:08 +08:00
|
|
|
};
|
|
|
|
|
2021-07-10 01:29:41 +08:00
|
|
|
template <typename MutexType>
|
2022-01-05 20:35:13 +08:00
|
|
|
class SANITIZER_SCOPED_LOCK GenericScopedReadLock {
|
2012-06-30 01:10:08 +08:00
|
|
|
public:
|
2022-01-05 20:35:13 +08:00
|
|
|
explicit GenericScopedReadLock(MutexType *mu) SANITIZER_ACQUIRE(mu)
|
|
|
|
: mu_(mu) {
|
2012-06-30 01:10:08 +08:00
|
|
|
mu_->ReadLock();
|
|
|
|
}
|
|
|
|
|
2022-01-05 20:35:13 +08:00
|
|
|
~GenericScopedReadLock() SANITIZER_RELEASE() { mu_->ReadUnlock(); }
|
2012-06-30 01:10:08 +08:00
|
|
|
|
|
|
|
private:
|
|
|
|
MutexType *mu_;
|
|
|
|
|
2021-07-01 11:48:23 +08:00
|
|
|
GenericScopedReadLock(const GenericScopedReadLock &) = delete;
|
|
|
|
void operator=(const GenericScopedReadLock &) = delete;
|
2012-06-30 01:10:08 +08:00
|
|
|
};
|
|
|
|
|
2021-08-02 17:06:47 +08:00
|
|
|
template <typename MutexType>
|
2022-01-05 20:35:13 +08:00
|
|
|
class SANITIZER_SCOPED_LOCK GenericScopedRWLock {
|
2021-08-02 17:06:47 +08:00
|
|
|
public:
|
|
|
|
ALWAYS_INLINE explicit GenericScopedRWLock(MutexType *mu, bool write)
|
2022-01-05 20:35:13 +08:00
|
|
|
SANITIZER_ACQUIRE(mu)
|
2021-08-02 17:06:47 +08:00
|
|
|
: mu_(mu), write_(write) {
|
|
|
|
if (write_)
|
|
|
|
mu_->Lock();
|
|
|
|
else
|
|
|
|
mu_->ReadLock();
|
|
|
|
}
|
|
|
|
|
2022-01-05 20:35:13 +08:00
|
|
|
ALWAYS_INLINE ~GenericScopedRWLock() SANITIZER_RELEASE() {
|
2021-08-02 17:06:47 +08:00
|
|
|
if (write_)
|
|
|
|
mu_->Unlock();
|
|
|
|
else
|
|
|
|
mu_->ReadUnlock();
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
MutexType *mu_;
|
|
|
|
bool write_;
|
|
|
|
|
|
|
|
GenericScopedRWLock(const GenericScopedRWLock &) = delete;
|
|
|
|
void operator=(const GenericScopedRWLock &) = delete;
|
|
|
|
};
|
|
|
|
|
2012-08-30 18:02:48 +08:00
|
|
|
typedef GenericScopedLock<StaticSpinMutex> SpinMutexLock;
|
2021-07-20 19:31:10 +08:00
|
|
|
typedef GenericScopedLock<Mutex> Lock;
|
|
|
|
typedef GenericScopedReadLock<Mutex> ReadLock;
|
2021-08-02 17:06:47 +08:00
|
|
|
typedef GenericScopedRWLock<Mutex> RWLock;
|
2012-07-02 14:54:24 +08:00
|
|
|
|
2012-06-30 01:10:08 +08:00
|
|
|
} // namespace __sanitizer
|
|
|
|
|
|
|
|
#endif // SANITIZER_MUTEX_H
|