From a3b21b1d143a198bd37d6eb714762786e7696c4d Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Tue, 18 Mar 2014 08:30:14 +0000 Subject: [PATCH] tsan: better addr->object hashmap still experimental llvm-svn: 204126 --- .../sanitizer_common/sanitizer_addrhashmap.h | 273 ++++++++++++------ .../lib/sanitizer_common/sanitizer_mutex.h | 84 ++++++ 2 files changed, 270 insertions(+), 87 deletions(-) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_addrhashmap.h b/compiler-rt/lib/sanitizer_common/sanitizer_addrhashmap.h index 4eb0d406a97d..d837a4cec588 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_addrhashmap.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_addrhashmap.h @@ -22,7 +22,6 @@ namespace __sanitizer { // Concurrent uptr->T hashmap. // T must be a POD type, kSize is preferrably a prime but can be any number. -// The hashmap is fixed size, it crashes on overflow. // Usage example: // // typedef AddrHashMap Map; @@ -44,11 +43,24 @@ template class AddrHashMap { private: struct Cell { - StaticSpinMutex mtx; atomic_uintptr_t addr; T val; }; + struct AddBucket { + uptr cap; + uptr size; + Cell cells[1]; // variable len + }; + + static const uptr kBucketSize = 3; + + struct Bucket { + RWMutex mtx; + atomic_uintptr_t add; + Cell cells[kBucketSize]; + }; + public: AddrHashMap(); @@ -61,23 +73,23 @@ class AddrHashMap { bool exists() const; private: + friend AddrHashMap; AddrHashMap *map_; + Bucket *bucket_; Cell *cell_; uptr addr_; + uptr addidx_; bool created_; bool remove_; }; private: friend class Handle; - Cell *table_; + Bucket *table_; - static const uptr kLocked = 1; - static const uptr kRemoved = 2; - - Cell *acquire(uptr addr, bool remove, bool *created); - void release(uptr addr, bool remove, bool created, Cell *c); - uptr hash(uptr addr); + void acquire(Handle *h); + void release(Handle *h); + uptr calcHash(uptr addr); }; template @@ -86,12 +98,12 @@ AddrHashMap::Handle::Handle(AddrHashMap *map, uptr addr, map_ = map; addr_ = addr; remove_ = remove; - cell_ = map_->acquire(addr_, remove_, &created_); + map_->acquire(this); } template AddrHashMap::Handle::~Handle() { - map_->release(addr_, remove_, created_, cell_); + map_->release(this); } template @@ -111,96 +123,183 @@ bool AddrHashMap::Handle::exists() const { template AddrHashMap::AddrHashMap() { - table_ = (Cell*)MmapOrDie(kSize * sizeof(Cell), "AddrHashMap"); + table_ = (Bucket*)MmapOrDie(kSize * sizeof(table_[0]), "AddrHashMap"); } template -typename AddrHashMap::Cell *AddrHashMap::acquire(uptr addr, - bool remove, bool *created) { - // When we access the element associated with addr, - // we lock its home cell (the cell associated with hash(addr). - // If the element was just created or is going to be removed, - // we lock the cell in write mode. Otherwise we lock in read mode. - // The locking protects the object lifetime (it's not removed while - // somebody else accesses it). And also it helps to resolve concurrent - // inserts. - // Note that the home cell is not necessary the cell where the element is - // stored. - *created = false; - uptr h0 = hash(addr); - Cell *c0 = &table_[h0]; +void AddrHashMap::acquire(Handle *h) { + uptr addr = h->addr_; + uptr hash = calcHash(addr); + Bucket *b = &table_[hash]; + + h->created_ = false; + h->addidx_ = -1; + h->bucket_ = b; + h->cell_ = 0; + + // If we want to remove the element, we need exclusive access to the bucket, + // so skip the lock-free phase. + if (h->remove_) + goto locked; + + retry: // First try to find an existing element w/o read mutex. - { - uptr h = h0; - for (;;) { - Cell *c = &table_[h]; - uptr addr1 = atomic_load(&c->addr, memory_order_acquire); - if (addr1 == 0) // empty cell denotes end of the cell chain for the elem - break; - // Locked cell means that another thread can be concurrently inserting - // the same element, fallback to mutex. - if (addr1 == kLocked) - break; - if (addr1 == addr) // ok, found it - return c; - h++; - if (h == kSize) - h = 0; - CHECK_NE(h, h0); // made the full cycle - } - } - if (remove) - return 0; - // Now try to create it under the mutex. - c0->mtx.Lock(); - uptr h = h0; - for (;;) { - Cell *c = &table_[h]; + CHECK(!h->remove_); + // Check the embed cells. + for (uptr i = 0; i < kBucketSize; i++) { + Cell *c = &b->cells[i]; uptr addr1 = atomic_load(&c->addr, memory_order_acquire); - if (addr1 == addr) { // another thread has inserted it ahead of us - c0->mtx.Unlock(); - return c; + if (addr1 == addr) { + h->cell_ = c; + return; } - // Skip kLocked, since we hold the home cell mutex, it can't be our elem. - if ((addr1 == 0 || addr1 == kRemoved) && - atomic_compare_exchange_strong(&c->addr, &addr1, kLocked, - memory_order_acq_rel)) { - // we've created the element - *created = true; - return c; - } - h++; - if (h == kSize) - h = 0; - CHECK_NE(h, h0); // made the full cycle } -} -template -void AddrHashMap::release(uptr addr, bool remove, bool created, - Cell *c) { - if (c == 0) + // Check the add cells with read lock. + if (atomic_load(&b->add, memory_order_relaxed)) { + b->mtx.ReadLock(); + AddBucket *add = (AddBucket*)atomic_load(&b->add, memory_order_relaxed); + for (uptr i = 0; i < add->size; i++) { + Cell *c = &add->cells[i]; + uptr addr1 = atomic_load(&c->addr, memory_order_relaxed); + if (addr1 == addr) { + h->addidx_ = i; + h->cell_ = c; + return; + } + } + b->mtx.ReadUnlock(); + } + + locked: + // Re-check existence under write lock. + // Embed cells. + b->mtx.Lock(); + for (uptr i = 0; i < kBucketSize; i++) { + Cell *c = &b->cells[i]; + uptr addr1 = atomic_load(&c->addr, memory_order_relaxed); + if (addr1 == addr) { + if (h->remove_) { + h->cell_ = c; + return; + } + b->mtx.Unlock(); + goto retry; + } + } + + // Add cells. + AddBucket *add = (AddBucket*)atomic_load(&b->add, memory_order_relaxed); + if (add) { + for (uptr i = 0; i < add->size; i++) { + Cell *c = &add->cells[i]; + uptr addr1 = atomic_load(&c->addr, memory_order_relaxed); + if (addr1 == addr) { + if (h->remove_) { + h->addidx_ = i; + h->cell_ = c; + return; + } + b->mtx.Unlock(); + goto retry; + } + } + } + + // The element does not exist, no need to create it if we want to remove. + if (h->remove_) { + b->mtx.Unlock(); return; - // if we are going to remove, we must hold write lock - uptr addr1 = atomic_load(&c->addr, memory_order_relaxed); - if (created) { - // denote completion of insertion - atomic_store(&c->addr, addr, memory_order_release); - // unlock the home cell - uptr h0 = hash(addr); - Cell *c0 = &table_[h0]; - c0->mtx.Unlock(); - } else { - CHECK_EQ(addr, addr1); - if (remove) { - // denote that the cell is empty now - atomic_store(&c->addr, kRemoved, memory_order_release); + } + + // Now try to create it under the mutex. + h->created_ = true; + // See if we have a free embed cell. + for (uptr i = 0; i < kBucketSize; i++) { + Cell *c = &b->cells[i]; + uptr addr1 = atomic_load(&c->addr, memory_order_relaxed); + if (addr1 == 0) { + h->cell_ = c; + return; } } + + // Store in the add cells. + if (add == 0) { + // Allocate a new add array. + const uptr kInitSize = 64; + add = (AddBucket*)InternalAlloc(kInitSize); + add->cap = (kInitSize - sizeof(*add)) / sizeof(add->cells[0]) + 1; + add->size = 0; + atomic_store(&b->add, (uptr)add, memory_order_relaxed); + } + if (add->size == add->cap) { + // Grow existing add array. + uptr oldsize = sizeof(*add) + (add->cap - 1) * sizeof(add->cells[0]); + uptr newsize = oldsize * 2; + AddBucket *add1 = (AddBucket*)InternalAlloc(oldsize * 2); + add1->cap = (newsize - sizeof(*add)) / sizeof(add->cells[0]) + 1; + add1->size = add->size; + internal_memcpy(add1->cells, add->cells, add->size * sizeof(add->cells[0])); + InternalFree(add); + atomic_store(&b->add, (uptr)add1, memory_order_relaxed); + add = add1; + } + // Store. + uptr i = add->size++; + Cell *c = &add->cells[i]; + h->addidx_ = i; + h->cell_ = c; +} + +template +void AddrHashMap::release(Handle *h) { + if (h->cell_ == 0) + return; + Bucket *b = h->bucket_; + Cell *c = h->cell_; + uptr addr1 = atomic_load(&c->addr, memory_order_relaxed); + if (h->created_) { + // Denote completion of insertion. + CHECK_EQ(addr1, 0); + // After the following store, the element becomes available + // for lock-free reads. + atomic_store(&c->addr, h->addr_, memory_order_release); + b->mtx.Unlock(); + } else if (h->remove_) { + // Denote that the cell is empty now. + CHECK_EQ(addr1, h->addr_); + atomic_store(&c->addr, 0, memory_order_release); + // See if we need to compact the bucket. + AddBucket *add = (AddBucket*)atomic_load(&b->add, memory_order_relaxed); + if (h->addidx_ == -1) { + // Removed from embed array, move an add element into the freed cell. + if (add) { + uptr last = --add->size; + Cell *c1 = &add->cells[last]; + c->val = c1->val; + uptr addr1 = atomic_load(&c1->addr, memory_order_relaxed); + atomic_store(&c->addr, addr1, memory_order_release); + } + } else { + // Removed from add array, compact it. + uptr last = --add->size; + Cell *c1 = &add->cells[last]; + *c = *c1; + } + if (add && add->size == 0) { + // FIXME(dvyukov): free add? + } + b->mtx.Unlock(); + } else { + CHECK_EQ(addr1, h->addr_); + if (h->addidx_ != -1) + b->mtx.ReadUnlock(); + } } template -uptr AddrHashMap::hash(uptr addr) { +uptr AddrHashMap::calcHash(uptr addr) { addr += addr << 10; addr ^= addr >> 6; return addr % kSize; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h b/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h index e812fce90598..c7589f76ed3b 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h @@ -83,6 +83,88 @@ class BlockingMutex { uptr owner_; // for debugging }; +// Reader-writer spin mutex. +class RWMutex { + public: + RWMutex() { + atomic_store(&state_, kUnlocked, memory_order_relaxed); + } + + ~RWMutex() { + CHECK_EQ(atomic_load(&state_, memory_order_relaxed), kUnlocked); + } + + void Lock() { + u32 cmp = kUnlocked; + if (atomic_compare_exchange_strong(&state_, &cmp, kWriteLock, + memory_order_acquire)) + return; + LockSlow(); + } + + void Unlock() { + u32 prev = atomic_fetch_sub(&state_, kWriteLock, memory_order_release); + DCHECK_NE(prev & kWriteLock, 0); + (void)prev; + } + + void ReadLock() { + u32 prev = atomic_fetch_add(&state_, kReadLock, memory_order_acquire); + if ((prev & kWriteLock) == 0) + return; + ReadLockSlow(); + } + + void ReadUnlock() { + u32 prev = atomic_fetch_sub(&state_, kReadLock, memory_order_release); + DCHECK_EQ(prev & kWriteLock, 0); + DCHECK_GT(prev & ~kWriteLock, 0); + (void)prev; + } + + void CheckLocked() { + CHECK_NE(atomic_load(&state_, memory_order_relaxed), kUnlocked); + } + + private: + atomic_uint32_t state_; + + enum { + kUnlocked = 0, + kWriteLock = 1, + kReadLock = 2 + }; + + void NOINLINE LockSlow() { + for (int i = 0;; i++) { + if (i < 10) + proc_yield(10); + else + internal_sched_yield(); + u32 cmp = atomic_load(&state_, memory_order_relaxed); + if (cmp == kUnlocked && + atomic_compare_exchange_weak(&state_, &cmp, kWriteLock, + memory_order_acquire)) + return; + } + } + + void NOINLINE ReadLockSlow() { + for (int i = 0;; i++) { + if (i < 10) + proc_yield(10); + else + internal_sched_yield(); + u32 prev = atomic_load(&state_, memory_order_acquire); + if ((prev & kWriteLock) == 0) + return; + } + } + + RWMutex(const RWMutex&); + void operator = (const RWMutex&); +}; + template class GenericScopedLock { public: @@ -123,6 +205,8 @@ class GenericScopedReadLock { typedef GenericScopedLock SpinMutexLock; typedef GenericScopedLock BlockingMutexLock; +typedef GenericScopedLock RWMutexLock; +typedef GenericScopedReadLock RWMutexReadLock; } // namespace __sanitizer