asan: Refactor asan memory quarantine.

llvm-svn: 172183

compiler-rt/lib/asan/asan_allocator.h

@@ -23,7 +23,7 @@
 // to a new one (version 2). The change is quite intrusive so both allocators
 // will co-exist in the source base for a while. The actual allocator is chosen
 // at build time by redefining this macrozz.
-#define ASAN_ALLOCATOR_VERSION 1
+#define ASAN_ALLOCATOR_VERSION 2
 
 namespace __asan {
 
@@ -98,16 +98,20 @@ class AsanChunkFifoList: public IntrusiveList<AsanChunk> {
 
 struct AsanThreadLocalMallocStorage {
   explicit AsanThreadLocalMallocStorage(LinkerInitialized x)
-      : quarantine_(x) { }
+#if ASAN_ALLOCATOR_VERSION == 1
+      : quarantine_(x)
+#endif
+      { }
   AsanThreadLocalMallocStorage() {
     CHECK(REAL(memset));
     REAL(memset)(this, 0, sizeof(AsanThreadLocalMallocStorage));
   }
 
-  AsanChunkFifoList quarantine_;
 #if ASAN_ALLOCATOR_VERSION == 1
+  AsanChunkFifoList quarantine_;
   AsanChunk *free_lists_[kNumberOfSizeClasses];
 #else
+  uptr quarantine_cache[16];
   uptr allocator2_cache[1024];  // Opaque.
 #endif
   void CommitBack();

compiler-rt/lib/asan/asan_allocator2.cc

@@ -27,6 +27,7 @@
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_list.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_quarantine.h"
 
 namespace __asan {
 
@@ -92,15 +93,6 @@ static const uptr kMaxThreadLocalQuarantine =
 
 static const uptr kReturnOnZeroMalloc = 2048;  // Zero page is protected.
 
-static int inited = 0;
-
-static void Init() {
-  if (inited) return;
-  __asan_init();
-  inited = true;  // this must happen before any threads are created.
-  allocator.Init();
-}
-
 // Every chunk of memory allocated by this allocator can be in one of 3 states:
 // CHUNK_AVAILABLE: the chunk is in the free list and ready to be allocated.
 // CHUNK_ALLOCATED: the chunk is allocated and not yet freed.
@@ -246,31 +238,26 @@ void AsanChunkView::GetFreeStack(StackTrace *stack) {
                                 chunk_->FreeStackSize());
 }
 
-class Quarantine: public AsanChunkFifoList {
- public:
-  void SwallowThreadLocalQuarantine(AsanThreadLocalMallocStorage *ms) {
-    AsanChunkFifoList *q = &ms->quarantine_;
-    if (!q->size()) return;
-    AsanChunkFifoList tmp;
-    uptr max_size = flags()->quarantine_size;
-    {
-      SpinMutexLock l(&mutex_);
-      PushList(q);
-      while (size() > max_size)
-        tmp.Push(Pop());
-    }
-    while (!tmp.empty())
-      Deallocate(tmp.Pop(), ms);
+struct QuarantineCallback;
+typedef Quarantine<QuarantineCallback, AsanChunk> AsanQuarantine;
+typedef AsanQuarantine::Cache QuarantineCache;
+static AsanQuarantine quarantine(LINKER_INITIALIZED);
+static QuarantineCache fallback_quarantine_cache(LINKER_INITIALIZED);
+static AllocatorCache fallback_allocator_cache;
+static SpinMutex fallback_mutex;
+
+QuarantineCache *GetQuarantineCache(AsanThreadLocalMallocStorage *ms) {
+  DCHECK(ms);
+  DCHECK_LE(sizeof(QuarantineCache), sizeof(ms->quarantine_cache));
+  return reinterpret_cast<QuarantineCache *>(ms->quarantine_cache);
+}
+
+struct QuarantineCallback {
+  explicit QuarantineCallback(AllocatorCache *cache)
+      : cache_(cache) {
   }
 
-  void BypassThreadLocalQuarantine(AsanChunk *m) {
-    SpinMutexLock l(&mutex_);
-    Push(m);
-  }
-
- private:
-  static void Deallocate(AsanChunk *m, AsanThreadLocalMallocStorage *ms) {
-    CHECK(m);
+  void Recycle(AsanChunk *m) {
     CHECK(m->chunk_state == CHUNK_QUARANTINE);
     m->chunk_state = CHUNK_AVAILABLE;
     CHECK_NE(m->alloc_tid, kInvalidTid);
@@ -290,34 +277,27 @@ class Quarantine: public AsanChunkFifoList {
     thread_stats.real_frees++;
     thread_stats.really_freed += m->UsedSize();
 
-    allocator.Deallocate(GetAllocatorCache(ms), p);
+    allocator.Deallocate(cache_, p);
   }
-  SpinMutex mutex_;
+
+  void *Allocate(uptr size) {
+    return allocator.Allocate(cache_, size, 0, false);
+  }
+
+  void Deallocate(void *p) {
+    allocator.Deallocate(cache_, p);
+  }
+
+  AllocatorCache *cache_;
 };
 
-static Quarantine quarantine;
-
-void AsanChunkFifoList::PushList(AsanChunkFifoList *q) {
-  CHECK(q->size() > 0);
-  size_ += q->size();
-  append_back(q);
-  q->clear();
-}
-
-void AsanChunkFifoList::Push(AsanChunk *n) {
-  push_back(n);
-  size_ += n->UsedSize();
-}
-
-// Interesting performance observation: this function takes up to 15% of overal
-// allocator time. That's because *first_ has been evicted from cache long time
-// ago. Not sure if we can or want to do anything with this.
-AsanChunk *AsanChunkFifoList::Pop() {
-  CHECK(first_);
-  AsanChunk *res = front();
-  size_ -= res->UsedSize();
-  pop_front();
-  return res;
+static void Init() {
+  static int inited = 0;
+  if (inited) return;
+  __asan_init();
+  inited = true;  // this must happen before any threads are created.
+  allocator.Init();
+  quarantine.Init((uptr)flags()->quarantine_size, kMaxThreadLocalQuarantine);
 }
 
 static void *Allocate(uptr size, uptr alignment, StackTrace *stack,
@@ -468,13 +448,13 @@ static void Deallocate(void *ptr, StackTrace *stack, AllocType alloc_type) {
 
   // Push into quarantine.
   if (t) {
-    AsanChunkFifoList &q = t->malloc_storage().quarantine_;
-    q.Push(m);
-
-    if (q.size() > kMaxThreadLocalQuarantine)
-      quarantine.SwallowThreadLocalQuarantine(&t->malloc_storage());
+    AsanThreadLocalMallocStorage *ms = &t->malloc_storage();
+    AllocatorCache *ac = GetAllocatorCache(ms);
+    quarantine.Put(GetQuarantineCache(ms), QuarantineCallback(ac), m);
   } else {
-    quarantine.BypassThreadLocalQuarantine(m);
+    SpinMutexLock l(&fallback_mutex);
+    AllocatorCache *ac = &fallback_allocator_cache;
+    quarantine.Put(&fallback_quarantine_cache, QuarantineCallback(ac), m);
   }
 
   ASAN_FREE_HOOK(ptr);
@@ -586,7 +566,8 @@ AsanChunkView FindHeapChunkByAddress(uptr addr) {
 }
 
 void AsanThreadLocalMallocStorage::CommitBack() {
-  quarantine.SwallowThreadLocalQuarantine(this);
+  AllocatorCache *ac = GetAllocatorCache(this);
+  quarantine.Drain(GetQuarantineCache(this), QuarantineCallback(ac));
   allocator.SwallowCache(GetAllocatorCache(this));
 }
 

compiler-rt/lib/sanitizer_common/CMakeLists.txt

@@ -38,6 +38,7 @@ set(SANITIZER_HEADERS
   sanitizer_placement_new.h
   sanitizer_platform_interceptors.h
   sanitizer_procmaps.h
+  sanitizer_quarantine.h
   sanitizer_report_decorator.h
   sanitizer_stackdepot.h
   sanitizer_stacktrace.h

compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h

@@ -0,0 +1,107 @@
+//===-- sanitizer_quarantine.h ----------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Memory quarantine for AddressSanitizer and potentially other tools.
+// Quarantine caches some specified amount of memory in per-thread caches,
+// then evicts to global FIFO queue. When the queue reaches specified threshold,
+// oldest memory is recycled.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_QUARANTINE_H
+#define SANITIZER_QUARANTINE_H
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_mutex.h"
+
+namespace __sanitizer {
+
+template<typename Node> class QuarantineCache;
+
+// The callback interface is:
+// Callback cb;
+// Node *ptr;
+// cb.Recycle(ptr);
+template<typename Callback, typename Node>
+class Quarantine {
+ public:
+  typedef QuarantineCache<Node> Cache;
+
+  explicit Quarantine(LinkerInitialized)
+      : cache_(LINKER_INITIALIZED) {
+  }
+
+  void Init(uptr size, uptr cache_size) {
+    max_size_ = size;
+    max_cache_size_ = cache_size;
+  }
+
+  void Put(Cache *c, Callback cb, Node *ptr) {
+    c->Enqueue(ptr);
+    if (c->Size() > max_cache_size_)
+      Drain(c, cb);
+  }
+
+  void Drain(Cache *c, Callback cb) {
+    SpinMutexLock l(&mutex_);
+    while (Node *ptr = c->Dequeue())
+      cache_.Enqueue(ptr);
+    while (cache_.Size() > max_size_) {
+      Node *ptr = cache_.Dequeue();
+      cb.Recycle(ptr);
+    }
+  }
+
+ private:
+  SpinMutex mutex_;
+  uptr max_size_;
+  uptr max_cache_size_;
+  Cache cache_;
+};
+
+// Per-thread cache of memory blocks (essentially FIFO queue).
+template<typename Node>
+class QuarantineCache {
+ public:
+  explicit QuarantineCache(LinkerInitialized) {
+  }
+
+  uptr Size() const {
+    return size_;
+  }
+
+  void Enqueue(Node *ptr) {
+    size_ += ptr->UsedSize();
+    ptr->next = 0;
+    if (tail_)
+      tail_->next = ptr;
+    else
+      head_ = ptr;
+    tail_ = ptr;
+  }
+
+  Node *Dequeue() {
+    Node *ptr = head_;
+    if (ptr == 0)
+      return 0;
+    head_ = ptr->next;
+    if (head_ == 0)
+      tail_ = 0;
+    size_ -= ptr->UsedSize();
+    return ptr;
+  }
+
+ private:
+  Node *head_;
+  Node *tail_;
+  uptr size_;
+};
+}
+
+#endif  // #ifndef SANITIZER_QUARANTINE_H